Skip to content

M
mybee
Commit 95a16dc9 authored by acud's avatar acud Committed by GitHub
Browse files
Options

cmd: deprecate utility binaries (#1579)

parent f723df53
Hide whitespace changes
Inline Side-by-side
Showing with 0 additions and 723 deletions
Makefile
View file @ 95a16dc9
......@@ -18,12 +18,6 @@ binary: dist FORCE
$(GO) version
$(GO) build -trimpath -ldflags "$(LDFLAGS)" -o dist/bee ./cmd/bee
.PHONY: binaries
binaries: binary
$(GO) build -trimpath -ldflags "$(LDFLAGS)" -o dist/bee-file ./cmd/bee-file
$(GO) build -trimpath -ldflags "$(LDFLAGS)" -o dist/bee-join ./cmd/bee-join
$(GO) build -trimpath -ldflags "$(LDFLAGS)" -o dist/bee-split ./cmd/bee-split
dist:
mkdir $@
......
cmd/bee-file/main.go deleted 100644 → 0
View file @ f723df53
// Copyright 2020 The Swarm Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package main
import (
"errors"
"fmt"
"os"
"github.com/spf13/cobra"
)
var (
filename string // flag variable, filename to use in metadata
mimeType string // flag variable, mime type to use in metadata
outDir string // flag variable, output dir for fsStore
outFileForce bool // flag variable, overwrite output file if exists
host string // flag variable, http api host
port int // flag variable, http api port
useHttp bool // flag variable, skips http api if not set
ssl bool // flag variable, uses https for api if set
retrieve bool // flag variable, if set will resolve and retrieve referenced file
verbosity string // flag variable, debug level
)
// Entry is the underlying procedure for the CLI command
func Entry(cmd *cobra.Command, args []string) (err error) {
return errors.New("command is deprecated")
}
func main() {
c := &cobra.Command{
Use: "entry <reference>",
Short: "Create or resolve a file entry",
Long: `Creates a file entry, or retrieve the data referenced by the entry and its metadata.
Example:
$ bee-file --mime-type text/plain --filename foo.txt 2387e8e7d8a48c2a9339c97c1dc3461a9a7aa07e994c5cb8b38fd7c1b3e6ea48
> 94434d3312320fab70428c39b79dffb4abc3dbedf3e1562384a61ceaf8a7e36b
$ bee-file --output-dir /tmp 94434d3312320fab70428c39b79dffb4abc3dbedf3e1562384a61ceaf8a7e36b
$ cat /tmp/bar.txt
Creating a file entry:
The default file name is the hex representation of the swarm hash passed as argument, and the default mime-type is application/octet-stream. Both can be explicitly set with --filename and --mime-type respectively. If --output-dir is given, the metadata and entry chunks are written to the specified directory.
Resolving a file entry:
If --output-dir is set, the retrieved file will be written to the speficied directory. Otherwise it will be written to the current directory. Use -f to force overwriting an existing file.`,
RunE: Entry,
SilenceUsage: true,
}
c.Flags().StringVar(&filename, "filename", "", "filename to use in entry")
c.Flags().StringVar(&mimeType, "mime-type", "", "mime-type to use in collection")
c.Flags().BoolVarP(&outFileForce, "force", "f", false, "overwrite existing output file")
c.Flags().StringVarP(&outDir, "output-dir", "d", "", "save directory")
c.Flags().StringVar(&host, "host", "127.0.0.1", "api host")
c.Flags().IntVar(&port, "port", 1633, "api port")
c.Flags().BoolVar(&ssl, "ssl", false, "use ssl")
c.Flags().BoolVarP(&retrieve, "retrieve", "r", false, "retrieve file from referenced entry")
c.Flags().BoolVar(&useHttp, "http", false, "save entry to bee http api")
c.Flags().StringVar(&verbosity, "info", "0", "log verbosity level 0=silent, 1=error, 2=warn, 3=info, 4=debug, 5=trace")
c.SetOutput(c.OutOrStdout())
err := c.Execute()
if err != nil {
fmt.Fprintln(os.Stderr, err.Error())
os.Exit(1)
}
}
cmd/bee-join/main.go deleted 100644 → 0
View file @ f723df53
// Copyright 2020 The Swarm Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package main
import (
"fmt"
"os"
"path/filepath"
cmdfile "github.com/ethersphere/bee/cmd/internal/file"
"github.com/ethersphere/bee/pkg/file"
"github.com/ethersphere/bee/pkg/file/joiner"
"github.com/ethersphere/bee/pkg/logging"
"github.com/ethersphere/bee/pkg/storage"
"github.com/ethersphere/bee/pkg/swarm"
"github.com/spf13/cobra"
)
var (
outFilePath string // flag variable, output file
outFileForce bool // flag variable, overwrite output file if exists
host string // flag variable, http api host
port int // flag variable, http api port
ssl bool // flag variable, uses https for api if set
indir string // flag variable, directory to retrieve chunks from
verbosity string // flag variable, debug level
logger logging.Logger
)
// Join is the underlying procedure for the CLI command
func Join(cmd *cobra.Command, args []string) (err error) {
logger, err = cmdfile.SetLogger(cmd, verbosity)
if err != nil {
return err
}
// if output file is specified, create it if it does not exist
var outFile *os.File
if outFilePath != "" {
// make sure we have full path
outDir := filepath.Dir(outFilePath)
if outDir != "." {
err := os.MkdirAll(outDir, 0o777) // skipcq: GSC-G301
if err != nil {
return err
}
}
// protect any existing file unless explicitly told not to
outFileFlags := os.O_CREATE | os.O_WRONLY
if outFileForce {
outFileFlags |= os.O_TRUNC
} else {
outFileFlags |= os.O_EXCL
}
// open the file
outFile, err = os.OpenFile(outFilePath, outFileFlags, 0o666) // skipcq: GSC-G302
if err != nil {
return err
}
defer outFile.Close()
logger.Debugf("writing to %s", outFilePath)
} else {
outFile = os.Stdout
logger.Debugf("writing to stdout")
}
// process the reference to retrieve
addr, err := swarm.ParseHexAddress(args[0])
if err != nil {
return err
}
// initialize interface with backend store
// either from directory if set, or HTTP API if not set
var store storage.Getter
if indir != "" {
store = cmdfile.NewFsStore(indir)
} else {
store = cmdfile.NewApiStore(host, port, ssl)
}
// create the join and get its data reader
j, _, err := joiner.New(cmd.Context(), store, addr)
if err != nil {
return err
}
_, err = file.JoinReadAll(cmd.Context(), j, outFile)
return err
}
func main() {
c := &cobra.Command{
Use: "join [hash]",
Args: cobra.ExactArgs(1),
Short: "Retrieve data from Swarm",
Long: `Assembles chunked data from referenced by a root Swarm Hash.
Will output retrieved data to stdout.`,
RunE: Join,
SilenceUsage: true,
}
c.Flags().StringVarP(&outFilePath, "output-file", "o", "", "file to write output to")
c.Flags().BoolVarP(&outFileForce, "force", "f", false, "overwrite existing output file")
c.Flags().StringVar(&host, "host", "127.0.0.1", "api host")
c.Flags().IntVar(&port, "port", 1633, "api port")
c.Flags().BoolVar(&ssl, "ssl", false, "use ssl")
c.Flags().StringVarP(&indir, "input-dir", "i", "", "retrieve chunks from directory")
c.Flags().StringVar(&verbosity, "info", "0", "log verbosity level 0=silent, 1=error, 2=warn, 3=info, 4=debug, 5=trace")
c.SetOutput(c.OutOrStdout())
err := c.Execute()
if err != nil {
fmt.Fprintln(os.Stderr, err.Error())
os.Exit(1)
}
}
cmd/bee-split/main.go deleted 100644 → 0
View file @ f723df53
// Copyright 2020 The Swarm Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package main
import (
"context"
"errors"
"fmt"
"io"
"io/ioutil"
"os"
cmdfile "github.com/ethersphere/bee/cmd/internal/file"
"github.com/ethersphere/bee/pkg/file/splitter"
"github.com/ethersphere/bee/pkg/logging"
"github.com/ethersphere/bee/pkg/storage"
"github.com/spf13/cobra"
)
var (
outdir string // flag variable, output dir for fsStore
inputLength int64 // flag variable, limit of data input
host string // flag variable, http api host
port int // flag variable, http api port
useHttp bool // flag variable, skips http api if not set
ssl bool // flag variable, uses https for api if set
verbosity string // flag variable, debug level
logger logging.Logger
)
// Split is the underlying procedure for the CLI command
func Split(cmd *cobra.Command, args []string) (err error) {
logger, err = cmdfile.SetLogger(cmd, verbosity)
if err != nil {
return err
}
// if one arg is set, this is the input file
// if not, we are reading from standard input
var infile io.ReadCloser
if len(args) > 0 {
// get the file length
info, err := os.Stat(args[0])
if err != nil {
return err
}
fileLength := info.Size()
// check if we are limiting the input, and if the limit is valid
if inputLength > 0 {
if inputLength > fileLength {
return fmt.Errorf("input data length set to %d on file with length %d", inputLength, fileLength)
}
} else {
inputLength = fileLength
}
// open file and wrap in limiter
f, err := os.Open(args[0])
if err != nil {
return err
}
fileReader := io.LimitReader(f, inputLength)
infile = ioutil.NopCloser(fileReader)
logger.Debugf("using %d bytes from file %s as input", fileLength, args[0])
} else {
// this simple splitter is too stupid to handle open-ended input, sadly
if inputLength == 0 {
return errors.New("must specify length of input on stdin")
}
stdinReader := io.LimitReader(os.Stdin, inputLength)
infile = ioutil.NopCloser(stdinReader)
logger.Debugf("using %d bytes from standard input", inputLength)
}
// add the fsStore and/or apiStore, depending on flags
stores := cmdfile.NewTeeStore()
if outdir != "" {
err := os.MkdirAll(outdir, 0o777) // skipcq: GSC-G301
if err != nil {
return err
}
store := cmdfile.NewFsStore(outdir)
stores.Add(store)
logger.Debugf("using directory %s for output", outdir)
}
if useHttp {
store := cmdfile.NewApiStore(host, port, ssl)
stores.Add(store)
logger.Debugf("using bee http (ssl=%v) api on %s:%d for output", ssl, host, port)
}
// split and rule
s := splitter.NewSimpleSplitter(stores, storage.ModePutUpload)
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
addr, err := s.Split(ctx, infile, inputLength, false)
if err != nil {
return err
}
// output the resulting hash
cmd.Println(addr)
return nil
}
func main() {
c := &cobra.Command{
Use: "split [datafile]",
Args: cobra.RangeArgs(0, 1),
Short: "Split data into swarm chunks",
Long: `Creates and stores Swarm chunks from input data.
If datafile is not given, data will be read from standard in. In this case the --count flag must be set
to the length of the input.
The application will expect to transmit the chunks to the bee HTTP API, unless the --no-http flag has been set.
If --output-dir is set, the chunks will be saved to the file system, using the flag argument as destination directory.
Chunks are saved in individual files, and the file names will be the hex addresses of the chunks.`,
RunE: Split,
SilenceUsage: true,
}
c.Flags().StringVarP(&outdir, "output-dir", "d", "", "saves chunks to given directory")
c.Flags().Int64VarP(&inputLength, "count", "c", 0, "read at most this many bytes")
c.Flags().StringVar(&host, "host", "127.0.0.1", "api host")
c.Flags().IntVar(&port, "port", 1633, "api port")
c.Flags().BoolVar(&ssl, "ssl", false, "use ssl")
c.Flags().BoolVar(&useHttp, "http", false, "save chunks to bee http api")
c.Flags().StringVar(&verbosity, "info", "0", "log verbosity level 0=silent, 1=error, 2=warn, 3=info, 4=debug, 5=trace")
c.SetOutput(c.OutOrStdout())
err := c.Execute()
if err != nil {
fmt.Fprintln(os.Stderr, err.Error())
os.Exit(1)
}
}
cmd/internal/file/io.go deleted 100644 → 0
View file @ f723df53
// Copyright 2020 The Swarm Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package file
import (
"bytes"
"context"
"errors"
"fmt"
"io"
"io/ioutil"
"net/http"
"net/url"
"path/filepath"
"strings"
"github.com/sirupsen/logrus"
"github.com/spf13/cobra"
"github.com/ethersphere/bee/pkg/logging"
"github.com/ethersphere/bee/pkg/storage"
"github.com/ethersphere/bee/pkg/swarm"
)
// nopWriteCloser wraps a io.Writer in the same manner as ioutil.NopCloser does
// with an io.Reader.
type nopWriteCloser struct {
io.Writer
}
// NopWriteCloser returns a new io.WriteCloser with the given writer as io.Writer.
func NopWriteCloser(w io.Writer) io.WriteCloser {
return &nopWriteCloser{
Writer: w,
}
}
// Close implements io.Closer
func (n *nopWriteCloser) Close() error {
return nil
}
// PutGetter wraps both storage.Putter and storage.Getter interfaces
type PutGetter interface {
storage.Putter
storage.Getter
}
// TeeStore provides a storage.Putter that can put to multiple underlying storage.Putters.
type TeeStore struct {
putters []storage.Putter
}
// NewTeeStore creates a new TeeStore.
func NewTeeStore() *TeeStore {
return &TeeStore{}
}
// Add adds a storage.Putter.
func (t *TeeStore) Add(putter storage.Putter) {
t.putters = append(t.putters, putter)
}
// Put implements storage.Putter.
func (t *TeeStore) Put(ctx context.Context, mode storage.ModePut, chs ...swarm.Chunk) (exist []bool, err error) {
for _, putter := range t.putters {
_, err := putter.Put(ctx, mode, chs...)
if err != nil {
return nil, err
}
}
exist = make([]bool, len(chs))
return exist, nil
}
// FsStore provides a storage.Putter that writes chunks directly to the filesystem.
// Each chunk is a separate file, where the hex address of the chunk is the file name.
type FsStore struct {
path string
}
// NewFsStore creates a new FsStore.
func NewFsStore(path string) PutGetter {
return &FsStore{
path: path,
}
}
// Put implements storage.Putter.
func (f *FsStore) Put(ctx context.Context, mode storage.ModePut, chs ...swarm.Chunk) (exist []bool, err error) {
for _, ch := range chs {
chunkPath := filepath.Join(f.path, ch.Address().String())
err := ioutil.WriteFile(chunkPath, ch.Data(), 0o666)
if err != nil {
return nil, err
}
}
exist = make([]bool, len(chs))
return exist, nil
}
// Get implements storage.Getter.
func (f *FsStore) Get(ctx context.Context, mode storage.ModeGet, address swarm.Address) (ch swarm.Chunk, err error) {
chunkPath := filepath.Join(f.path, address.String())
data, err := ioutil.ReadFile(chunkPath)
if err != nil {
return nil, err
}
return swarm.NewChunk(address, data), nil
}
// ApiStore provies a storage.Putter that adds chunks to swarm through the HTTP chunk API.
type ApiStore struct {
Client *http.Client
baseUrl string
}
// NewApiStore creates a new ApiStore.
func NewApiStore(host string, port int, ssl bool) PutGetter {
scheme := "http"
if ssl {
scheme += "s"
}
u := &url.URL{
Host: fmt.Sprintf("%s:%d", host, port),
Scheme: scheme,
Path: "chunks",
}
return &ApiStore{
Client: http.DefaultClient,
baseUrl: u.String(),
}
}
// Put implements storage.Putter.
func (a *ApiStore) Put(ctx context.Context, mode storage.ModePut, chs ...swarm.Chunk) (exist []bool, err error) {
for _, ch := range chs {
buf := bytes.NewReader(ch.Data())
url := strings.Join([]string{a.baseUrl}, "/")
res, err := a.Client.Post(url, "application/octet-stream", buf)
if err != nil {
return nil, err
}
if res.StatusCode != http.StatusOK {
return nil, fmt.Errorf("upload failed: %v", res.Status)
}
}
exist = make([]bool, len(chs))
return exist, nil
}
// Get implements storage.Getter.
func (a *ApiStore) Get(ctx context.Context, mode storage.ModeGet, address swarm.Address) (ch swarm.Chunk, err error) {
addressHex := address.String()
url := strings.Join([]string{a.baseUrl, addressHex}, "/")
res, err := http.DefaultClient.Get(url)
if err != nil {
return nil, err
}
if res.StatusCode != http.StatusOK {
return nil, fmt.Errorf("chunk %s not found", addressHex)
}
chunkData, err := ioutil.ReadAll(res.Body)
if err != nil {
return nil, err
}
ch = swarm.NewChunk(address, chunkData)
return ch, nil
}
// LimitWriteCloser limits the output from the application.
type LimitWriteCloser struct {
io.WriteCloser
total int64
limit int64
}
// NewLimitWriteCloser creates a new LimitWriteCloser.
func NewLimitWriteCloser(w io.WriteCloser, c int64) io.WriteCloser {
return &LimitWriteCloser{
WriteCloser: w,
limit: c,
}
}
// Write implements io.Writer.
func (l *LimitWriteCloser) Write(b []byte) (int, error) {
if l.total+int64(len(b)) > l.limit {
return 0, errors.New("overflow")
}
c, err := l.WriteCloser.Write(b)
l.total += int64(c)
return c, err
}
func SetLogger(cmd *cobra.Command, verbosityString string) (logger logging.Logger, err error) {
v := strings.ToLower(verbosityString)
switch v {
case "0", "silent":
logger = logging.New(ioutil.Discard, 0)
case "1", "error":
logger = logging.New(cmd.OutOrStderr(), logrus.ErrorLevel)
case "2", "warn":
logger = logging.New(cmd.OutOrStderr(), logrus.WarnLevel)
case "3", "info":
logger = logging.New(cmd.OutOrStderr(), logrus.InfoLevel)
case "4", "debug":
logger = logging.New(cmd.OutOrStderr(), logrus.DebugLevel)
case "5", "trace":
logger = logging.New(cmd.OutOrStderr(), logrus.TraceLevel)
default:
return nil, fmt.Errorf("unknown verbosity level %q", v)
}
return logger, nil
}
cmd/internal/file/io_test.go deleted 100644 → 0
View file @ f723df53
// Copyright 2020 The Swarm Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package file_test
import (
"bytes"
"context"
"io/ioutil"
"net/http/httptest"
"net/url"
"os"
"path/filepath"
"strconv"
"testing"
cmdfile "github.com/ethersphere/bee/cmd/internal/file"
"github.com/ethersphere/bee/pkg/api"
"github.com/ethersphere/bee/pkg/logging"
statestore "github.com/ethersphere/bee/pkg/statestore/mock"
"github.com/ethersphere/bee/pkg/storage"
"github.com/ethersphere/bee/pkg/storage/mock"
testingc "github.com/ethersphere/bee/pkg/storage/testing"
"github.com/ethersphere/bee/pkg/swarm"
"github.com/ethersphere/bee/pkg/tags"
)
const (
hashOfFoo = "2387e8e7d8a48c2a9339c97c1dc3461a9a7aa07e994c5cb8b38fd7c1b3e6ea48"
)
// TestApiStore verifies that the api store layer does not distort data, and that same
// data successfully posted can be retrieved from http backend.
func TestApiStore(t *testing.T) {
storer := mock.NewStorer()
ctx := context.Background()
srvUrl := newTestServer(t, storer)
host := srvUrl.Hostname()
port, err := strconv.Atoi(srvUrl.Port())
if err != nil {
t.Fatal(err)
}
a := cmdfile.NewApiStore(host, port, false)
ch := testingc.GenerateTestRandomChunk()
_, err = a.Put(ctx, storage.ModePutUpload, ch)
if err != nil {
t.Fatal(err)
}
_, err = storer.Get(ctx, storage.ModeGetRequest, ch.Address())
if err != nil {
t.Fatal(err)
}
chResult, err := a.Get(ctx, storage.ModeGetRequest, ch.Address())
if err != nil {
t.Fatal(err)
}
if !ch.Equal(chResult) {
t.Fatal("chunk mismatch")
}
}
// TestFsStore verifies that the fs store layer does not distort data, and that the
// resulting stored data matches what is submitted.
func TestFsStore(t *testing.T) {
tmpPath, err := ioutil.TempDir("", "cli-test")
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(tmpPath)
storer := cmdfile.NewFsStore(tmpPath)
chunkAddr := swarm.MustParseHexAddress(hashOfFoo)
chunkData := []byte("foo")
ch := swarm.NewChunk(chunkAddr, chunkData)
ctx := context.Background()
_, err = storer.Put(ctx, storage.ModePutUpload, ch)
if err != nil {
t.Fatal(err)
}
chunkFilename := filepath.Join(tmpPath, hashOfFoo)
chunkDataResult, err := ioutil.ReadFile(chunkFilename)
if err != nil {
t.Fatal(err)
}
chResult := swarm.NewChunk(chunkAddr, chunkDataResult)
if !ch.Equal(chResult) {
t.Fatal("chunk mismatch")
}
}
// TestTeeStore verifies that the TeeStore writes to all added stores.
func TestTeeStore(t *testing.T) {
storeFee := mock.NewStorer()
storeFi := mock.NewStorer()
storeFo := mock.NewStorer()
storeFum := cmdfile.NewTeeStore()
storeFum.Add(storeFee)
storeFum.Add(storeFi)
storeFum.Add(storeFo)
chunkAddr := swarm.MustParseHexAddress(hashOfFoo)
chunkData := []byte("foo")
ch := swarm.NewChunk(chunkAddr, chunkData)
ctx := context.Background()
var err error
_, err = storeFum.Put(ctx, storage.ModePutUpload, ch)
if err != nil {
t.Fatal(err)
}
_, err = storeFee.Get(ctx, storage.ModeGetRequest, chunkAddr)
if err != nil {
t.Fatal(err)
}
_, err = storeFi.Get(ctx, storage.ModeGetRequest, chunkAddr)
if err != nil {
t.Fatal(err)
}
_, err = storeFo.Get(ctx, storage.ModeGetRequest, chunkAddr)
if err != nil {
t.Fatal(err)
}
}
// TestLimitWriter verifies that writing will fail when capacity is exceeded.
func TestLimitWriter(t *testing.T) {
buf := bytes.NewBuffer(nil)
data := []byte("foo")
writeCloser := cmdfile.NopWriteCloser(buf)
w := cmdfile.NewLimitWriteCloser(writeCloser, int64(len(data)))
c, err := w.Write(data)
if err != nil {
t.Fatal(err)
}
if c < 3 {
t.Fatal("short write")
}
if !bytes.Equal(buf.Bytes(), data) {
t.Fatalf("expected written data %x, got %x", data, buf.Bytes())
}
_, err = w.Write(data[:1])
if err == nil {
t.Fatal("expected overflow error")
}
}
// newTestServer creates an http server to serve the bee http api endpoints.
func newTestServer(t *testing.T, storer storage.Storer) *url.URL {
t.Helper()
logger := logging.New(ioutil.Discard, 0)
store := statestore.NewStateStore()
s := api.New(tags.NewTags(store, logger), storer, nil, nil, nil, nil, logger, nil, api.Options{})
ts := httptest.NewServer(s)
srvUrl, err := url.Parse(ts.URL)
if err != nil {
t.Fatal(err)
}
return srvUrl
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment