cmd: add db import export (#1580)

cmd/bee/cmd/cmd.go

@@ -7,11 +7,14 @@ package cmd
 import (
 	"errors"
 	"fmt"
+	"io/ioutil"
 	"os"
 	"path/filepath"
 	"strings"
 
+	"github.com/ethersphere/bee/pkg/logging"
 	"github.com/ethersphere/bee/pkg/swarm"
+	"github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
 )
@@ -111,6 +114,7 @@ func newCommand(opts ...option) (c *command, err error) {
 	}
 
 	c.initVersionCmd()
+	c.initDBCmd()
 
 	if err := c.initConfigurateOptionsCmd(); err != nil {
 		return nil, err
@@ -221,3 +225,24 @@ func (c *command) setAllFlags(cmd *cobra.Command) {
 	cmd.Flags().String(optionNameSwapInitialDeposit, "100000000000000000", "initial deposit if deploying a new chequebook")
 	cmd.Flags().Bool(optionNameSwapEnable, true, "enable swap")
 }
+
+func newLogger(cmd *cobra.Command, verbosity string) (logging.Logger, error) {
+	var logger logging.Logger
+	switch verbosity {
+	case "0", "silent":
+		logger = logging.New(ioutil.Discard, 0)
+	case "1", "error":
+		logger = logging.New(cmd.OutOrStdout(), logrus.ErrorLevel)
+	case "2", "warn":
+		logger = logging.New(cmd.OutOrStdout(), logrus.WarnLevel)
+	case "3", "info":
+		logger = logging.New(cmd.OutOrStdout(), logrus.InfoLevel)
+	case "4", "debug":
+		logger = logging.New(cmd.OutOrStdout(), logrus.DebugLevel)
+	case "5", "trace":
+		logger = logging.New(cmd.OutOrStdout(), logrus.TraceLevel)
+	default:
+		return nil, fmt.Errorf("unknown verbosity level %q", verbosity)
+	}
+	return logger, nil
+}

cmd/bee/cmd/db.go

+// Copyright 2021 The Swarm Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package cmd
+
+import (
+	"errors"
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/ethersphere/bee/pkg/localstore"
+	"github.com/spf13/cobra"
+)
+
+func (c *command) initDBCmd() {
+	cmd := &cobra.Command{
+		Use:   "db",
+		Short: "Perform basic DB related operations",
+	}
+
+	dbExportCmd(cmd)
+	dbImportCmd(cmd)
+
+	c.root.AddCommand(cmd)
+}
+
+func dbExportCmd(cmd *cobra.Command) {
+	c := &cobra.Command{
+		Use:   "export <filename>",
+		Short: "Perform DB export to a file. Use \"-\" as filename in order to write to STDOUT",
+		RunE: func(cmd *cobra.Command, args []string) (err error) {
+			if (len(args)) != 1 {
+				return cmd.Help()
+			}
+			v, err := cmd.Flags().GetString(optionNameVerbosity)
+			if err != nil {
+				return fmt.Errorf("get verbosity: %v", err)
+			}
+			v = strings.ToLower(v)
+			logger, err := newLogger(cmd, v)
+			if err != nil {
+				return fmt.Errorf("new logger: %v", err)
+			}
+
+			dataDir, err := cmd.Flags().GetString(optionNameDataDir)
+			if err != nil {
+				return fmt.Errorf("get data-dir: %v", err)
+			}
+			if dataDir == "" {
+				return errors.New("no data-dir provided")
+			}
+
+			logger.Infof("starting export process with data-dir at %s", dataDir)
+
+			path := filepath.Join(dataDir, "localstore")
+
+			storer, err := localstore.New(path, nil, nil, logger)
+			if err != nil {
+				return fmt.Errorf("localstore: %w", err)
+			}
+
+			var out io.Writer
+			if args[0] == "-" {
+				out = os.Stdout
+			} else {
+				f, err := os.Create(args[0])
+				if err != nil {
+					return fmt.Errorf("error opening output file: %s", err)
+				}
+				defer f.Close()
+				out = f
+			}
+			c, err := storer.Export(out)
+			if err != nil {
+				return fmt.Errorf("error exporting database: %v", err)
+			}
+
+			logger.Infof("database exported %d records successfully", c)
+
+			return nil
+		},
+	}
+	c.Flags().String(optionNameDataDir, "", "data directory")
+	c.Flags().String(optionNameVerbosity, "info", "verbosity level")
+	cmd.AddCommand(c)
+}
+
+func dbImportCmd(cmd *cobra.Command) {
+	c := &cobra.Command{
+		Use:   "import <filename>",
+		Short: "Perform DB import from a file. Use \"-\" as filename in order to feed from STDIN",
+		RunE: func(cmd *cobra.Command, args []string) (err error) {
+			if (len(args)) != 1 {
+				return cmd.Help()
+			}
+			v, err := cmd.Flags().GetString(optionNameVerbosity)
+			if err != nil {
+				return fmt.Errorf("get verbosity: %v", err)
+			}
+			v = strings.ToLower(v)
+			logger, err := newLogger(cmd, v)
+			if err != nil {
+				return fmt.Errorf("new logger: %v", err)
+			}
+			dataDir, err := cmd.Flags().GetString(optionNameDataDir)
+			if err != nil {
+				return fmt.Errorf("get data-dir: %v", err)
+			}
+			if dataDir == "" {
+				return errors.New("no data-dir provided")
+			}
+
+			fmt.Printf("starting import process with data-dir at %s\n", dataDir)
+
+			path := filepath.Join(dataDir, "localstore")
+
+			storer, err := localstore.New(path, nil, nil, logger)
+			if err != nil {
+				return fmt.Errorf("localstore: %w", err)
+			}
+
+			var in io.Reader
+			if args[0] == "-" {
+				in = os.Stdin
+			} else {
+				f, err := os.Open(args[0])
+				if err != nil {
+					return fmt.Errorf("error opening input file: %s", err)
+				}
+				defer f.Close()
+				in = f
+			}
+			c, err := storer.Import(cmd.Context(), in)
+			if err != nil {
+				return fmt.Errorf("error importing database: %v", err)
+			}
+
+			fmt.Printf("database imported %d records successfully\n", c)
+
+			return nil
+		},
+	}
+	c.Flags().String(optionNameDataDir, "", "data directory")
+	c.Flags().String(optionNameVerbosity, "info", "verbosity level")
+	cmd.AddCommand(c)
+}

cmd/bee/cmd/deploy.go

@@ -6,12 +6,9 @@ package cmd
 
 import (
 	"fmt"
-	"io/ioutil"
 	"strings"
 
-	"github.com/ethersphere/bee/pkg/logging"
 	"github.com/ethersphere/bee/pkg/node"
-	"github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
 )
 
@@ -24,22 +21,10 @@ func (c *command) initDeployCmd() error {
 				return cmd.Help()
 			}
 
-			var logger logging.Logger
-			switch v := strings.ToLower(c.config.GetString(optionNameVerbosity)); v {
-			case "0", "silent":
-				logger = logging.New(ioutil.Discard, 0)
-			case "1", "error":
-				logger = logging.New(cmd.OutOrStdout(), logrus.ErrorLevel)
-			case "2", "warn":
-				logger = logging.New(cmd.OutOrStdout(), logrus.WarnLevel)
-			case "3", "info":
-				logger = logging.New(cmd.OutOrStdout(), logrus.InfoLevel)
-			case "4", "debug":
-				logger = logging.New(cmd.OutOrStdout(), logrus.DebugLevel)
-			case "5", "trace":
-				logger = logging.New(cmd.OutOrStdout(), logrus.TraceLevel)
-			default:
-				return fmt.Errorf("unknown verbosity level %q", v)
+			v := strings.ToLower(c.config.GetString(optionNameVerbosity))
+			logger, err := newLogger(cmd, v)
+			if err != nil {
+				return fmt.Errorf("new logger: %v", err)
 			}
 
 			dataDir := c.config.GetString(optionNameDataDir)

cmd/bee/cmd/init.go

@@ -6,12 +6,9 @@ package cmd
 
 import (
 	"fmt"
-	"io/ioutil"
 	"strings"
 
-	"github.com/ethersphere/bee/pkg/logging"
 	"github.com/ethersphere/bee/pkg/node"
-	"github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
 )
 
@@ -24,24 +21,11 @@ func (c *command) initInitCmd() (err error) {
 				return cmd.Help()
 			}
 
-			var logger logging.Logger
-			switch v := strings.ToLower(c.config.GetString(optionNameVerbosity)); v {
-			case "0", "silent":
-				logger = logging.New(ioutil.Discard, 0)
-			case "1", "error":
-				logger = logging.New(cmd.OutOrStdout(), logrus.ErrorLevel)
-			case "2", "warn":
-				logger = logging.New(cmd.OutOrStdout(), logrus.WarnLevel)
-			case "3", "info":
-				logger = logging.New(cmd.OutOrStdout(), logrus.InfoLevel)
-			case "4", "debug":
-				logger = logging.New(cmd.OutOrStdout(), logrus.DebugLevel)
-			case "5", "trace":
-				logger = logging.New(cmd.OutOrStdout(), logrus.TraceLevel)
-			default:
-				return fmt.Errorf("unknown verbosity level %q", v)
+			v := strings.ToLower(c.config.GetString(optionNameVerbosity))
+			logger, err := newLogger(cmd, v)
+			if err != nil {
+				return fmt.Errorf("new logger: %v", err)
 			}
-
 			signerConfig, err := c.configureSigner(cmd, logger)
 			if err != nil {
 				return err

cmd/bee/cmd/start.go

@@ -32,7 +32,6 @@ import (
 	"github.com/ethersphere/bee/pkg/resolver/multiresolver"
 	"github.com/ethersphere/bee/pkg/swarm"
 	"github.com/kardianos/service"
-	"github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
 )
 
@@ -50,22 +49,10 @@ func (c *command) initStartCmd() (err error) {
 				return cmd.Help()
 			}
 
-			var logger logging.Logger
-			switch v := strings.ToLower(c.config.GetString(optionNameVerbosity)); v {
-			case "0", "silent":
-				logger = logging.New(ioutil.Discard, 0)
-			case "1", "error":
-				logger = logging.New(cmd.OutOrStdout(), logrus.ErrorLevel)
-			case "2", "warn":
-				logger = logging.New(cmd.OutOrStdout(), logrus.WarnLevel)
-			case "3", "info":
-				logger = logging.New(cmd.OutOrStdout(), logrus.InfoLevel)
-			case "4", "debug":
-				logger = logging.New(cmd.OutOrStdout(), logrus.DebugLevel)
-			case "5", "trace":
-				logger = logging.New(cmd.OutOrStdout(), logrus.TraceLevel)
-			default:
-				return fmt.Errorf("unknown verbosity level %q", v)
+			v := strings.ToLower(c.config.GetString(optionNameVerbosity))
+			logger, err := newLogger(cmd, v)
+			if err != nil {
+				return fmt.Errorf("new logger: %v", err)
 			}
 
 			isWindowsService, err := isWindowsService()

pkg/localstore/export.go

@@ -80,12 +80,9 @@ func (db *DB) Export(w io.Writer) (count int64, err error) {
 // Import reads a tar structured data from the reader and
 // stores chunks in the database. It returns the number of
 // chunks imported.
-func (db *DB) Import(r io.Reader, legacy bool) (count int64, err error) {
+func (db *DB) Import(ctx context.Context, r io.Reader) (count int64, err error) {
 	tr := tar.NewReader(r)
 
-	ctx, cancel := context.WithCancel(context.Background())
-	defer cancel()
-
 	errC := make(chan error)
 	doneC := make(chan struct{})
 	tokenPool := make(chan struct{}, 100)

pkg/localstore/export_test.go

@@ -57,7 +57,7 @@ func TestExportImport(t *testing.T) {
 
 	db2 := newTestDB(t, nil)
 
-	c, err = db2.Import(&buf, false)
+	c, err = db2.Import(context.Background(), &buf)
 	if err != nil {
 		t.Fatal(err)
 	}