9/12: localstore migration - remove legacy migration code (#87)

* remove legacy migration code

pkg/storage/localstore/export.go

@@ -33,10 +33,8 @@ const (
 	// filename in tar archive that holds the information
 	// about exported data format version
 	exportVersionFilename = ".swarm-export-version"
-	// legacy version for previous LDBStore
-	legacyExportVersion = "1"
 	// current export format version
-	currentExportVersion = "2"
+	currentExportVersion = "1"
 )
 
 // Export writes a tar structured data to the writer of
@@ -94,9 +92,10 @@ func (db *DB) Import(r io.Reader, legacy bool) (count int64, err error) {
 	go func() {
 		var (
 			firstFile = true
+
 			// if exportVersionFilename file is not present
-			// assume legacy version
-			version = legacyExportVersion
+			// assume current version
+			version = currentExportVersion
 		)
 		for {
 			hdr, err := tr.Next()
@@ -146,11 +145,6 @@ func (db *DB) Import(r io.Reader, legacy bool) (count int64, err error) {
 
 			var ch chunk.Chunk
 			switch version {
-			case legacyExportVersion:
-				// LDBStore Export exported chunk data prefixed with the chunk key.
-				// That is not necessary, as the key is in the chunk filename,
-				// but backward compatibility needs to be preserved.
-				ch = chunk.NewChunk(key, data[32:])
 			case currentExportVersion:
 				ch = chunk.NewChunk(key, data)
 			default:

pkg/storage/localstore/migration.go

@@ -17,13 +17,8 @@
 package localstore
 
 import (
-	"encoding/binary"
 	"errors"
 	"fmt"
-
-	"github.com/ethersphere/swarm/chunk"
-	"github.com/ethersphere/swarm/shed"
-	"github.com/syndtr/goleveldb/leveldb"
 )
 
 var errMissingCurrentSchema = errors.New("could not find current db schema")
@@ -37,10 +32,7 @@ type migration struct {
 // schemaMigrations contains an ordered list of the database schemes, that is
 // in order to run data migrations in the correct sequence
 var schemaMigrations = []migration{
-	{name: DbSchemaPurity, fn: func(db *DB) error { return nil }},
-	{name: DbSchemaHalloween, fn: func(db *DB) error { return nil }},
-	{name: DbSchemaSanctuary, fn: func(db *DB) error { return nil }},
-	{name: DbSchemaDiwali, fn: migrateSanctuary},
+	{name: DbSchemaCode, fn: func(db *DB) error { return nil }},
 }
 
 func (db *DB) migrate(schemaName string) error {
@@ -106,79 +98,3 @@ func getMigrations(currentSchema, targetSchema string, allSchemeMigrations []mig
 	}
 	return migrations, nil
 }
-
-// this function migrates Sanctuary schema to the Diwali schema
-func migrateSanctuary(db *DB) error {
-	// just rename the pull index
-	renamed, err := db.shed.RenameIndex("PO|BinID->Hash", "PO|BinID->Hash|Tag")
-	if err != nil {
-		return err
-	}
-	if !renamed {
-		return errors.New("pull index was not successfully renamed")
-	}
-
-	if db.tags == nil {
-		return errors.New("had an error accessing the tags object")
-	}
-
-	batch := new(leveldb.Batch)
-	db.batchMu.Lock()
-	defer db.batchMu.Unlock()
-
-	// since pullIndex points to the Tag value, we should eliminate possible
-	// pushIndex leak due to items that were used by previous pull sync tag
-	// increment logic. we need to build the index first since db object is
-	// still not initialised at this stage
-	db.pushIndex, err = db.shed.NewIndex("StoreTimestamp|Hash->Tags", shed.IndexFuncs{
-		EncodeKey: func(fields shed.Item) (key []byte, err error) {
-			key = make([]byte, 40)
-			binary.BigEndian.PutUint64(key[:8], uint64(fields.StoreTimestamp))
-			copy(key[8:], fields.Address[:])
-			return key, nil
-		},
-		DecodeKey: func(key []byte) (e shed.Item, err error) {
-			e.Address = key[8:]
-			e.StoreTimestamp = int64(binary.BigEndian.Uint64(key[:8]))
-			return e, nil
-		},
-		EncodeValue: func(fields shed.Item) (value []byte, err error) {
-			tag := make([]byte, 4)
-			binary.BigEndian.PutUint32(tag, fields.Tag)
-			return tag, nil
-		},
-		DecodeValue: func(keyItem shed.Item, value []byte) (e shed.Item, err error) {
-			if value != nil {
-				e.Tag = binary.BigEndian.Uint32(value)
-			}
-			return e, nil
-		},
-	})
-	if err != nil {
-		return err
-	}
-
-	err = db.pushIndex.Iterate(func(item shed.Item) (stop bool, err error) {
-		tag, err := db.tags.Get(item.Tag)
-		if err != nil {
-			if err == chunk.TagNotFoundErr {
-				return false, nil
-			}
-			return true, err
-		}
-
-		// anonymous tags should no longer appear in pushIndex
-		if tag != nil && tag.Anonymous {
-			err = db.pushIndex.DeleteInBatch(batch, item)
-			if err != nil {
-				return true, nil
-			}
-		}
-		return false, nil
-	}, nil)
-	if err != nil {
-		return err
-	}
-
-	return db.shed.WriteBatch(batch)
-}

pkg/storage/localstore/migration_test.go

@@ -17,17 +17,13 @@
 package localstore
 
 import (
-	"io"
 	"io/ioutil"
-	"log"
 	"math/rand"
 	"os"
-	"path"
 	"strings"
 	"testing"
 
 	"github.com/ethersphere/bee/pkg/logging"
-	"github.com/ethersphere/swarm/chunk"
 )
 
 func TestOneMigration(t *testing.T) {
@@ -36,16 +32,17 @@ func TestOneMigration(t *testing.T) {
 		DbSchemaCurrent = s
 	}(schemaMigrations, DbSchemaCurrent)
 
-	DbSchemaCurrent = DbSchemaSanctuary
+	DbSchemaCurrent = DbSchemaCode
+	dbSchemaNext := "dbSchemaNext"
 
 	ran := false
 	shouldNotRun := false
 	schemaMigrations = []migration{
-		{name: DbSchemaSanctuary, fn: func(db *DB) error {
+		{name: DbSchemaCode, fn: func(db *DB) error {
 			shouldNotRun = true // this should not be executed
 			return nil
 		}},
-		{name: DbSchemaDiwali, fn: func(db *DB) error {
+		{name: dbSchemaNext, fn: func(db *DB) error {
 			ran = true
 			return nil
 		}},
@@ -74,7 +71,7 @@ func TestOneMigration(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	DbSchemaCurrent = DbSchemaDiwali
+	DbSchemaCurrent = dbSchemaNext
 
 	// start the existing localstore and expect the migration to run
 	db, err = New(dir, baseKey, nil, logger)
@@ -87,8 +84,8 @@ func TestOneMigration(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	if schemaName != DbSchemaDiwali {
-		t.Errorf("schema name mismatch. got '%s', want '%s'", schemaName, DbSchemaDiwali)
+	if schemaName != dbSchemaNext {
+		t.Errorf("schema name mismatch. got '%s', want '%s'", schemaName, dbSchemaNext)
 	}
 
 	if !ran {
@@ -111,17 +108,17 @@ func TestManyMigrations(t *testing.T) {
 		DbSchemaCurrent = s
 	}(schemaMigrations, DbSchemaCurrent)
 
-	DbSchemaCurrent = DbSchemaSanctuary
+	DbSchemaCurrent = DbSchemaCode
 
 	shouldNotRun := false
 	executionOrder := []int{-1, -1, -1, -1}
 
 	schemaMigrations = []migration{
-		{name: DbSchemaSanctuary, fn: func(db *DB) error {
+		{name: DbSchemaCode, fn: func(db *DB) error {
 			shouldNotRun = true // this should not be executed
 			return nil
 		}},
-		{name: DbSchemaDiwali, fn: func(db *DB) error {
+		{name: "keju", fn: func(db *DB) error {
 			executionOrder[0] = 0
 			return nil
 		}},
@@ -314,73 +311,3 @@ func TestMigrationFailTo(t *testing.T) {
 		t.Errorf("migration ran but shouldnt have")
 	}
 }
-
-// TestMigrateSanctuaryFixture migrates an actual Sanctuary localstore
-// to the most recent schema.
-func TestMigrateSanctuaryFixture(t *testing.T) {
-
-	tmpdir, err := ioutil.TempDir("", "localstore-test")
-	if err != nil {
-		t.Fatal(err)
-	}
-	defer os.RemoveAll(tmpdir)
-
-	dir := path.Join(".", "testdata", "sanctuary")
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	files, err := ioutil.ReadDir(dir)
-	if err != nil {
-		log.Fatal(err)
-	}
-
-	for _, f := range files {
-		err = copyFileContents(path.Join(dir, f.Name()), path.Join(tmpdir, f.Name()))
-		if err != nil {
-			t.Fatal(err)
-		}
-	}
-
-	baseKey := make([]byte, 32)
-	if _, err := rand.Read(baseKey); err != nil {
-		t.Fatal(err)
-	}
-	logger := logging.New(ioutil.Discard, 0)
-
-	// start localstore with the copied fixture
-	db, err := New(tmpdir, baseKey, &Options{Tags: chunk.NewTags()}, logger)
-	if err != nil {
-		t.Fatal(err)
-	}
-	schemaName, err := db.schemaName.Get()
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	if schemaName != DbSchemaCurrent {
-		t.Fatalf("schema name mismatch, want '%s' got '%s'", DbSchemaCurrent, schemaName)
-	}
-
-	err = db.Close()
-	if err != nil {
-		t.Fatal(err)
-	}
-}
-
-func copyFileContents(src, dst string) (err error) {
-	in, err := os.Open(src)
-	if err != nil {
-		return err
-	}
-	defer in.Close()
-	out, err := os.Create(dst)
-	if err != nil {
-		return err
-	}
-	defer out.Close()
-	if _, err = io.Copy(out, in); err != nil {
-		return err
-	}
-	return out.Sync()
-}

pkg/storage/localstore/schema.go

@@ -16,52 +16,12 @@
 
 package localstore
 
-import (
-	"github.com/syndtr/goleveldb/leveldb"
-	"github.com/syndtr/goleveldb/leveldb/opt"
-)
-
 // The DB schema we want to use. The actual/current DB schema might differ
 // until migrations are run.
-var DbSchemaCurrent = DbSchemaDiwali
+var DbSchemaCurrent = DbSchemaCode
 
 // There was a time when we had no schema at all.
 const DbSchemaNone = ""
 
-// "purity" is the first formal schema of LevelDB we release together with Swarm 0.3.5
-const DbSchemaPurity = "purity"
-
-// "halloween" is here because we had a screw in the garbage collector index.
-// Because of that we had to rebuild the GC index to get rid of erroneous
-// entries and that takes a long time. This schema is used for bookkeeping,
-// so rebuild index will run just once.
-const DbSchemaHalloween = "halloween"
-
-const DbSchemaSanctuary = "sanctuary"
-
-// the "diwali" migration simply renames the pullIndex in localstore
-const DbSchemaDiwali = "diwali"
-
-// returns true if legacy database is in the datadir
-func IsLegacyDatabase(datadir string) bool {
-
-	var (
-		legacyDbSchemaKey = []byte{8}
-	)
-
-	db, err := leveldb.OpenFile(datadir, &opt.Options{OpenFilesCacheCapacity: 128})
-	if err != nil {
-		return false
-	}
-	defer db.Close()
-
-	data, err := db.Get(legacyDbSchemaKey, nil)
-	if err != nil {
-		if err == leveldb.ErrNotFound {
-			// if we haven't found anything under the legacy db schema key- we are not on legacy
-			return false
-		}
-
-	}
-	return string(data) == DbSchemaHalloween || string(data) == DbSchemaPurity
-}
+// DbSchemaCode is the first bee schema identifier
+const DbSchemaCode = "code"

pkg/storage/localstore/testdata/sanctuary/CURRENT

-MANIFEST-000003

pkg/storage/localstore/testdata/sanctuary/CURRENT.bak

-MANIFEST-000000

pkg/storage/localstore/testdata/sanctuary/LOG

-=============== Nov 13, 2019 (+0530) ===============
-15:55:56.807910 log@legend F·NumFile S·FileSize N·Entry C·BadEntry B·BadBlock Ke·KeyError D·DroppedEntry L·Level Q·SeqNum T·TimeElapsed
-15:55:56.810616 db@open opening
-15:55:56.810899 version@stat F·[] S·0B[] Sc·[]
-15:55:56.812573 db@janitor F·2 G·0
-15:55:56.812586 db@open done T·1.947185ms
-15:55:56.812598 db@close closing
-15:55:56.812664 db@close done T·62.171µs
-=============== Nov 13, 2019 (+0530) ===============
-15:55:56.812745 log@legend F·NumFile S·FileSize N·Entry C·BadEntry B·BadBlock Ke·KeyError D·DroppedEntry L·Level Q·SeqNum T·TimeElapsed
-15:55:56.812833 version@stat F·[] S·0B[] Sc·[]
-15:55:56.812845 db@open opening
-15:55:56.812875 journal@recovery F·1
-15:55:56.812980 journal@recovery recovering @1
-15:55:56.813117 version@stat F·[] S·0B[] Sc·[]
-15:55:56.817949 db@janitor F·2 G·0
-15:55:56.817966 db@open done T·5.117043ms
-15:55:58.621297 db@close closing
-15:55:58.621392 db@close done T·93.745µs