Commit a5d16425 authored by Zahoor Mohamed's avatar Zahoor Mohamed Committed by GitHub

shed: import shed from swarm (#70)

* shed: import shed from swarm (PR 1/12)
parent 04defd20
......@@ -5,6 +5,8 @@ go 1.14
require (
github.com/btcsuite/btcd v0.20.1-beta
github.com/coreos/go-semver v0.3.0
github.com/ethereum/go-ethereum v1.9.12
github.com/ethersphere/swarm v0.5.7
github.com/gogo/protobuf v1.3.1
github.com/gorilla/handlers v1.4.2
github.com/gorilla/mux v1.7.3
......@@ -17,15 +19,15 @@ require (
github.com/libp2p/go-ws-transport v0.2.0
github.com/multiformats/go-multiaddr v0.2.0
github.com/multiformats/go-multistream v0.1.0
github.com/opentracing/opentracing-go v1.0.2
github.com/opentracing/opentracing-go v1.1.0
github.com/prometheus/client_golang v1.3.0
github.com/sirupsen/logrus v1.4.2
github.com/spf13/cobra v0.0.5
github.com/spf13/viper v1.6.2
github.com/syndtr/goleveldb v1.0.0
github.com/syndtr/goleveldb v1.0.1-0.20190923125748-758128399b1d
github.com/uber/jaeger-client-go v2.22.1+incompatible
github.com/uber/jaeger-lib v2.2.0+incompatible // indirect
golang.org/x/crypto v0.0.0-20191219195013-becbf705a915
golang.org/x/crypto v0.0.0-20200311171314-f7b00557c8c4
golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e
resenje.org/web v0.4.0
)
This diff is collapsed.
// Copyright 2018 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
// Package shed provides a simple abstraction components to compose
// more complex operations on storage data organized in fields and indexes.
//
// Only type which holds logical information about swarm storage chunks data
// and metadata is Item. This part is not generalized mostly for
// performance reasons.
package shed
import (
"github.com/syndtr/goleveldb/leveldb"
"github.com/syndtr/goleveldb/leveldb/iterator"
"github.com/syndtr/goleveldb/leveldb/opt"
)
const (
openFileLimit = 128 // The limit for LevelDB OpenFilesCacheCapacity.
)
// DB provides abstractions over LevelDB in order to
// implement complex structures using fields and ordered indexes.
// It provides a schema functionality to store fields and indexes
// information about naming and types.
type DB struct {
ldb *leveldb.DB
quit chan struct{} // Quit channel to stop the metrics collection before closing the database
}
// NewDB constructs a new DB and validates the schema
// if it exists in database on the given path.
// metricsPrefix is used for metrics collection for the given DB.
func NewDB(path string, metricsPrefix string) (db *DB, err error) {
ldb, err := leveldb.OpenFile(path, &opt.Options{
OpenFilesCacheCapacity: openFileLimit,
})
if err != nil {
return nil, err
}
db = &DB{
ldb: ldb,
}
if _, err = db.getSchema(); err != nil {
if err == leveldb.ErrNotFound {
// save schema with initialized default fields
if err = db.putSchema(schema{
Fields: make(map[string]fieldSpec),
Indexes: make(map[byte]indexSpec),
}); err != nil {
return nil, err
}
} else {
return nil, err
}
}
// Create a quit channel for the periodic metrics collector and run it
db.quit = make(chan struct{})
return db, nil
}
// Put wraps LevelDB Put method to increment metrics counter.
func (db *DB) Put(key []byte, value []byte) (err error) {
err = db.ldb.Put(key, value, nil)
if err != nil {
return err
}
return nil
}
// Get wraps LevelDB Get method to increment metrics counter.
func (db *DB) Get(key []byte) (value []byte, err error) {
value, err = db.ldb.Get(key, nil)
if err != nil {
return nil, err
}
return value, nil
}
// Has wraps LevelDB Has method to increment metrics counter.
func (db *DB) Has(key []byte) (yes bool, err error) {
yes, err = db.ldb.Has(key, nil)
if err != nil {
return false, err
}
return yes, nil
}
// Delete wraps LevelDB Delete method to increment metrics counter.
func (db *DB) Delete(key []byte) (err error) {
err = db.ldb.Delete(key, nil)
if err != nil {
return err
}
return nil
}
// NewIterator wraps LevelDB NewIterator method to increment metrics counter.
func (db *DB) NewIterator() iterator.Iterator {
return db.ldb.NewIterator(nil, nil)
}
// WriteBatch wraps LevelDB Write method to increment metrics counter.
func (db *DB) WriteBatch(batch *leveldb.Batch) (err error) {
err = db.ldb.Write(batch, nil)
if err != nil {
return err
}
return nil
}
// Close closes LevelDB database.
func (db *DB) Close() (err error) {
close(db.quit)
return db.ldb.Close()
}
// Copyright 2018 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package shed
import (
"io/ioutil"
"os"
"testing"
)
// TestNewDB constructs a new DB
// and validates if the schema is initialized properly.
func TestNewDB(t *testing.T) {
db, cleanupFunc := newTestDB(t)
defer cleanupFunc()
s, err := db.getSchema()
if err != nil {
t.Fatal(err)
}
if s.Fields == nil {
t.Error("schema fields are empty")
}
if len(s.Fields) != 0 {
t.Errorf("got schema fields length %v, want %v", len(s.Fields), 0)
}
if s.Indexes == nil {
t.Error("schema indexes are empty")
}
if len(s.Indexes) != 0 {
t.Errorf("got schema indexes length %v, want %v", len(s.Indexes), 0)
}
}
// TestDB_persistence creates one DB, saves a field and closes that DB.
// Then, it constructs another DB and trues to retrieve the saved value.
func TestDB_persistence(t *testing.T) {
dir, err := ioutil.TempDir("", "shed-test-persistence")
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(dir)
db, err := NewDB(dir, "")
if err != nil {
t.Fatal(err)
}
stringField, err := db.NewStringField("preserve-me")
if err != nil {
t.Fatal(err)
}
want := "persistent value"
err = stringField.Put(want)
if err != nil {
t.Fatal(err)
}
err = db.Close()
if err != nil {
t.Fatal(err)
}
db2, err := NewDB(dir, "")
if err != nil {
t.Fatal(err)
}
stringField2, err := db2.NewStringField("preserve-me")
if err != nil {
t.Fatal(err)
}
got, err := stringField2.Get()
if err != nil {
t.Fatal(err)
}
if got != want {
t.Errorf("got string %q, want %q", got, want)
}
}
// newTestDB is a helper function that constructs a
// temporary database and returns a cleanup function that must
// be called to remove the data.
func newTestDB(t *testing.T) (db *DB, cleanupFunc func()) {
t.Helper()
dir, err := ioutil.TempDir("", "shed-test")
if err != nil {
t.Fatal(err)
}
db, err = NewDB(dir, "")
if err != nil {
os.RemoveAll(dir)
t.Fatal(err)
}
return db, func() {
db.Close()
os.RemoveAll(dir)
}
}
// Copyright 2018 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package shed_test
import (
"bytes"
"context"
"encoding/binary"
"fmt"
"io/ioutil"
"log"
"os"
"time"
"github.com/ethersphere/swarm/shed"
"github.com/ethersphere/swarm/storage"
"github.com/syndtr/goleveldb/leveldb"
)
// Store holds fields and indexes (including their encoding functions)
// and defines operations on them by composing data from them.
// It implements storage.ChunkStore interface.
// It is just an example without any support for parallel operations
// or real world implementation.
type Store struct {
db *shed.DB
// fields and indexes
schemaName shed.StringField
accessCounter shed.Uint64Field
retrievalIndex shed.Index
accessIndex shed.Index
gcIndex shed.Index
}
// New returns new Store. All fields and indexes are initialized
// and possible conflicts with schema from existing database is checked
// automatically.
func New(path string) (s *Store, err error) {
db, err := shed.NewDB(path, "")
if err != nil {
return nil, err
}
s = &Store{
db: db,
}
// Identify current storage schema by arbitrary name.
s.schemaName, err = db.NewStringField("schema-name")
if err != nil {
return nil, err
}
// Global ever incrementing index of chunk accesses.
s.accessCounter, err = db.NewUint64Field("access-counter")
if err != nil {
return nil, err
}
// Index storing actual chunk address, data and store timestamp.
s.retrievalIndex, err = db.NewIndex("Address->StoreTimestamp|Data", shed.IndexFuncs{
EncodeKey: func(fields shed.Item) (key []byte, err error) {
return fields.Address, nil
},
DecodeKey: func(key []byte) (e shed.Item, err error) {
e.Address = key
return e, nil
},
EncodeValue: func(fields shed.Item) (value []byte, err error) {
b := make([]byte, 8)
binary.BigEndian.PutUint64(b, uint64(fields.StoreTimestamp))
value = append(b, fields.Data...)
return value, nil
},
DecodeValue: func(keyItem shed.Item, value []byte) (e shed.Item, err error) {
e.StoreTimestamp = int64(binary.BigEndian.Uint64(value[:8]))
e.Data = value[8:]
return e, nil
},
})
if err != nil {
return nil, err
}
// Index storing access timestamp for a particular address.
// It is needed in order to update gc index keys for iteration order.
s.accessIndex, err = db.NewIndex("Address->AccessTimestamp", shed.IndexFuncs{
EncodeKey: func(fields shed.Item) (key []byte, err error) {
return fields.Address, nil
},
DecodeKey: func(key []byte) (e shed.Item, err error) {
e.Address = key
return e, nil
},
EncodeValue: func(fields shed.Item) (value []byte, err error) {
b := make([]byte, 8)
binary.BigEndian.PutUint64(b, uint64(fields.AccessTimestamp))
return b, nil
},
DecodeValue: func(keyItem shed.Item, value []byte) (e shed.Item, err error) {
e.AccessTimestamp = int64(binary.BigEndian.Uint64(value))
return e, nil
},
})
if err != nil {
return nil, err
}
// Index with keys ordered by access timestamp for garbage collection prioritization.
s.gcIndex, err = db.NewIndex("AccessTimestamp|StoredTimestamp|Address->nil", shed.IndexFuncs{
EncodeKey: func(fields shed.Item) (key []byte, err error) {
b := make([]byte, 16, 16+len(fields.Address))
binary.BigEndian.PutUint64(b[:8], uint64(fields.AccessTimestamp))
binary.BigEndian.PutUint64(b[8:16], uint64(fields.StoreTimestamp))
key = append(b, fields.Address...)
return key, nil
},
DecodeKey: func(key []byte) (e shed.Item, err error) {
e.AccessTimestamp = int64(binary.BigEndian.Uint64(key[:8]))
e.StoreTimestamp = int64(binary.BigEndian.Uint64(key[8:16]))
e.Address = key[16:]
return e, nil
},
EncodeValue: func(fields shed.Item) (value []byte, err error) {
return nil, nil
},
DecodeValue: func(keyItem shed.Item, value []byte) (e shed.Item, err error) {
return e, nil
},
})
if err != nil {
return nil, err
}
return s, nil
}
// Put stores the chunk and sets it store timestamp.
func (s *Store) Put(_ context.Context, ch storage.Chunk) (err error) {
return s.retrievalIndex.Put(shed.Item{
Address: ch.Address(),
Data: ch.Data(),
StoreTimestamp: time.Now().UTC().UnixNano(),
})
}
// Get retrieves a chunk with the provided address.
// It updates access and gc indexes by removing the previous
// items from them and adding new items as keys of index entries
// are changed.
func (s *Store) Get(_ context.Context, addr storage.Address) (c storage.Chunk, err error) {
batch := new(leveldb.Batch)
// Get the chunk data and storage timestamp.
item, err := s.retrievalIndex.Get(shed.Item{
Address: addr,
})
if err != nil {
if err == leveldb.ErrNotFound {
return nil, storage.ErrChunkNotFound
}
return nil, err
}
// Get the chunk access timestamp.
accessItem, err := s.accessIndex.Get(shed.Item{
Address: addr,
})
switch err {
case nil:
// Remove gc index entry if access timestamp is found.
err = s.gcIndex.DeleteInBatch(batch, shed.Item{
Address: item.Address,
StoreTimestamp: accessItem.AccessTimestamp,
AccessTimestamp: item.StoreTimestamp,
})
if err != nil {
return nil, err
}
case leveldb.ErrNotFound:
// Access timestamp is not found. Do not do anything.
// This is the firs get request.
default:
return nil, err
}
// Specify new access timestamp
accessTimestamp := time.Now().UTC().UnixNano()
// Put new access timestamp in access index.
err = s.accessIndex.PutInBatch(batch, shed.Item{
Address: addr,
AccessTimestamp: accessTimestamp,
})
if err != nil {
return nil, err
}
// Put new access timestamp in gc index.
err = s.gcIndex.PutInBatch(batch, shed.Item{
Address: item.Address,
AccessTimestamp: accessTimestamp,
StoreTimestamp: item.StoreTimestamp,
})
if err != nil {
return nil, err
}
// Increment access counter.
// Currently this information is not used anywhere.
_, err = s.accessCounter.IncInBatch(batch)
if err != nil {
return nil, err
}
// Write the batch.
err = s.db.WriteBatch(batch)
if err != nil {
return nil, err
}
// Return the chunk.
return storage.NewChunk(item.Address, item.Data), nil
}
// CollectGarbage is an example of index iteration.
// It provides no reliable garbage collection functionality.
func (s *Store) CollectGarbage() (err error) {
const maxTrashSize = 100
maxRounds := 10 // arbitrary number, needs to be calculated
// Run a few gc rounds.
for roundCount := 0; roundCount < maxRounds; roundCount++ {
var garbageCount int
// New batch for a new cg round.
trash := new(leveldb.Batch)
// Iterate through all index items and break when needed.
err = s.gcIndex.Iterate(func(item shed.Item) (stop bool, err error) {
// Remove the chunk.
err = s.retrievalIndex.DeleteInBatch(trash, item)
if err != nil {
return false, err
}
// Remove the element in gc index.
err = s.gcIndex.DeleteInBatch(trash, item)
if err != nil {
return false, err
}
// Remove the relation in access index.
err = s.accessIndex.DeleteInBatch(trash, item)
if err != nil {
return false, err
}
garbageCount++
if garbageCount >= maxTrashSize {
return true, nil
}
return false, nil
}, nil)
if err != nil {
return err
}
if garbageCount == 0 {
return nil
}
err = s.db.WriteBatch(trash)
if err != nil {
return err
}
}
return nil
}
// GetSchema is an example of retrieveing the most simple
// string from a database field.
func (s *Store) GetSchema() (name string, err error) {
name, err = s.schemaName.Get()
if err == leveldb.ErrNotFound {
return "", nil
}
return name, err
}
// PutSchema is an example of storing the most simple
// string in a database field.
func (s *Store) PutSchema(name string) (err error) {
return s.schemaName.Put(name)
}
// Close closes the underlying database.
func (s *Store) Close() error {
return s.db.Close()
}
// Example_store constructs a simple storage implementation using shed package.
func Example_store() {
dir, err := ioutil.TempDir("", "ephemeral")
if err != nil {
log.Fatal(err)
}
defer os.RemoveAll(dir)
s, err := New(dir)
if err != nil {
log.Fatal(err)
}
defer s.Close()
ch := storage.GenerateRandomChunk(1024)
err = s.Put(context.Background(), ch)
if err != nil {
log.Fatal(err)
}
got, err := s.Get(context.Background(), ch.Address())
if err != nil {
log.Fatal(err)
}
fmt.Println(bytes.Equal(got.Data(), ch.Data()))
//Output: true
}
// Copyright 2018 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package shed
import (
"github.com/syndtr/goleveldb/leveldb"
)
// StringField is the most simple field implementation
// that stores an arbitrary string under a specific LevelDB key.
type StringField struct {
db *DB
key []byte
}
// NewStringField retruns a new Instance of StringField.
// It validates its name and type against the database schema.
func (db *DB) NewStringField(name string) (f StringField, err error) {
key, err := db.schemaFieldKey(name, "string")
if err != nil {
return f, err
}
return StringField{
db: db,
key: key,
}, nil
}
// Get returns a string value from database.
// If the value is not found, an empty string is returned
// an no error.
func (f StringField) Get() (val string, err error) {
b, err := f.db.Get(f.key)
if err != nil {
if err == leveldb.ErrNotFound {
return "", nil
}
return "", err
}
return string(b), nil
}
// Put stores a string in the database.
func (f StringField) Put(val string) (err error) {
return f.db.Put(f.key, []byte(val))
}
// PutInBatch stores a string in a batch that can be
// saved later in database.
func (f StringField) PutInBatch(batch *leveldb.Batch, val string) {
batch.Put(f.key, []byte(val))
}
// Copyright 2018 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package shed
import (
"testing"
"github.com/syndtr/goleveldb/leveldb"
)
// TestStringField validates put and get operations
// of the StringField.
func TestStringField(t *testing.T) {
db, cleanupFunc := newTestDB(t)
defer cleanupFunc()
simpleString, err := db.NewStringField("simple-string")
if err != nil {
t.Fatal(err)
}
t.Run("get empty", func(t *testing.T) {
got, err := simpleString.Get()
if err != nil {
t.Fatal(err)
}
want := ""
if got != want {
t.Errorf("got string %q, want %q", got, want)
}
})
t.Run("put", func(t *testing.T) {
want := "simple string value"
err = simpleString.Put(want)
if err != nil {
t.Fatal(err)
}
got, err := simpleString.Get()
if err != nil {
t.Fatal(err)
}
if got != want {
t.Errorf("got string %q, want %q", got, want)
}
t.Run("overwrite", func(t *testing.T) {
want := "overwritten string value"
err = simpleString.Put(want)
if err != nil {
t.Fatal(err)
}
got, err := simpleString.Get()
if err != nil {
t.Fatal(err)
}
if got != want {
t.Errorf("got string %q, want %q", got, want)
}
})
})
t.Run("put in batch", func(t *testing.T) {
batch := new(leveldb.Batch)
want := "simple string batch value"
simpleString.PutInBatch(batch, want)
err = db.WriteBatch(batch)
if err != nil {
t.Fatal(err)
}
got, err := simpleString.Get()
if err != nil {
t.Fatal(err)
}
if got != want {
t.Errorf("got string %q, want %q", got, want)
}
t.Run("overwrite", func(t *testing.T) {
batch := new(leveldb.Batch)
want := "overwritten string batch value"
simpleString.PutInBatch(batch, want)
err = db.WriteBatch(batch)
if err != nil {
t.Fatal(err)
}
got, err := simpleString.Get()
if err != nil {
t.Fatal(err)
}
if got != want {
t.Errorf("got string %q, want %q", got, want)
}
})
})
}
// Copyright 2018 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package shed
import (
"github.com/ethereum/go-ethereum/rlp"
"github.com/syndtr/goleveldb/leveldb"
)
// StructField is a helper to store complex structure by
// encoding it in RLP format.
type StructField struct {
db *DB
key []byte
}
// NewStructField returns a new StructField.
// It validates its name and type against the database schema.
func (db *DB) NewStructField(name string) (f StructField, err error) {
key, err := db.schemaFieldKey(name, "struct-rlp")
if err != nil {
return f, err
}
return StructField{
db: db,
key: key,
}, nil
}
// Get unmarshals data from the database to a provided val.
// If the data is not found leveldb.ErrNotFound is returned.
func (f StructField) Get(val interface{}) (err error) {
b, err := f.db.Get(f.key)
if err != nil {
return err
}
return rlp.DecodeBytes(b, val)
}
// Put marshals provided val and saves it to the database.
func (f StructField) Put(val interface{}) (err error) {
b, err := rlp.EncodeToBytes(val)
if err != nil {
return err
}
return f.db.Put(f.key, b)
}
// PutInBatch marshals provided val and puts it into the batch.
func (f StructField) PutInBatch(batch *leveldb.Batch, val interface{}) (err error) {
b, err := rlp.EncodeToBytes(val)
if err != nil {
return err
}
batch.Put(f.key, b)
return nil
}
// Copyright 2018 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package shed
import (
"testing"
"github.com/syndtr/goleveldb/leveldb"
)
// TestStructField validates put and get operations
// of the StructField.
func TestStructField(t *testing.T) {
db, cleanupFunc := newTestDB(t)
defer cleanupFunc()
complexField, err := db.NewStructField("complex-field")
if err != nil {
t.Fatal(err)
}
type complexStructure struct {
A string
}
t.Run("get empty", func(t *testing.T) {
var s complexStructure
err := complexField.Get(&s)
if err != leveldb.ErrNotFound {
t.Fatalf("got error %v, want %v", err, leveldb.ErrNotFound)
}
want := ""
if s.A != want {
t.Errorf("got string %q, want %q", s.A, want)
}
})
t.Run("put", func(t *testing.T) {
want := complexStructure{
A: "simple string value",
}
err = complexField.Put(want)
if err != nil {
t.Fatal(err)
}
var got complexStructure
err = complexField.Get(&got)
if err != nil {
t.Fatal(err)
}
if got.A != want.A {
t.Errorf("got string %q, want %q", got.A, want.A)
}
t.Run("overwrite", func(t *testing.T) {
want := complexStructure{
A: "overwritten string value",
}
err = complexField.Put(want)
if err != nil {
t.Fatal(err)
}
var got complexStructure
err = complexField.Get(&got)
if err != nil {
t.Fatal(err)
}
if got.A != want.A {
t.Errorf("got string %q, want %q", got.A, want.A)
}
})
})
t.Run("put in batch", func(t *testing.T) {
batch := new(leveldb.Batch)
want := complexStructure{
A: "simple string batch value",
}
err = complexField.PutInBatch(batch, want)
if err != nil {
t.Fatal(err)
}
err = db.WriteBatch(batch)
if err != nil {
t.Fatal(err)
}
var got complexStructure
err := complexField.Get(&got)
if err != nil {
t.Fatal(err)
}
if got.A != want.A {
t.Errorf("got string %q, want %q", got, want)
}
t.Run("overwrite", func(t *testing.T) {
batch := new(leveldb.Batch)
want := complexStructure{
A: "overwritten string batch value",
}
err = complexField.PutInBatch(batch, want)
if err != nil {
t.Fatal(err)
}
err = db.WriteBatch(batch)
if err != nil {
t.Fatal(err)
}
var got complexStructure
err := complexField.Get(&got)
if err != nil {
t.Fatal(err)
}
if got.A != want.A {
t.Errorf("got string %q, want %q", got, want)
}
})
})
}
// Copyright 2018 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package shed
import (
"encoding/binary"
"github.com/syndtr/goleveldb/leveldb"
)
// Uint64Field provides a way to have a simple counter in the database.
// It transparently encodes uint64 type value to bytes.
type Uint64Field struct {
db *DB
key []byte
}
// NewUint64Field returns a new Uint64Field.
// It validates its name and type against the database schema.
func (db *DB) NewUint64Field(name string) (f Uint64Field, err error) {
key, err := db.schemaFieldKey(name, "uint64")
if err != nil {
return f, err
}
return Uint64Field{
db: db,
key: key,
}, nil
}
// Get retrieves a uint64 value from the database.
// If the value is not found in the database a 0 value
// is returned and no error.
func (f Uint64Field) Get() (val uint64, err error) {
b, err := f.db.Get(f.key)
if err != nil {
if err == leveldb.ErrNotFound {
return 0, nil
}
return 0, err
}
return binary.BigEndian.Uint64(b), nil
}
// Put encodes uin64 value and stores it in the database.
func (f Uint64Field) Put(val uint64) (err error) {
return f.db.Put(f.key, encodeUint64(val))
}
// PutInBatch stores a uint64 value in a batch
// that can be saved later in the database.
func (f Uint64Field) PutInBatch(batch *leveldb.Batch, val uint64) {
batch.Put(f.key, encodeUint64(val))
}
// Inc increments a uint64 value in the database.
// This operation is not goroutine save.
func (f Uint64Field) Inc() (val uint64, err error) {
val, err = f.Get()
if err != nil {
if err == leveldb.ErrNotFound {
val = 0
} else {
return 0, err
}
}
val++
return val, f.Put(val)
}
// IncInBatch increments a uint64 value in the batch
// by retreiving a value from the database, not the same batch.
// This operation is not goroutine save.
func (f Uint64Field) IncInBatch(batch *leveldb.Batch) (val uint64, err error) {
val, err = f.Get()
if err != nil {
if err == leveldb.ErrNotFound {
val = 0
} else {
return 0, err
}
}
val++
f.PutInBatch(batch, val)
return val, nil
}
// Dec decrements a uint64 value in the database.
// This operation is not goroutine save.
// The field is protected from overflow to a negative value.
func (f Uint64Field) Dec() (val uint64, err error) {
val, err = f.Get()
if err != nil {
if err == leveldb.ErrNotFound {
val = 0
} else {
return 0, err
}
}
if val != 0 {
val--
}
return val, f.Put(val)
}
// DecInBatch decrements a uint64 value in the batch
// by retreiving a value from the database, not the same batch.
// This operation is not goroutine save.
// The field is protected from overflow to a negative value.
func (f Uint64Field) DecInBatch(batch *leveldb.Batch) (val uint64, err error) {
val, err = f.Get()
if err != nil {
if err == leveldb.ErrNotFound {
val = 0
} else {
return 0, err
}
}
if val != 0 {
val--
}
f.PutInBatch(batch, val)
return val, nil
}
// encode transforms uint64 to 8 byte long
// slice in big endian encoding.
func encodeUint64(val uint64) (b []byte) {
b = make([]byte, 8)
binary.BigEndian.PutUint64(b, val)
return b
}
// Copyright 2018 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package shed
import (
"testing"
"github.com/syndtr/goleveldb/leveldb"
)
// TestUint64Field validates put and get operations
// of the Uint64Field.
func TestUint64Field(t *testing.T) {
db, cleanupFunc := newTestDB(t)
defer cleanupFunc()
counter, err := db.NewUint64Field("counter")
if err != nil {
t.Fatal(err)
}
t.Run("get empty", func(t *testing.T) {
got, err := counter.Get()
if err != nil {
t.Fatal(err)
}
var want uint64
if got != want {
t.Errorf("got uint64 %v, want %v", got, want)
}
})
t.Run("put", func(t *testing.T) {
var want uint64 = 42
err = counter.Put(want)
if err != nil {
t.Fatal(err)
}
got, err := counter.Get()
if err != nil {
t.Fatal(err)
}
if got != want {
t.Errorf("got uint64 %v, want %v", got, want)
}
t.Run("overwrite", func(t *testing.T) {
var want uint64 = 84
err = counter.Put(want)
if err != nil {
t.Fatal(err)
}
got, err := counter.Get()
if err != nil {
t.Fatal(err)
}
if got != want {
t.Errorf("got uint64 %v, want %v", got, want)
}
})
})
t.Run("put in batch", func(t *testing.T) {
batch := new(leveldb.Batch)
var want uint64 = 42
counter.PutInBatch(batch, want)
err = db.WriteBatch(batch)
if err != nil {
t.Fatal(err)
}
got, err := counter.Get()
if err != nil {
t.Fatal(err)
}
if got != want {
t.Errorf("got uint64 %v, want %v", got, want)
}
t.Run("overwrite", func(t *testing.T) {
batch := new(leveldb.Batch)
var want uint64 = 84
counter.PutInBatch(batch, want)
err = db.WriteBatch(batch)
if err != nil {
t.Fatal(err)
}
got, err := counter.Get()
if err != nil {
t.Fatal(err)
}
if got != want {
t.Errorf("got uint64 %v, want %v", got, want)
}
})
})
}
// TestUint64Field_Inc validates Inc operation
// of the Uint64Field.
func TestUint64Field_Inc(t *testing.T) {
db, cleanupFunc := newTestDB(t)
defer cleanupFunc()
counter, err := db.NewUint64Field("counter")
if err != nil {
t.Fatal(err)
}
var want uint64 = 1
got, err := counter.Inc()
if err != nil {
t.Fatal(err)
}
if got != want {
t.Errorf("got uint64 %v, want %v", got, want)
}
want = 2
got, err = counter.Inc()
if err != nil {
t.Fatal(err)
}
if got != want {
t.Errorf("got uint64 %v, want %v", got, want)
}
}
// TestUint64Field_IncInBatch validates IncInBatch operation
// of the Uint64Field.
func TestUint64Field_IncInBatch(t *testing.T) {
db, cleanupFunc := newTestDB(t)
defer cleanupFunc()
counter, err := db.NewUint64Field("counter")
if err != nil {
t.Fatal(err)
}
batch := new(leveldb.Batch)
var want uint64 = 1
got, err := counter.IncInBatch(batch)
if err != nil {
t.Fatal(err)
}
if got != want {
t.Errorf("got uint64 %v, want %v", got, want)
}
err = db.WriteBatch(batch)
if err != nil {
t.Fatal(err)
}
got, err = counter.Get()
if err != nil {
t.Fatal(err)
}
if got != want {
t.Errorf("got uint64 %v, want %v", got, want)
}
batch2 := new(leveldb.Batch)
want = 2
got, err = counter.IncInBatch(batch2)
if err != nil {
t.Fatal(err)
}
if got != want {
t.Errorf("got uint64 %v, want %v", got, want)
}
err = db.WriteBatch(batch2)
if err != nil {
t.Fatal(err)
}
got, err = counter.Get()
if err != nil {
t.Fatal(err)
}
if got != want {
t.Errorf("got uint64 %v, want %v", got, want)
}
}
// TestUint64Field_Dec validates Dec operation
// of the Uint64Field.
func TestUint64Field_Dec(t *testing.T) {
db, cleanupFunc := newTestDB(t)
defer cleanupFunc()
counter, err := db.NewUint64Field("counter")
if err != nil {
t.Fatal(err)
}
// test overflow protection
var want uint64
got, err := counter.Dec()
if err != nil {
t.Fatal(err)
}
if got != want {
t.Errorf("got uint64 %v, want %v", got, want)
}
want = 32
err = counter.Put(want)
if err != nil {
t.Fatal(err)
}
want = 31
got, err = counter.Dec()
if err != nil {
t.Fatal(err)
}
if got != want {
t.Errorf("got uint64 %v, want %v", got, want)
}
}
// TestUint64Field_DecInBatch validates DecInBatch operation
// of the Uint64Field.
func TestUint64Field_DecInBatch(t *testing.T) {
db, cleanupFunc := newTestDB(t)
defer cleanupFunc()
counter, err := db.NewUint64Field("counter")
if err != nil {
t.Fatal(err)
}
batch := new(leveldb.Batch)
var want uint64
got, err := counter.DecInBatch(batch)
if err != nil {
t.Fatal(err)
}
if got != want {
t.Errorf("got uint64 %v, want %v", got, want)
}
err = db.WriteBatch(batch)
if err != nil {
t.Fatal(err)
}
got, err = counter.Get()
if err != nil {
t.Fatal(err)
}
if got != want {
t.Errorf("got uint64 %v, want %v", got, want)
}
batch2 := new(leveldb.Batch)
want = 42
counter.PutInBatch(batch2, want)
err = db.WriteBatch(batch2)
if err != nil {
t.Fatal(err)
}
got, err = counter.Get()
if err != nil {
t.Fatal(err)
}
if got != want {
t.Errorf("got uint64 %v, want %v", got, want)
}
batch3 := new(leveldb.Batch)
want = 41
got, err = counter.DecInBatch(batch3)
if err != nil {
t.Fatal(err)
}
if got != want {
t.Errorf("got uint64 %v, want %v", got, want)
}
err = db.WriteBatch(batch3)
if err != nil {
t.Fatal(err)
}
got, err = counter.Get()
if err != nil {
t.Fatal(err)
}
if got != want {
t.Errorf("got uint64 %v, want %v", got, want)
}
}
This diff is collapsed.
This diff is collapsed.
// Copyright 2018 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package shed
import (
"encoding/json"
"errors"
"fmt"
)
var (
// LevelDB key value for storing the schema.
keySchema = []byte{0}
// LevelDB key prefix for all field type.
// LevelDB keys will be constructed by appending name values to this prefix.
keyPrefixFields byte = 1
// LevelDB key prefix from which indexing keys start.
// Every index has its own key prefix and this value defines the first one.
keyPrefixIndexStart byte = 2 // Q: or maybe a higher number like 7, to have more space for potential specific perfixes
)
// schema is used to serialize known database structure information.
type schema struct {
Fields map[string]fieldSpec `json:"fields"` // keys are field names
Indexes map[byte]indexSpec `json:"indexes"` // keys are index prefix bytes
}
// fieldSpec holds information about a particular field.
// It does not need Name field as it is contained in the
// schema.Field map key.
type fieldSpec struct {
Type string `json:"type"`
}
// indexSpec holds information about a particular index.
// It does not contain index type, as indexes do not have type.
type indexSpec struct {
Name string `json:"name"`
}
// schemaFieldKey retrieves the complete LevelDB key for
// a particular field from the schema definition.
func (db *DB) schemaFieldKey(name, fieldType string) (key []byte, err error) {
if name == "" {
return nil, errors.New("field name cannot be blank")
}
if fieldType == "" {
return nil, errors.New("field type cannot be blank")
}
s, err := db.getSchema()
if err != nil {
return nil, err
}
var found bool
for n, f := range s.Fields {
if n == name {
if f.Type != fieldType {
return nil, fmt.Errorf("field %q of type %q stored as %q in db", name, fieldType, f.Type)
}
break
}
}
if !found {
s.Fields[name] = fieldSpec{
Type: fieldType,
}
err := db.putSchema(s)
if err != nil {
return nil, err
}
}
return append([]byte{keyPrefixFields}, []byte(name)...), nil
}
// RenameIndex changes the schema so that an existing index name is changed
// while preserving its data by keeping the same internal key prefix.
// Renaming indexes is useful when encoding functions can be backward compatible
// to avoid data migrations.
func (db *DB) RenameIndex(name, newName string) (renamed bool, err error) {
if name == "" {
return false, errors.New("index name cannot be blank")
}
if newName == "" {
return false, errors.New("new index name cannot be blank")
}
if newName == name {
return false, nil
}
s, err := db.getSchema()
if err != nil {
return false, err
}
for i, f := range s.Indexes {
if f.Name == name {
s.Indexes[i] = indexSpec{
Name: newName,
}
return true, db.putSchema(s)
}
if f.Name == newName {
return true, nil
}
}
return false, nil
}
// schemaIndexID retrieves the complete LevelDB prefix for
// a particular index.
func (db *DB) schemaIndexPrefix(name string) (id byte, err error) {
if name == "" {
return 0, errors.New("index name cannot be blank")
}
s, err := db.getSchema()
if err != nil {
return 0, err
}
nextID := keyPrefixIndexStart
for i, f := range s.Indexes {
if i >= nextID {
nextID = i + 1
}
if f.Name == name {
return i, nil
}
}
id = nextID
s.Indexes[id] = indexSpec{
Name: name,
}
return id, db.putSchema(s)
}
// getSchema retrieves the complete schema from
// the database.
func (db *DB) getSchema() (s schema, err error) {
b, err := db.Get(keySchema)
if err != nil {
return s, err
}
err = json.Unmarshal(b, &s)
return s, err
}
// putSchema stores the complete schema to
// the database.
func (db *DB) putSchema(s schema) (err error) {
b, err := json.Marshal(s)
if err != nil {
return err
}
return db.Put(keySchema, b)
}
// Copyright 2018 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package shed
import (
"bytes"
"testing"
)
// TestDB_schemaFieldKey validates correctness of schemaFieldKey.
func TestDB_schemaFieldKey(t *testing.T) {
db, cleanupFunc := newTestDB(t)
defer cleanupFunc()
t.Run("empty name or type", func(t *testing.T) {
_, err := db.schemaFieldKey("", "")
if err == nil {
t.Error("error not returned, but expected")
}
_, err = db.schemaFieldKey("", "type")
if err == nil {
t.Error("error not returned, but expected")
}
_, err = db.schemaFieldKey("test", "")
if err == nil {
t.Error("error not returned, but expected")
}
})
t.Run("same field", func(t *testing.T) {
key1, err := db.schemaFieldKey("test", "undefined")
if err != nil {
t.Fatal(err)
}
key2, err := db.schemaFieldKey("test", "undefined")
if err != nil {
t.Fatal(err)
}
if !bytes.Equal(key1, key2) {
t.Errorf("schema keys for the same field name are not the same: %q, %q", string(key1), string(key2))
}
})
t.Run("different fields", func(t *testing.T) {
key1, err := db.schemaFieldKey("test1", "undefined")
if err != nil {
t.Fatal(err)
}
key2, err := db.schemaFieldKey("test2", "undefined")
if err != nil {
t.Fatal(err)
}
if bytes.Equal(key1, key2) {
t.Error("schema keys for the same field name are the same, but must not be")
}
})
t.Run("same field name different types", func(t *testing.T) {
_, err := db.schemaFieldKey("the-field", "one-type")
if err != nil {
t.Fatal(err)
}
_, err = db.schemaFieldKey("the-field", "another-type")
if err == nil {
t.Error("error not returned, but expected")
}
})
}
// TestDB_schemaIndexPrefix validates correctness of schemaIndexPrefix.
func TestDB_schemaIndexPrefix(t *testing.T) {
db, cleanupFunc := newTestDB(t)
defer cleanupFunc()
t.Run("same name", func(t *testing.T) {
id1, err := db.schemaIndexPrefix("test")
if err != nil {
t.Fatal(err)
}
id2, err := db.schemaIndexPrefix("test")
if err != nil {
t.Fatal(err)
}
if id1 != id2 {
t.Errorf("schema keys for the same field name are not the same: %v, %v", id1, id2)
}
})
t.Run("different names", func(t *testing.T) {
id1, err := db.schemaIndexPrefix("test1")
if err != nil {
t.Fatal(err)
}
id2, err := db.schemaIndexPrefix("test2")
if err != nil {
t.Fatal(err)
}
if id1 == id2 {
t.Error("schema ids for the same index name are the same, but must not be")
}
})
}
// TestDB_RenameIndex checks if index name is correctly changed.
func TestDB_RenameIndex(t *testing.T) {
t.Run("empty names", func(t *testing.T) {
db, cleanupFunc := newTestDB(t)
defer cleanupFunc()
// empty names
renamed, err := db.RenameIndex("", "")
if err == nil {
t.Error("error not returned, but expected")
}
if renamed {
t.Fatal("index should not be renamed")
}
// empty index name
renamed, err = db.RenameIndex("", "new")
if err == nil {
t.Error("error not returned, but expected")
}
if renamed {
t.Fatal("index should not be renamed")
}
// empty new index name
renamed, err = db.RenameIndex("current", "")
if err == nil {
t.Error("error not returned, but expected")
}
if renamed {
t.Fatal("index should not be renamed")
}
})
t.Run("same names", func(t *testing.T) {
db, cleanupFunc := newTestDB(t)
defer cleanupFunc()
renamed, err := db.RenameIndex("index1", "index1")
if err != nil {
t.Error(err)
}
if renamed {
t.Fatal("index should not be renamed")
}
})
t.Run("unknown name", func(t *testing.T) {
db, cleanupFunc := newTestDB(t)
defer cleanupFunc()
renamed, err := db.RenameIndex("index1", "index1new")
if err != nil {
t.Error(err)
}
if renamed {
t.Fatal("index should not be renamed")
}
})
t.Run("valid names", func(t *testing.T) {
db, cleanupFunc := newTestDB(t)
defer cleanupFunc()
// initial indexes
key1, err := db.schemaIndexPrefix("index1")
if err != nil {
t.Fatal(err)
}
key2, err := db.schemaIndexPrefix("index2")
if err != nil {
t.Fatal(err)
}
// name the first one
renamed, err := db.RenameIndex("index1", "index1new")
if err != nil {
t.Fatal(err)
}
if !renamed {
t.Fatal("index not renamed")
}
// validate that the index key stays the same
key1same, err := db.schemaIndexPrefix("index1new")
if err != nil {
t.Fatal(err)
}
if key1 != key1same {
t.Fatal("indexes renamed, but keys are not the same")
}
// validate that the independent index is not changed
key2same, err := db.schemaIndexPrefix("index2")
if err != nil {
t.Fatal(err)
}
if key2 != key2same {
t.Fatal("independent index key has changed")
}
// validate that it is safe to create a new index with previous name
key1renew, err := db.schemaIndexPrefix("index1")
if err != nil {
t.Fatal(err)
}
if key1 == key1renew {
t.Fatal("renewed index and the original one have the same key")
}
if key2 == key1renew {
t.Fatal("renewed index and the independent one have the same key")
}
})
}
// Copyright 2018 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package shed
import (
"encoding/binary"
"github.com/syndtr/goleveldb/leveldb"
)
// Uint64Vector provides a way to have multiple counters in the database.
// It transparently encodes uint64 type value to bytes.
type Uint64Vector struct {
db *DB
key []byte
}
// NewUint64Vector returns a new Uint64Vector.
// It validates its name and type against the database schema.
func (db *DB) NewUint64Vector(name string) (f Uint64Vector, err error) {
key, err := db.schemaFieldKey(name, "vector-uint64")
if err != nil {
return f, err
}
return Uint64Vector{
db: db,
key: key,
}, nil
}
// Get retrieves a uint64 value at index i from the database.
// If the value is not found in the database a 0 value
// is returned and no error.
func (f Uint64Vector) Get(i uint64) (val uint64, err error) {
b, err := f.db.Get(f.indexKey(i))
if err != nil {
if err == leveldb.ErrNotFound {
return 0, nil
}
return 0, err
}
return binary.BigEndian.Uint64(b), nil
}
// Put encodes uin64 value and stores it in the database.
func (f Uint64Vector) Put(i, val uint64) (err error) {
return f.db.Put(f.indexKey(i), encodeUint64(val))
}
// PutInBatch stores a uint64 value at index i in a batch
// that can be saved later in the database.
func (f Uint64Vector) PutInBatch(batch *leveldb.Batch, i, val uint64) {
batch.Put(f.indexKey(i), encodeUint64(val))
}
// Inc increments a uint64 value in the database.
// This operation is not goroutine safe.
func (f Uint64Vector) Inc(i uint64) (val uint64, err error) {
val, err = f.Get(i)
if err != nil {
if err == leveldb.ErrNotFound {
val = 0
} else {
return 0, err
}
}
val++
return val, f.Put(i, val)
}
// IncInBatch increments a uint64 value at index i in the batch
// by retreiving a value from the database, not the same batch.
// This operation is not goroutine safe.
func (f Uint64Vector) IncInBatch(batch *leveldb.Batch, i uint64) (val uint64, err error) {
val, err = f.Get(i)
if err != nil {
if err == leveldb.ErrNotFound {
val = 0
} else {
return 0, err
}
}
val++
f.PutInBatch(batch, i, val)
return val, nil
}
// Dec decrements a uint64 value at index i in the database.
// This operation is not goroutine safe.
// The field is protected from overflow to a negative value.
func (f Uint64Vector) Dec(i uint64) (val uint64, err error) {
val, err = f.Get(i)
if err != nil {
if err == leveldb.ErrNotFound {
val = 0
} else {
return 0, err
}
}
if val != 0 {
val--
}
return val, f.Put(i, val)
}
// DecInBatch decrements a uint64 value at index i in the batch
// by retreiving a value from the database, not the same batch.
// This operation is not goroutine safe.
// The field is protected from overflow to a negative value.
func (f Uint64Vector) DecInBatch(batch *leveldb.Batch, i uint64) (val uint64, err error) {
val, err = f.Get(i)
if err != nil {
if err == leveldb.ErrNotFound {
val = 0
} else {
return 0, err
}
}
if val != 0 {
val--
}
f.PutInBatch(batch, i, val)
return val, nil
}
// indexKey concatenates field prefix and vector index
// returning a unique database key for a specific vector element.
func (f Uint64Vector) indexKey(i uint64) (key []byte) {
b := make([]byte, 8)
binary.BigEndian.PutUint64(b, i)
return append(f.key, b...)
}
// Copyright 2018 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package shed
import (
"testing"
"github.com/syndtr/goleveldb/leveldb"
)
// TestUint64Vector validates put and get operations
// of the Uint64Vector.
func TestUint64Vector(t *testing.T) {
db, cleanupFunc := newTestDB(t)
defer cleanupFunc()
bins, err := db.NewUint64Vector("bins")
if err != nil {
t.Fatal(err)
}
t.Run("get empty", func(t *testing.T) {
got, err := bins.Get(0)
if err != nil {
t.Fatal(err)
}
var want uint64
if got != want {
t.Errorf("got uint64 %v, want %v", got, want)
}
})
t.Run("put", func(t *testing.T) {
for _, index := range []uint64{0, 1, 2, 5, 100} {
var want uint64 = 42 + index
err = bins.Put(index, want)
if err != nil {
t.Fatal(err)
}
got, err := bins.Get(index)
if err != nil {
t.Fatal(err)
}
if got != want {
t.Errorf("got %v uint64 %v, want %v", index, got, want)
}
t.Run("overwrite", func(t *testing.T) {
var want uint64 = 84 + index
err = bins.Put(index, want)
if err != nil {
t.Fatal(err)
}
got, err := bins.Get(index)
if err != nil {
t.Fatal(err)
}
if got != want {
t.Errorf("got %v uint64 %v, want %v", index, got, want)
}
})
}
})
t.Run("put in batch", func(t *testing.T) {
for _, index := range []uint64{0, 1, 2, 3, 5, 10} {
batch := new(leveldb.Batch)
var want uint64 = 43 + index
bins.PutInBatch(batch, index, want)
err = db.WriteBatch(batch)
if err != nil {
t.Fatal(err)
}
got, err := bins.Get(index)
if err != nil {
t.Fatal(err)
}
if got != want {
t.Errorf("got %v uint64 %v, want %v", index, got, want)
}
t.Run("overwrite", func(t *testing.T) {
batch := new(leveldb.Batch)
var want uint64 = 85 + index
bins.PutInBatch(batch, index, want)
err = db.WriteBatch(batch)
if err != nil {
t.Fatal(err)
}
got, err := bins.Get(index)
if err != nil {
t.Fatal(err)
}
if got != want {
t.Errorf("got %v uint64 %v, want %v", index, got, want)
}
})
}
})
}
// TestUint64Vector_Inc validates Inc operation
// of the Uint64Vector.
func TestUint64Vector_Inc(t *testing.T) {
db, cleanupFunc := newTestDB(t)
defer cleanupFunc()
bins, err := db.NewUint64Vector("bins")
if err != nil {
t.Fatal(err)
}
for _, index := range []uint64{0, 1, 2, 3, 5, 10} {
var want uint64 = 1
got, err := bins.Inc(index)
if err != nil {
t.Fatal(err)
}
if got != want {
t.Errorf("got %v uint64 %v, want %v", index, got, want)
}
want = 2
got, err = bins.Inc(index)
if err != nil {
t.Fatal(err)
}
if got != want {
t.Errorf("got %v uint64 %v, want %v", index, got, want)
}
}
}
// TestUint64Vector_IncInBatch validates IncInBatch operation
// of the Uint64Vector.
func TestUint64Vector_IncInBatch(t *testing.T) {
db, cleanupFunc := newTestDB(t)
defer cleanupFunc()
bins, err := db.NewUint64Vector("bins")
if err != nil {
t.Fatal(err)
}
for _, index := range []uint64{0, 1, 2, 3, 5, 10} {
batch := new(leveldb.Batch)
var want uint64 = 1
got, err := bins.IncInBatch(batch, index)
if err != nil {
t.Fatal(err)
}
if got != want {
t.Errorf("got %v uint64 %v, want %v", index, got, want)
}
err = db.WriteBatch(batch)
if err != nil {
t.Fatal(err)
}
got, err = bins.Get(index)
if err != nil {
t.Fatal(err)
}
if got != want {
t.Errorf("got %v uint64 %v, want %v", index, got, want)
}
batch2 := new(leveldb.Batch)
want = 2
got, err = bins.IncInBatch(batch2, index)
if err != nil {
t.Fatal(err)
}
if got != want {
t.Errorf("got %v uint64 %v, want %v", index, got, want)
}
err = db.WriteBatch(batch2)
if err != nil {
t.Fatal(err)
}
got, err = bins.Get(index)
if err != nil {
t.Fatal(err)
}
if got != want {
t.Errorf("got %v uint64 %v, want %v", index, got, want)
}
}
}
// TestUint64Vector_Dec validates Dec operation
// of the Uint64Vector.
func TestUint64Vector_Dec(t *testing.T) {
db, cleanupFunc := newTestDB(t)
defer cleanupFunc()
bins, err := db.NewUint64Vector("bins")
if err != nil {
t.Fatal(err)
}
for _, index := range []uint64{0, 1, 2, 3, 5, 10} {
// test overflow protection
var want uint64
got, err := bins.Dec(index)
if err != nil {
t.Fatal(err)
}
if got != want {
t.Errorf("got %v uint64 %v, want %v", index, got, want)
}
want = 32 + index
err = bins.Put(index, want)
if err != nil {
t.Fatal(err)
}
want = 31 + index
got, err = bins.Dec(index)
if err != nil {
t.Fatal(err)
}
if got != want {
t.Errorf("got %v uint64 %v, want %v", index, got, want)
}
}
}
// TestUint64Vector_DecInBatch validates DecInBatch operation
// of the Uint64Vector.
func TestUint64Vector_DecInBatch(t *testing.T) {
db, cleanupFunc := newTestDB(t)
defer cleanupFunc()
bins, err := db.NewUint64Vector("bins")
if err != nil {
t.Fatal(err)
}
for _, index := range []uint64{0, 1, 2, 3, 5, 10} {
batch := new(leveldb.Batch)
var want uint64
got, err := bins.DecInBatch(batch, index)
if err != nil {
t.Fatal(err)
}
if got != want {
t.Errorf("got %v uint64 %v, want %v", index, got, want)
}
err = db.WriteBatch(batch)
if err != nil {
t.Fatal(err)
}
got, err = bins.Get(index)
if err != nil {
t.Fatal(err)
}
if got != want {
t.Errorf("got %v uint64 %v, want %v", index, got, want)
}
batch2 := new(leveldb.Batch)
want = 42 + index
bins.PutInBatch(batch2, index, want)
err = db.WriteBatch(batch2)
if err != nil {
t.Fatal(err)
}
got, err = bins.Get(index)
if err != nil {
t.Fatal(err)
}
if got != want {
t.Errorf("got %v uint64 %v, want %v", index, got, want)
}
batch3 := new(leveldb.Batch)
want = 41 + index
got, err = bins.DecInBatch(batch3, index)
if err != nil {
t.Fatal(err)
}
if got != want {
t.Errorf("got %v uint64 %v, want %v", index, got, want)
}
err = db.WriteBatch(batch3)
if err != nil {
t.Fatal(err)
}
got, err = bins.Get(index)
if err != nil {
t.Fatal(err)
}
if got != want {
t.Errorf("got %v uint64 %v, want %v", index, got, want)
}
}
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment