Skip to content

M
mybee
Commit 898a3fda authored by Zahoor Mohamed's avatar Zahoor Mohamed Committed by GitHub
Browse files
Options

shed: Replace levelDB with badgerDB (#75) 

* replace levelDB with badgerDB
parent 787130a2
Expand all Hide whitespace changes
Inline Side-by-side
Showing with 936 additions and 393 deletions
.gitignore
View file @ 898a3fda
/dist
/.idea
/.vscode
vendor/*
# Compiled Object files, Static and Dynamic libs (Shared Objects)
*.o
......
go.mod
View file @ 898a3fda
......@@ -5,29 +5,29 @@ go 1.14
require (
github.com/btcsuite/btcd v0.20.1-beta
github.com/coreos/go-semver v0.3.0
github.com/ethereum/go-ethereum v1.9.12
github.com/dgraph-io/badger/v2 v2.0.3
github.com/ethersphere/swarm v0.5.7
github.com/gogo/protobuf v1.3.1
github.com/gorilla/handlers v1.4.2
github.com/gorilla/mux v1.7.3
github.com/libp2p/go-libp2p v0.5.1
github.com/gorilla/mux v1.7.4
github.com/libp2p/go-libp2p v0.7.4
github.com/libp2p/go-libp2p-autonat-svc v0.1.0
github.com/libp2p/go-libp2p-core v0.3.0
github.com/libp2p/go-libp2p-peerstore v0.1.4
github.com/libp2p/go-libp2p-quic-transport v0.2.2
github.com/libp2p/go-tcp-transport v0.1.1
github.com/libp2p/go-ws-transport v0.2.0
github.com/multiformats/go-multiaddr v0.2.0
github.com/multiformats/go-multistream v0.1.0
github.com/libp2p/go-libp2p-core v0.5.1
github.com/libp2p/go-libp2p-peerstore v0.2.3
github.com/libp2p/go-libp2p-quic-transport v0.3.3
github.com/libp2p/go-tcp-transport v0.2.0
github.com/libp2p/go-ws-transport v0.3.0
github.com/multiformats/go-multiaddr v0.2.1
github.com/multiformats/go-multistream v0.1.1
github.com/opentracing/opentracing-go v1.1.0
github.com/prometheus/client_golang v1.3.0
github.com/sirupsen/logrus v1.4.2
github.com/spf13/cobra v0.0.5
github.com/spf13/viper v1.6.2
github.com/syndtr/goleveldb v1.0.1-0.20190923125748-758128399b1d
github.com/prometheus/client_golang v1.5.1
github.com/sirupsen/logrus v1.5.0
github.com/spf13/cobra v0.0.7
github.com/spf13/viper v1.6.3
github.com/syndtr/goleveldb v1.0.0
github.com/uber/jaeger-client-go v2.22.1+incompatible
github.com/uber/jaeger-lib v2.2.0+incompatible // indirect
golang.org/x/crypto v0.0.0-20200311171314-f7b00557c8c4
golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e
resenje.org/web v0.4.0
golang.org/x/crypto v0.0.0-20200406173513-056763e48d71
golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a
resenje.org/web v0.4.3
)
go.sum
View file @ 898a3fda
This diff is collapsed.
pkg/shed/db.go
View file @ 898a3fda
This diff is collapsed.
pkg/shed/example_store_test.go
View file @ 898a3fda
......@@ -29,7 +29,6 @@ import (
"github.com/ethersphere/bee/pkg/logging"
"github.com/ethersphere/bee/pkg/shed"
"github.com/ethersphere/swarm/storage"
"github.com/syndtr/goleveldb/leveldb"
)
// Store holds fields and indexes (including their encoding functions)
......@@ -159,16 +158,13 @@ func (s *Store) Put(_ context.Context, ch storage.Chunk) (err error) {
// items from them and adding new items as keys of index entries
// are changed.
func (s *Store) Get(_ context.Context, addr storage.Address) (c storage.Chunk, err error) {
batch := new(leveldb.Batch)
batch := s.db.GetBatch(true)
// Get the chunk data and storage timestamp.
item, err := s.retrievalIndex.Get(shed.Item{
Address: addr,
})
if err != nil {
if err == leveldb.ErrNotFound {
return nil, storage.ErrChunkNotFound
}
return nil, err
}
......@@ -187,7 +183,7 @@ func (s *Store) Get(_ context.Context, addr storage.Address) (c storage.Chunk, e
if err != nil {
return nil, err
}
case leveldb.ErrNotFound:
case shed.ErrNotFound:
// Access timestamp is not found. Do not do anything.
// This is the firs get request.
default:
......@@ -243,7 +239,7 @@ func (s *Store) CollectGarbage() (err error) {
for roundCount := 0; roundCount < maxRounds; roundCount++ {
var garbageCount int
// New batch for a new cg round.
trash := new(leveldb.Batch)
trash := s.db.GetBatch(true)
// Iterate through all index items and break when needed.
err = s.gcIndex.Iterate(func(item shed.Item) (stop bool, err error) {
// Remove the chunk.
......@@ -285,7 +281,7 @@ func (s *Store) CollectGarbage() (err error) {
// string from a database field.
func (s *Store) GetSchema() (name string, err error) {
name, err = s.schemaName.Get()
if err == leveldb.ErrNotFound {
if err == shed.ErrNotFound {
return "", nil
}
return name, err
......
pkg/shed/field_string.go
View file @ 898a3fda
......@@ -17,8 +17,8 @@
package shed
import (
"github.com/dgraph-io/badger/v2"
"github.com/ethersphere/bee/pkg/logging"
"github.com/syndtr/goleveldb/leveldb"
)
// StringField is the most simple field implementation
......@@ -49,10 +49,6 @@ func (db *DB) NewStringField(name string, logger logging.Logger) (f StringField,
func (f StringField) Get() (val string, err error) {
b, err := f.db.Get(f.key)
if err != nil {
if err == leveldb.ErrNotFound {
f.logger.Errorf("key %s not found", string(f.key))
return "", nil
}
return "", err
}
return string(b), nil
......@@ -65,6 +61,6 @@ func (f StringField) Put(val string) (err error) {
// PutInBatch stores a string in a batch that can be
// saved later in database.
func (f StringField) PutInBatch(batch *leveldb.Batch, val string) {
batch.Put(f.key, []byte(val))
func (f StringField) PutInBatch(batch *badger.Txn, val string) (err error) {
return batch.Set(f.key, []byte(val))
}
pkg/shed/field_string_test.go
View file @ 898a3fda
......@@ -21,7 +21,6 @@ import (
"testing"
"github.com/ethersphere/bee/pkg/logging"
"github.com/syndtr/goleveldb/leveldb"
)
// TestStringField validates put and get operations
......@@ -38,7 +37,7 @@ func TestStringField(t *testing.T) {
t.Run("get empty", func(t *testing.T) {
got, err := simpleString.Get()
if err != nil {
if err == nil {
t.Fatal(err)
}
want := ""
......@@ -78,9 +77,12 @@ func TestStringField(t *testing.T) {
})
t.Run("put in batch", func(t *testing.T) {
batch := new(leveldb.Batch)
batch := db.GetBatch(true)
want := "simple string batch value"
simpleString.PutInBatch(batch, want)
err = simpleString.PutInBatch(batch, want)
if err != nil {
t.Fatal(err)
}
err = db.WriteBatch(batch)
if err != nil {
t.Fatal(err)
......@@ -94,9 +96,12 @@ func TestStringField(t *testing.T) {
}
t.Run("overwrite", func(t *testing.T) {
batch := new(leveldb.Batch)
batch := db.GetBatch(true)
want := "overwritten string batch value"
simpleString.PutInBatch(batch, want)
err = simpleString.PutInBatch(batch, want)
if err != nil {
t.Fatal(err)
}
err = db.WriteBatch(batch)
if err != nil {
t.Fatal(err)
......
pkg/shed/field_struct.go
View file @ 898a3fda
......@@ -19,8 +19,8 @@ package shed
import (
"encoding/json"
"github.com/dgraph-io/badger/v2"
"github.com/ethersphere/bee/pkg/logging"
"github.com/syndtr/goleveldb/leveldb"
)
// StructField is a helper to store complex structure by
......@@ -46,7 +46,7 @@ func (db *DB) NewStructField(name string, logger logging.Logger) (f StructField,
}
// Get unmarshals data from the database to a provided val.
// If the data is not found leveldb.ErrNotFound is returned.
// If the data is not found ErrNotFound is returned.
func (f StructField) Get(val interface{}) (err error) {
b, err := f.db.Get(f.key)
if err != nil {
......@@ -67,12 +67,14 @@ func (f StructField) Put(val interface{}) (err error) {
}
// PutInBatch marshals provided val and puts it into the batch.
func (f StructField) PutInBatch(batch *leveldb.Batch, val interface{}) (err error) {
func (f StructField) PutInBatch(batch *badger.Txn, val interface{}) (err error) {
b, err := json.Marshal(val)
if err != nil {
f.logger.Debugf("could not PUT key %s in batch", string(f.key))
return err
}
batch.Put(f.key, b)
err = batch.Set(f.key, b)
if err != nil {
return err
}
return nil
}
pkg/shed/field_struct_test.go
View file @ 898a3fda
......@@ -21,7 +21,6 @@ import (
"testing"
"github.com/ethersphere/bee/pkg/logging"
"github.com/syndtr/goleveldb/leveldb"
)
// TestStructField validates put and get operations
......@@ -42,8 +41,8 @@ func TestStructField(t *testing.T) {
t.Run("get empty", func(t *testing.T) {
var s complexStructure
err := complexField.Get(&s)
if err != leveldb.ErrNotFound {
t.Fatalf("got error %v, want %v", err, leveldb.ErrNotFound)
if err != ErrNotFound {
t.Fatalf("got error %v, want %v", err, ErrNotFound)
}
want := ""
if s.A != want {
......@@ -88,7 +87,7 @@ func TestStructField(t *testing.T) {
})
t.Run("put in batch", func(t *testing.T) {
batch := new(leveldb.Batch)
batch := db.GetBatch(true)
want := complexStructure{
A: "simple string batch value",
}
......@@ -110,7 +109,7 @@ func TestStructField(t *testing.T) {
}
t.Run("overwrite", func(t *testing.T) {
batch := new(leveldb.Batch)
batch := db.GetBatch(true)
want := complexStructure{
A: "overwritten string batch value",
}
......
pkg/shed/field_uint64.go
View file @ 898a3fda
......@@ -19,8 +19,8 @@ package shed
import (
"encoding/binary"
"github.com/dgraph-io/badger/v2"
"github.com/ethersphere/bee/pkg/logging"
"github.com/syndtr/goleveldb/leveldb"
)
// Uint64Field provides a way to have a simple counter in the database.
......@@ -51,7 +51,7 @@ func (db *DB) NewUint64Field(name string, logger logging.Logger) (f Uint64Field,
func (f Uint64Field) Get() (val uint64, err error) {
b, err := f.db.Get(f.key)
if err != nil {
if err == leveldb.ErrNotFound {
if err == ErrNotFound {
f.logger.Errorf("key %s not found", string(f.key))
return 0, nil
}
......@@ -67,8 +67,8 @@ func (f Uint64Field) Put(val uint64) (err error) {
// PutInBatch stores a uint64 value in a batch
// that can be saved later in the database.
func (f Uint64Field) PutInBatch(batch *leveldb.Batch, val uint64) {
batch.Put(f.key, encodeUint64(val))
func (f Uint64Field) PutInBatch(batch *badger.Txn, val uint64) (err error){
return batch.Set(f.key, encodeUint64(val))
}
// Inc increments a uint64 value in the database.
......@@ -76,13 +76,7 @@ func (f Uint64Field) PutInBatch(batch *leveldb.Batch, val uint64) {
func (f Uint64Field) Inc() (val uint64, err error) {
val, err = f.Get()
if err != nil {
if err == leveldb.ErrNotFound {
f.logger.Debugf("key %s not found", string(f.key))
val = 0
} else {
f.logger.Errorf("key %s not found. Error: %s", string(f.key), err.Error())
return 0, err
}
return 0, err
}
val++
return val, f.Put(val)
......@@ -91,19 +85,16 @@ func (f Uint64Field) Inc() (val uint64, err error) {
// IncInBatch increments a uint64 value in the batch
// by retreiving a value from the database, not the same batch.
// This operation is not goroutine save.
func (f Uint64Field) IncInBatch(batch *leveldb.Batch) (val uint64, err error) {
func (f Uint64Field) IncInBatch(batch *badger.Txn) (val uint64, err error) {
val, err = f.Get()
if err != nil {
if err == leveldb.ErrNotFound {
f.logger.Debugf("key %s not found", string(f.key))
val = 0
} else {
f.logger.Errorf("key %s not found. Error: %s", string(f.key), err.Error())
return 0, err
}
return 0, err
}
val++
f.PutInBatch(batch, val)
err = f.PutInBatch(batch, val)
if err != nil {
return 0, err
}
return val, nil
}
......@@ -113,13 +104,7 @@ func (f Uint64Field) IncInBatch(batch *leveldb.Batch) (val uint64, err error) {
func (f Uint64Field) Dec() (val uint64, err error) {
val, err = f.Get()
if err != nil {
if err == leveldb.ErrNotFound {
f.logger.Debugf("key %s not found", string(f.key))
val = 0
} else {
f.logger.Errorf("key %s not found. Error: %s", string(f.key), err.Error())
return 0, err
}
return 0, err
}
if val != 0 {
val--
......@@ -131,21 +116,18 @@ func (f Uint64Field) Dec() (val uint64, err error) {
// by retreiving a value from the database, not the same batch.
// This operation is not goroutine save.
// The field is protected from overflow to a negative value.
func (f Uint64Field) DecInBatch(batch *leveldb.Batch) (val uint64, err error) {
func (f Uint64Field) DecInBatch(batch *badger.Txn) (val uint64, err error) {
val, err = f.Get()
if err != nil {
if err == leveldb.ErrNotFound {
f.logger.Debugf("key %s not found", string(f.key))
val = 0
} else {
f.logger.Errorf("key %s not found. Error: %s", string(f.key), err.Error())
return 0, err
}
return 0, err
}
if val != 0 {
val--
}
f.PutInBatch(batch, val)
err = f.PutInBatch(batch, val)
if err != nil {
return 0, err
}
return val, nil
}
......
pkg/shed/field_uint64_test.go
View file @ 898a3fda
......@@ -21,7 +21,6 @@ import (
"testing"
"github.com/ethersphere/bee/pkg/logging"
"github.com/syndtr/goleveldb/leveldb"
)
// TestUint64Field validates put and get operations
......@@ -77,9 +76,12 @@ func TestUint64Field(t *testing.T) {
})
t.Run("put in batch", func(t *testing.T) {
batch := new(leveldb.Batch)
batch := db.GetBatch(true)
var want uint64 = 42
counter.PutInBatch(batch, want)
err = counter.PutInBatch(batch, want)
if err != nil {
t.Fatal(err)
}
err = db.WriteBatch(batch)
if err != nil {
t.Fatal(err)
......@@ -93,9 +95,12 @@ func TestUint64Field(t *testing.T) {
}
t.Run("overwrite", func(t *testing.T) {
batch := new(leveldb.Batch)
batch := db.GetBatch(true)
var want uint64 = 84
counter.PutInBatch(batch, want)
err = counter.PutInBatch(batch, want)
if err != nil {
t.Fatal(err)
}
err = db.WriteBatch(batch)
if err != nil {
t.Fatal(err)
......@@ -152,7 +157,7 @@ func TestUint64Field_IncInBatch(t *testing.T) {
t.Fatal(err)
}
batch := new(leveldb.Batch)
batch := db.GetBatch(true)
var want uint64 = 1
got, err := counter.IncInBatch(batch)
if err != nil {
......@@ -173,7 +178,7 @@ func TestUint64Field_IncInBatch(t *testing.T) {
t.Errorf("got uint64 %v, want %v", got, want)
}
batch2 := new(leveldb.Batch)
batch2 := db.GetBatch(true)
want = 2
got, err = counter.IncInBatch(batch2)
if err != nil {
......@@ -243,7 +248,7 @@ func TestUint64Field_DecInBatch(t *testing.T) {
t.Fatal(err)
}
batch := new(leveldb.Batch)
batch := db.GetBatch(true)
var want uint64
got, err := counter.DecInBatch(batch)
if err != nil {
......@@ -264,9 +269,12 @@ func TestUint64Field_DecInBatch(t *testing.T) {
t.Errorf("got uint64 %v, want %v", got, want)
}
batch2 := new(leveldb.Batch)
batch2 := db.GetBatch(true)
want = 42
counter.PutInBatch(batch2, want)
err = counter.PutInBatch(batch2, want)
if err != nil {
t.Fatal(err)
}
err = db.WriteBatch(batch2)
if err != nil {
t.Fatal(err)
......@@ -279,7 +287,7 @@ func TestUint64Field_DecInBatch(t *testing.T) {
t.Errorf("got uint64 %v, want %v", got, want)
}
batch3 := new(leveldb.Batch)
batch3 := db.GetBatch(true)
want = 41
got, err = counter.DecInBatch(batch3)
if err != nil {
......
pkg/shed/index.go
View file @ 898a3fda
This diff is collapsed.
pkg/shed/index_test.go
View file @ 898a3fda
......@@ -26,7 +26,6 @@ import (
"time"
"github.com/ethersphere/bee/pkg/logging"
"github.com/syndtr/goleveldb/leveldb"
)
// Index functions for the index that is used in tests in this file.
......@@ -108,7 +107,7 @@ func TestIndex(t *testing.T) {
StoreTimestamp: time.Now().UTC().UnixNano(),
}
batch := new(leveldb.Batch)
batch := db.GetBatch(true)
err = index.PutInBatch(batch, want)
if err != nil {
t.Fatal(err)
......@@ -132,7 +131,7 @@ func TestIndex(t *testing.T) {
StoreTimestamp: time.Now().UTC().UnixNano(),
}
batch := new(leveldb.Batch)
batch := db.GetBatch(true)
err = index.PutInBatch(batch, want)
if err != nil {
t.Fatal(err)
......@@ -154,7 +153,7 @@ func TestIndex(t *testing.T) {
t.Run("put in batch twice", func(t *testing.T) {
// ensure that the last item of items with the same db keys
// is actually saved
batch := new(leveldb.Batch)
batch := db.GetBatch(true)
address := []byte("put-in-batch-twice-hash")
// put the first item
......@@ -218,7 +217,7 @@ func TestIndex(t *testing.T) {
}
has, err = index.Has(dontWant)
if err != nil {
if err != nil && err != ErrNotFound{
t.Fatal(err)
}
if has {
......@@ -252,7 +251,7 @@ func TestIndex(t *testing.T) {
t.Fatal(err)
}
wantErr := leveldb.ErrNotFound
wantErr := ErrNotFound
_, err = index.Get(Item{
Address: want.Address,
})
......@@ -280,7 +279,7 @@ func TestIndex(t *testing.T) {
}
checkItem(t, got, want)
batch := new(leveldb.Batch)
batch := db.GetBatch(true)
err = index.DeleteInBatch(batch, Item{
Address: want.Address,
})
......@@ -292,7 +291,7 @@ func TestIndex(t *testing.T) {
t.Fatal(err)
}
wantErr := leveldb.ErrNotFound
wantErr := ErrNotFound
_, err = index.Get(Item{
Address: want.Address,
})
......@@ -355,7 +354,7 @@ func TestIndex(t *testing.T) {
items = append(items, Item{
Address: []byte("put-hash-missing"),
})
want := leveldb.ErrNotFound
want := ErrNotFound
err := index.Fill(items)
if err != want {
t.Errorf("got error %v, want %v", err, want)
......@@ -397,7 +396,7 @@ func TestIndex_Iterate(t *testing.T) {
Data: []byte("data1"),
},
}
batch := new(leveldb.Batch)
batch := db.GetBatch(true)
for _, i := range items {
err = index.PutInBatch(batch, i)
if err != nil {
......@@ -572,7 +571,7 @@ func TestIndex_Iterate_withPrefix(t *testing.T) {
{Address: []byte("want-hash-09"), Data: []byte("data89")},
{Address: []byte("skip-hash-10"), Data: []byte("data90")},
}
batch := new(leveldb.Batch)
batch := db.GetBatch(true)
for _, i := range allItems {
err = index.PutInBatch(batch, i)
if err != nil {
......@@ -769,7 +768,7 @@ func TestIndex_count(t *testing.T) {
Data: []byte("data1"),
},
}
batch := new(leveldb.Batch)
batch := db.GetBatch(true)
for _, i := range items {
err = index.PutInBatch(batch, i)
if err != nil {
......@@ -942,7 +941,7 @@ func TestIndex_firstAndLast(t *testing.T) {
return bytes.Compare(addrs[i], addrs[j]) == -1
})
batch := new(leveldb.Batch)
batch := db.GetBatch(true)
for _, addr := range addrs {
err = index.PutInBatch(batch, Item{
Address: addr,
......@@ -1004,11 +1003,11 @@ func TestIndex_firstAndLast(t *testing.T) {
},
{
prefix: []byte{0, 3},
err: leveldb.ErrNotFound,
err: ErrNotFound,
},
{
prefix: []byte{222},
err: leveldb.ErrNotFound,
err: ErrNotFound,
},
} {
got, err := index.Last(tc.prefix)
......@@ -1093,7 +1092,7 @@ func TestIndex_HasMulti(t *testing.T) {
Data: []byte("data0"),
}
batch := new(leveldb.Batch)
batch := db.GetBatch(true)
for _, i := range items {
err = index.PutInBatch(batch, i)
if err != nil {
......
pkg/shed/metrics.go
View file @ 898a3fda
......@@ -13,95 +13,179 @@ type metrics struct {
// all metrics fields must be exported
// to be able to return them by Metrics()
// using reflection
PutCounter prometheus.Counter
PutFailCounter prometheus.Counter
GetCounter prometheus.Counter
GetFailCounter prometheus.Counter
GetNotFoundCounter prometheus.Counter
HasCounter prometheus.Counter
HasFailCounter prometheus.Counter
DeleteCounter prometheus.Counter
DeleteFailCounter prometheus.Counter
IteratorCounter prometheus.Counter
WriteBatchCounter prometheus.Counter
WriteBatchFailCounter prometheus.Counter
GetCount prometheus.Counter
GetFailCount prometheus.Counter
GetNotFoundCount prometheus.Counter
PutCount prometheus.Counter
PutFailCount prometheus.Counter
HasCount prometheus.Counter
HasFailCount prometheus.Counter
DeleteCount prometheus.Counter
DeleteFailCount prometheus.Counter
TotalCount prometheus.Counter
TotalFailCount prometheus.Counter
CountPrefixCount prometheus.Counter
CountPrefixFailCount prometheus.Counter
CountFromCount prometheus.Counter
CountFromFailCount prometheus.Counter
IterationCount prometheus.Counter
IterationFailCount prometheus.Counter
FirstCount prometheus.Counter
FirstFailCount prometheus.Counter
LastCount prometheus.Counter
LastFailCount prometheus.Counter
GetBatchCount prometheus.Counter
WriteBatchCount prometheus.Counter
WriteBatchFailCount prometheus.Counter
}
func newMetrics() metrics {
subsystem := "shed"
return metrics{
PutCounter: prometheus.NewCounter(prometheus.CounterOpts{
GetCount: prometheus.NewCounter(prometheus.CounterOpts{
Namespace: m.Namespace,
Subsystem: subsystem,
Name: "put_count",
Help: "Number of times the PUT operation is done.",
Name: "get_count",
Help: "Number of times a GET operation is performed.",
}),
PutFailCounter: prometheus.NewCounter(prometheus.CounterOpts{
GetFailCount: prometheus.NewCounter(prometheus.CounterOpts{
Namespace: m.Namespace,
Subsystem: subsystem,
Name: "put_fail_count",
Help: "Number of times the PUT operation failed.",
Name: "get_failure_count",
Help: "Number of times a GET operation failed.",
}),
GetCounter: prometheus.NewCounter(prometheus.CounterOpts{
GetNotFoundCount: prometheus.NewCounter(prometheus.CounterOpts{
Namespace: m.Namespace,
Subsystem: subsystem,
Name: "get_count",
Help: "Number of times the GET operation is done.",
Name: "get_not_found_count",
Help: "Number of times a GET operation failed.",
}),
GetNotFoundCounter: prometheus.NewCounter(prometheus.CounterOpts{
PutCount: prometheus.NewCounter(prometheus.CounterOpts{
Namespace: m.Namespace,
Subsystem: subsystem,
Name: "get_not_found_count",
Help: "Number of times the GET operation could not find key.",
Name: "put_count",
Help: "Number of times a PUT operation is performed.",
}),
GetFailCounter: prometheus.NewCounter(prometheus.CounterOpts{
PutFailCount: prometheus.NewCounter(prometheus.CounterOpts{
Namespace: m.Namespace,
Subsystem: subsystem,
Name: "get_fail_count",
Help: "Number of times the GET operation is failed.",
Name: "put_failure_count",
Help: "Number of times a PUT operation failed.",
}),
HasCounter: prometheus.NewCounter(prometheus.CounterOpts{
HasCount: prometheus.NewCounter(prometheus.CounterOpts{
Namespace: m.Namespace,
Subsystem: subsystem,
Name: "has_count",
Help: "Number of times the HAS operation is done.",
Help: "Number of times a HAS operation is performed.",
}),
HasFailCounter: prometheus.NewCounter(prometheus.CounterOpts{
HasFailCount: prometheus.NewCounter(prometheus.CounterOpts{
Namespace: m.Namespace,
Subsystem: subsystem,
Name: "has_fail_count",
Help: "Number of times the HAS operation failed.",
Name: "has_failure_count",
Help: "Number of times a HAS operation failed.",
}),
DeleteCounter: prometheus.NewCounter(prometheus.CounterOpts{
DeleteCount: prometheus.NewCounter(prometheus.CounterOpts{
Namespace: m.Namespace,
Subsystem: subsystem,
Name: "delete_count",
Help: "Number of times the DELETE operation is done.",
Help: "Number of times a DELETE operation is performed.",
}),
DeleteFailCount: prometheus.NewCounter(prometheus.CounterOpts{
Namespace: m.Namespace,
Subsystem: subsystem,
Name: "delete_failure_count",
Help: "Number of times a DELETE operation failed.",
}),
TotalCount: prometheus.NewCounter(prometheus.CounterOpts{
Namespace: m.Namespace,
Subsystem: subsystem,
Name: "total_count",
Help: "Number of times a COUNT operation is performed.",
}),
TotalFailCount: prometheus.NewCounter(prometheus.CounterOpts{
Namespace: m.Namespace,
Subsystem: subsystem,
Name: "total_failure_count",
Help: "Number of times a COUNT operation failed.",
}),
CountPrefixCount: prometheus.NewCounter(prometheus.CounterOpts{
Namespace: m.Namespace,
Subsystem: subsystem,
Name: "count_prefix_count",
Help: "Number of times a COUNT_PREFIX operation is performed.",
}),
CountFromFailCount: prometheus.NewCounter(prometheus.CounterOpts{
Namespace: m.Namespace,
Subsystem: subsystem,
Name: "count_from_failure_count",
Help: "Number of times a COUNT_FROM operation failed.",
}),
CountFromCount: prometheus.NewCounter(prometheus.CounterOpts{
Namespace: m.Namespace,
Subsystem: subsystem,
Name: "count_from_count",
Help: "Number of times a COUNT_FROM operation is performed.",
}),
CountPrefixFailCount: prometheus.NewCounter(prometheus.CounterOpts{
Namespace: m.Namespace,
Subsystem: subsystem,
Name: "count_prefix_failure_count",
Help: "Number of times a COUNT_PREFIX operation failed.",
}),
IterationCount: prometheus.NewCounter(prometheus.CounterOpts{
Namespace: m.Namespace,
Subsystem: subsystem,
Name: "iteration_count",
Help: "Number of times a ITERATION operation is performed.",
}),
IterationFailCount: prometheus.NewCounter(prometheus.CounterOpts{
Namespace: m.Namespace,
Subsystem: subsystem,
Name: "iteration_failure_count",
Help: "Number of times a ITERATION operation failed.",
}),
FirstCount: prometheus.NewCounter(prometheus.CounterOpts{
Namespace: m.Namespace,
Subsystem: subsystem,
Name: "first_count",
Help: "Number of times a FIRST operation is performed.",
}),
FirstFailCount: prometheus.NewCounter(prometheus.CounterOpts{
Namespace: m.Namespace,
Subsystem: subsystem,
Name: "first_failure_count",
Help: "Number of times a FIRST operation failed.",
}),
LastCount: prometheus.NewCounter(prometheus.CounterOpts{
Namespace: m.Namespace,
Subsystem: subsystem,
Name: "last_count",
Help: "Number of times a LAST operation is performed.",
}),
DeleteFailCounter: prometheus.NewCounter(prometheus.CounterOpts{
LastFailCount: prometheus.NewCounter(prometheus.CounterOpts{
Namespace: m.Namespace,
Subsystem: subsystem,
Name: "delete_fail_count",
Help: "Number of times the DELETE operation failed.",
Name: "last_failure_count",
Help: "Number of times a LAST operation failed.",
}),
IteratorCounter: prometheus.NewCounter(prometheus.CounterOpts{
GetBatchCount: prometheus.NewCounter(prometheus.CounterOpts{
Namespace: m.Namespace,
Subsystem: subsystem,
Name: "iterator_count",
Help: "Number of times the ITERATOR operation is done.",
Name: "getbatch_count",
Help: "Number of times a GET_BATCH operation is performed.",
}),
WriteBatchCounter: prometheus.NewCounter(prometheus.CounterOpts{
WriteBatchCount: prometheus.NewCounter(prometheus.CounterOpts{
Namespace: m.Namespace,
Subsystem: subsystem,
Name: "write_batch_count",
Help: "Number of times the WRITE_BATCH operation is done.",
Help: "Number of times a WRITE_BATCH operation is performed.",
}),
WriteBatchFailCounter: prometheus.NewCounter(prometheus.CounterOpts{
WriteBatchFailCount: prometheus.NewCounter(prometheus.CounterOpts{
Namespace: m.Namespace,
Subsystem: subsystem,
Name: "write_batch_fail_count",
Help: "Number of times the WRITE_BATCH operation failed.",
Name: "write_batch_failure_count",
Help: "Number of times a WRITE_BATCH operation failed.",
}),
}
}
......
pkg/shed/vector_uint64.go
View file @ 898a3fda
......@@ -19,8 +19,8 @@ package shed
import (
"encoding/binary"
"github.com/dgraph-io/badger/v2"
"github.com/ethersphere/bee/pkg/logging"
"github.com/syndtr/goleveldb/leveldb"
)
// Uint64Vector provides a way to have multiple counters in the database.
......@@ -51,7 +51,7 @@ func (db *DB) NewUint64Vector(name string, logger logging.Logger) (f Uint64Vecto
func (f Uint64Vector) Get(i uint64) (val uint64, err error) {
b, err := f.db.Get(f.indexKey(i))
if err != nil {
if err == leveldb.ErrNotFound {
if err == ErrNotFound {
return 0, nil
}
return 0, err
......@@ -66,8 +66,8 @@ func (f Uint64Vector) Put(i, val uint64) (err error) {
// PutInBatch stores a uint64 value at index i in a batch
// that can be saved later in the database.
func (f Uint64Vector) PutInBatch(batch *leveldb.Batch, i, val uint64) {
batch.Put(f.indexKey(i), encodeUint64(val))
func (f Uint64Vector) PutInBatch(batch *badger.Txn, i, val uint64) (err error){
return batch.Set(f.indexKey(i), encodeUint64(val))
}
// Inc increments a uint64 value in the database.
......@@ -75,12 +75,7 @@ func (f Uint64Vector) PutInBatch(batch *leveldb.Batch, i, val uint64) {
func (f Uint64Vector) Inc(i uint64) (val uint64, err error) {
val, err = f.Get(i)
if err != nil {
if err == leveldb.ErrNotFound {
val = 0
} else {
f.logger.Debugf("error getiing value while doing Inc. Error: %s", err.Error())
return 0, err
}
return 0, err
}
val++
return val, f.Put(i, val)
......@@ -89,18 +84,16 @@ func (f Uint64Vector) Inc(i uint64) (val uint64, err error) {
// IncInBatch increments a uint64 value at index i in the batch
// by retreiving a value from the database, not the same batch.
// This operation is not goroutine safe.
func (f Uint64Vector) IncInBatch(batch *leveldb.Batch, i uint64) (val uint64, err error) {
func (f Uint64Vector) IncInBatch(batch *badger.Txn, i uint64) (val uint64, err error) {
val, err = f.Get(i)
if err != nil {
if err == leveldb.ErrNotFound {
val = 0
} else {
f.logger.Debugf("error getiing value while doing IncInBatch. Error: %s", err.Error())
return 0, err
}
return 0, err
}
val++
f.PutInBatch(batch, i, val)
err = f.PutInBatch(batch, i, val)
if err != nil {
return 0, err
}
return val, nil
}
......@@ -110,7 +103,7 @@ func (f Uint64Vector) IncInBatch(batch *leveldb.Batch, i uint64) (val uint64, er
func (f Uint64Vector) Dec(i uint64) (val uint64, err error) {
val, err = f.Get(i)
if err != nil {
if err == leveldb.ErrNotFound {
if err == ErrNotFound {
val = 0
} else {
f.logger.Debugf("error getiing value while doing Dec. Error: %s", err.Error())
......@@ -127,20 +120,18 @@ func (f Uint64Vector) Dec(i uint64) (val uint64, err error) {
// by retreiving a value from the database, not the same batch.
// This operation is not goroutine safe.
// The field is protected from overflow to a negative value.
func (f Uint64Vector) DecInBatch(batch *leveldb.Batch, i uint64) (val uint64, err error) {
func (f Uint64Vector) DecInBatch(batch *badger.Txn, i uint64) (val uint64, err error) {
val, err = f.Get(i)
if err != nil {
if err == leveldb.ErrNotFound {
val = 0
} else {
f.logger.Debugf("error getiing value while doing DecInBatch. Error: %s", err.Error())
return 0, err
}
return 0, err
}
if val != 0 {
val--
}
f.PutInBatch(batch, i, val)
err = f.PutInBatch(batch, i, val)
if err != nil {
return 0, err
}
return val, nil
}
......
pkg/shed/vector_uint64_test.go
View file @ 898a3fda
......@@ -21,7 +21,6 @@ import (
"testing"
"github.com/ethersphere/bee/pkg/logging"
"github.com/syndtr/goleveldb/leveldb"
)
// TestUint64Vector validates put and get operations
......@@ -80,9 +79,12 @@ func TestUint64Vector(t *testing.T) {
t.Run("put in batch", func(t *testing.T) {
for _, index := range []uint64{0, 1, 2, 3, 5, 10} {
batch := new(leveldb.Batch)
batch := db.GetBatch(true)
var want uint64 = 43 + index
bins.PutInBatch(batch, index, want)
err = bins.PutInBatch(batch, index, want)
if err != nil {
t.Fatal(err)
}
err = db.WriteBatch(batch)
if err != nil {
t.Fatal(err)
......@@ -96,9 +98,12 @@ func TestUint64Vector(t *testing.T) {
}
t.Run("overwrite", func(t *testing.T) {
batch := new(leveldb.Batch)
batch := db.GetBatch(true)
var want uint64 = 85 + index
bins.PutInBatch(batch, index, want)
err = bins.PutInBatch(batch, index, want)
if err != nil {
t.Fatal(err)
}
err = db.WriteBatch(batch)
if err != nil {
t.Fatal(err)
......@@ -159,7 +164,7 @@ func TestUint64Vector_IncInBatch(t *testing.T) {
}
for _, index := range []uint64{0, 1, 2, 3, 5, 10} {
batch := new(leveldb.Batch)
batch := db.GetBatch(true)
var want uint64 = 1
got, err := bins.IncInBatch(batch, index)
if err != nil {
......@@ -180,7 +185,7 @@ func TestUint64Vector_IncInBatch(t *testing.T) {
t.Errorf("got %v uint64 %v, want %v", index, got, want)
}
batch2 := new(leveldb.Batch)
batch2 := db.GetBatch(true)
want = 2
got, err = bins.IncInBatch(batch2, index)
if err != nil {
......@@ -254,7 +259,7 @@ func TestUint64Vector_DecInBatch(t *testing.T) {
}
for _, index := range []uint64{0, 1, 2, 3, 5, 10} {
batch := new(leveldb.Batch)
batch := db.GetBatch(true)
var want uint64
got, err := bins.DecInBatch(batch, index)
if err != nil {
......@@ -275,9 +280,12 @@ func TestUint64Vector_DecInBatch(t *testing.T) {
t.Errorf("got %v uint64 %v, want %v", index, got, want)
}
batch2 := new(leveldb.Batch)
batch2 := db.GetBatch(true)
want = 42 + index
bins.PutInBatch(batch2, index, want)
err = bins.PutInBatch(batch2, index, want)
if err != nil {
t.Fatal(err)
}
err = db.WriteBatch(batch2)
if err != nil {
t.Fatal(err)
......@@ -290,7 +298,7 @@ func TestUint64Vector_DecInBatch(t *testing.T) {
t.Errorf("got %v uint64 %v, want %v", index, got, want)
}
batch3 := new(leveldb.Batch)
batch3 := db.GetBatch(true)
want = 41 + index
got, err = bins.DecInBatch(batch3, index)
if err != nil {
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment