Add chunk validation interface and content addressed validator (#98)

* Add chunk validation interface and content addressed validator

go.mod

@@ -4,8 +4,10 @@ go 1.14
 
 require (
 	github.com/btcsuite/btcd v0.20.1-beta
+	github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd // indirect
 	github.com/coreos/go-semver v0.3.0
 	github.com/dgraph-io/badger/v2 v2.0.3
+	github.com/ethersphere/bmt v0.1.0
 	github.com/gogo/protobuf v1.3.1
 	github.com/gorilla/handlers v1.4.2
 	github.com/gorilla/mux v1.7.4

pkg/api/bzzchunk_test.go

@@ -14,6 +14,7 @@ import (
 	"github.com/ethersphere/bee/pkg/jsonhttp"
 	"github.com/ethersphere/bee/pkg/jsonhttp/jsonhttptest"
 	"github.com/ethersphere/bee/pkg/storage/mock"
+	"github.com/ethersphere/bee/pkg/storage/mock/validator"
 	"github.com/ethersphere/bee/pkg/swarm"
 )
 
@@ -37,18 +38,8 @@ func TestChunkUploadDownload(t *testing.T) {
 	validContent := []byte("bbaatt")
 	invalidContent := []byte("bbaattss")
 
-	validatorF := func(addr swarm.Address, data []byte) bool {
-		if !addr.Equal(validHash) {
-			return false
-
-		}
-		if !bytes.Equal(data, validContent) {
-			return false
-		}
-		return true
-	}
-
-	mockValidatingStorer := mock.NewValidatingStorer(validatorF)
+	mockValidator := validator.NewMockValidator(validHash, validContent)
+	mockValidatingStorer := mock.NewValidatingStorer(mockValidator)
 	client, cleanup := newTestServer(t, testServerOptions{
 		Storer: mockValidatingStorer,
 	})

pkg/storage/mock/storer.go

@@ -13,7 +13,7 @@ import (
 
 type mockStorer struct {
 	store     map[string][]byte
-	validator storage.ChunkValidatorFunc
+	validator swarm.ChunkValidator
 }
 
 func NewStorer() storage.Storer {
@@ -22,10 +22,10 @@ func NewStorer() storage.Storer {
 	}
 }
 
-func NewValidatingStorer(f storage.ChunkValidatorFunc) storage.Storer {
+func NewValidatingStorer(v swarm.ChunkValidator) storage.Storer {
 	return &mockStorer{
 		store:     make(map[string][]byte),
-		validator: f,
+		validator: v,
 	}
 }
 func (m *mockStorer) Get(ctx context.Context, mode storage.ModeGet, addr swarm.Address) (ch swarm.Chunk, err error) {
@@ -39,7 +39,7 @@ func (m *mockStorer) Get(ctx context.Context, mode storage.ModeGet, addr swarm.A
 func (m *mockStorer) Put(ctx context.Context, mode storage.ModePut, chs ...swarm.Chunk) (exist []bool, err error) {
 	for _, ch := range chs {
 		if m.validator != nil {
-			if !m.validator(ch.Address(), ch.Data()) {
+			if !m.validator.Validate(ch) {
 				return nil, storage.ErrInvalidChunk
 			}
 		}

pkg/storage/mock/storer_test.go

@@ -7,6 +7,7 @@ import (
 
 	"github.com/ethersphere/bee/pkg/storage"
 	"github.com/ethersphere/bee/pkg/storage/mock"
+	"github.com/ethersphere/bee/pkg/storage/mock/validator"
 	"github.com/ethersphere/bee/pkg/swarm"
 )
 
@@ -48,48 +49,32 @@ func TestMockStorer(t *testing.T) {
 }
 
 func TestMockValidatingStorer(t *testing.T) {
-	validAddr := "aabbcc"
-	invalidAddr := "bbccdd"
+	validAddressHex := "aabbcc"
+	invalidAddressHex := "bbccdd"
 
-	keyValid, err := swarm.ParseHexAddress(validAddr)
-	if err != nil {
-		t.Fatal(err)
-	}
-	keyInvalid, err := swarm.ParseHexAddress(invalidAddr)
-	if err != nil {
-		t.Fatal(err)
-	}
+	validAddress := swarm.MustParseHexAddress(validAddressHex)
+	invalidAddress := swarm.MustParseHexAddress(invalidAddressHex)
 
 	validContent := []byte("bbaatt")
 	invalidContent := []byte("bbaattss")
 
-	validatorF := func(addr swarm.Address, data []byte) bool {
-		if !addr.Equal(keyValid) {
-			return false
-		}
-		if !bytes.Equal(data, validContent) {
-			return false
-		}
-		return true
-	}
-
-	s := mock.NewValidatingStorer(validatorF)
+	s := mock.NewValidatingStorer(validator.NewMockValidator(validAddress, validContent))
 
 	ctx := context.Background()
 
-	if _, err := s.Put(ctx, storage.ModePutUpload, swarm.NewChunk(keyValid, validContent)); err != nil {
+	if _, err := s.Put(ctx, storage.ModePutUpload, swarm.NewChunk(validAddress, validContent)); err != nil {
 		t.Fatalf("expected not error but got: %v", err)
 	}
 
-	if _, err := s.Put(ctx, storage.ModePutUpload, swarm.NewChunk(keyInvalid, validContent)); err == nil {
+	if _, err := s.Put(ctx, storage.ModePutUpload, swarm.NewChunk(invalidAddress, validContent)); err == nil {
 		t.Fatalf("expected error but got none")
 	}
 
-	if _, err := s.Put(ctx, storage.ModePutUpload, swarm.NewChunk(keyInvalid, invalidContent)); err == nil {
+	if _, err := s.Put(ctx, storage.ModePutUpload, swarm.NewChunk(invalidAddress, invalidContent)); err == nil {
 		t.Fatalf("expected error but got none")
 	}
 
-	if chunk, err := s.Get(ctx, storage.ModeGetRequest, keyValid); err != nil {
+	if chunk, err := s.Get(ctx, storage.ModeGetRequest, validAddress); err != nil {
 		t.Fatalf("got error on get but expected none: %v", err)
 	} else {
 		if !bytes.Equal(chunk.Data(), validContent) {
@@ -97,7 +82,7 @@ func TestMockValidatingStorer(t *testing.T) {
 		}
 	}
 
-	if _, err := s.Get(ctx, storage.ModeGetRequest, keyInvalid); err == nil {
+	if _, err := s.Get(ctx, storage.ModeGetRequest, invalidAddress); err == nil {
 		t.Fatal("got no error on get but expected one")
 	}
 

pkg/storage/mock/validator/validator.go

+// Copyright 2020 The Swarm Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+package validator
+
+import (
+	"bytes"
+
+	"github.com/ethersphere/bee/pkg/swarm"
+)
+
+// MockValidator returns true if the data and address passed in the Validate method
+// are a byte-wise match to the data and address passed to the constructor
+type MockValidator struct {
+	validAddress swarm.Address
+	validContent []byte
+	swarm.ChunkValidator
+}
+
+// NewMockValidator constructs a new MockValidator
+func NewMockValidator(address swarm.Address, data []byte) *MockValidator {
+	return &MockValidator{
+		validAddress: address,
+		validContent: data,
+	}
+}
+
+// Validate checkes the passed chunk for validity
+func (v *MockValidator) Validate(ch swarm.Chunk) (valid bool) {
+	if !v.validAddress.Equal(ch.Address()) {
+		return false
+	}
+	if !bytes.Equal(v.validContent, ch.Data()) {
+		return false
+	}
+	return true
+}

pkg/storage/store.go

@@ -18,9 +18,6 @@ var (
 	ErrInvalidChunk = errors.New("storage: invalid chunk")
 )
 
-// ChunkValidatorFunc validates Swarm chunk address and chunk data
-type ChunkValidatorFunc func(swarm.Address, []byte) (valid bool)
-
 // ModeGet enumerates different Getter modes.
 type ModeGet int
 

pkg/swarm/swarm.go

@@ -14,6 +14,7 @@ import (
 
 const (
 	ChunkSize = 4096
+	SectionSize = 32
 	MaxPO     = 16
 )
 
@@ -142,3 +143,7 @@ func (c *chunk) TagID() uint32 {
 func (self *chunk) String() string {
 	return fmt.Sprintf("Address: %v Chunksize: %v", self.addr.String(), len(self.sdata))
 }
+
+type ChunkValidator interface {
+	Validate(ch Chunk) (valid bool)
+}

pkg/validator/validator.go

+// Copyright 2020 The Swarm Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package validator contains file-oriented chunk validation implementations
+package validator
+
+import (
+	"encoding/binary"
+	"hash"
+
+	"github.com/ethersphere/bee/pkg/swarm"
+	"github.com/ethersphere/bmt"
+	"github.com/ethersphere/bee/pkg/logging"
+	bmtlegacy "github.com/ethersphere/bmt/legacy"
+	"golang.org/x/crypto/sha3"
+)
+
+var _ swarm.ChunkValidator = (*ContentAddressValidator)(nil)
+
+func hashFunc() hash.Hash {
+	return sha3.NewLegacyKeccak256()
+}
+
+// ContentAddressValidator validates that the address of a given chunk
+// is the content address of its contents
+type ContentAddressValidator struct {
+	hasher bmt.Hash
+	logger logging.Logger
+}
+
+// New constructs a new ContentAddressValidator
+func NewContentAddressValidator() *ContentAddressValidator {
+	p := bmtlegacy.NewTreePool(hashFunc, swarm.SectionSize, bmtlegacy.PoolSize)
+
+	return &ContentAddressValidator{
+		hasher: bmtlegacy.New(p),
+	}
+}
+
+// Validate performs the validation check
+func (v *ContentAddressValidator) Validate(ch swarm.Chunk) (valid bool) {
+
+	// prepare data
+	data := ch.Data()
+	address := ch.Address()
+	span := binary.LittleEndian.Uint64(data[:8])
+
+	// execute hash, compare and return result
+	v.hasher.Reset()
+	err := v.hasher.SetSpan(int64(span))
+	if err != nil {
+		v.logger.Debugf("SetSpan on bmt legacy hasher gave error: %v", err)
+		return false
+	}
+	_, err = v.hasher.Write(data[8:])
+	if err != nil {
+		v.logger.Debugf("Write on bmt legacy hasher gave error: %v", err)
+		return false
+	}
+	s := v.hasher.Sum(nil)
+
+	return address.Equal(swarm.NewAddress(s))
+}

pkg/validator/validator_test.go

+// Copyright 2020 The Swarm Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+package validator_test
+
+import (
+	"encoding/binary"
+	"testing"
+
+	"github.com/ethersphere/bee/pkg/validator"
+	"github.com/ethersphere/bee/pkg/swarm"
+)
+
+// TestContentAddressValidator checks that the validator evaluates correctly
+// on valid and invalid input
+func TestContentAddressValidator(t *testing.T) {
+
+	// instantiate validator
+	validator := validator.NewContentAddressValidator()
+
+	// generate address from pre-generated hex of 'foo' from legacy bmt
+	bmtHashOfFoo := "b9d678ef39fa973b430795a1f04e0f2541b47c996fd300552a1e8bfb5824325f"
+	address := swarm.MustParseHexAddress(bmtHashOfFoo)
+
+	// set up a chunk object with correct expected length prefix
+	// and test validation result
+	foo := "foo"
+	fooLength := len(foo)
+	fooBytes := make([]byte, 8+fooLength)
+	binary.LittleEndian.PutUint64(fooBytes, uint64(fooLength))
+	copy(fooBytes[8:], []byte(foo))
+	ch := swarm.NewChunk(address, fooBytes)
+	if !validator.Validate(ch) {
+		t.Fatalf("data '%s' should have validated to hash '%x'", ch.Data(), ch.Address())
+	}
+
+	// now test with incorrect data
+	ch = swarm.NewChunk(address, fooBytes[:len(fooBytes)-1])
+	if validator.Validate(ch) {
+		t.Fatalf("data '%s' should not have validated to hash '%x'", ch.Data(), ch.Address())
+	}
+}