Random manifest obfuscation key only when encrypting (#1065)

go.mod

@@ -10,7 +10,7 @@ require (
 	github.com/ethereum/go-ethereum v1.9.20
 	github.com/ethersphere/bmt v0.1.4
 	github.com/ethersphere/langos v1.0.0
-	github.com/ethersphere/manifest v0.3.5
+	github.com/ethersphere/manifest v0.3.6
 	github.com/ethersphere/sw3-bindings/v2 v2.1.0
 	github.com/gogo/protobuf v1.3.1
 	github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e // indirect

go.sum

@@ -167,8 +167,8 @@ github.com/ethersphere/bmt v0.1.4 h1:+rkWYNtMgDx6bkNqGdWu+U9DgGI1rRZplpSW3YhBr1Q
 github.com/ethersphere/bmt v0.1.4/go.mod h1:Yd8ft1U69WDuHevZc/rwPxUv1rzPSMpMnS6xbU53aY8=
 github.com/ethersphere/langos v1.0.0 h1:NBtNKzXTTRSue95uOlzPN4py7Aofs0xWPzyj4AI1Vcc=
 github.com/ethersphere/langos v1.0.0/go.mod h1:dlcN2j4O8sQ+BlCaxeBu43bgr4RQ+inJ+pHwLeZg5Tw=
-github.com/ethersphere/manifest v0.3.5 h1:/UMN4X4eKyTCARS9dv2HqqdFCJI2Emu09tivYsp5FZM=
-github.com/ethersphere/manifest v0.3.5/go.mod h1:frSxQFT67hQvmTN5CBtgVuqHzGQpg0V0oIIm/B3Am+U=
+github.com/ethersphere/manifest v0.3.6 h1:38WgYoXAQyC2lrSTArj+HM62AecX8JfUn1oVr1q+CVg=
+github.com/ethersphere/manifest v0.3.6/go.mod h1:frSxQFT67hQvmTN5CBtgVuqHzGQpg0V0oIIm/B3Am+U=
 github.com/ethersphere/sw3-bindings/v2 v2.1.0 h1:QefDtzU94UelICMPXWr7m52E2oj6r018Yc0XLoCWOxw=
 github.com/ethersphere/sw3-bindings/v2 v2.1.0/go.mod h1:ozMVBZZlAirS/FcUpFwzV60v8gC0nVbA/5ZXtCX3xCc=
 github.com/fatih/color v1.3.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=

pkg/api/bzz_test.go

@@ -99,7 +99,7 @@ func TestBzz(t *testing.T) {
 		}
 
 		// save manifest
-		m, err := manifest.NewDefaultManifest(loadsave.New(storer, storage.ModePutRequest, false))
+		m, err := manifest.NewDefaultManifest(loadsave.New(storer, storage.ModePutRequest, false), false)
 		if err != nil {
 			t.Fatal(err)
 		}

pkg/api/dirs.go

@@ -62,8 +62,9 @@ func (s *server) dirUploadHandler(w http.ResponseWriter, r *http.Request) {
 	// Add the tag to the context
 	ctx := sctx.SetTag(r.Context(), tag)
 	p := requestPipelineFn(s.Storer, r)
-	l := loadsave.New(s.Storer, requestModePut(r), requestEncrypt(r))
-	reference, err := storeDir(ctx, r.Body, s.Logger, p, l, r.Header.Get(SwarmIndexDocumentHeader), r.Header.Get(SwarmErrorDocumentHeader))
+	encrypt := requestEncrypt(r)
+	l := loadsave.New(s.Storer, requestModePut(r), encrypt)
+	reference, err := storeDir(ctx, encrypt, r.Body, s.Logger, p, l, r.Header.Get(SwarmIndexDocumentHeader), r.Header.Get(SwarmErrorDocumentHeader))
 	if err != nil {
 		logger.Debugf("dir upload: store dir err: %v", err)
 		logger.Errorf("dir upload: store dir")
@@ -103,10 +104,10 @@ func validateRequest(r *http.Request) error {
 
 // storeDir stores all files recursively contained in the directory given as a tar
 // it returns the hash for the uploaded manifest corresponding to the uploaded dir
-func storeDir(ctx context.Context, reader io.ReadCloser, log logging.Logger, p pipelineFunc, ls file.LoadSaver, indexFilename string, errorFilename string) (swarm.Address, error) {
+func storeDir(ctx context.Context, encrypt bool, reader io.ReadCloser, log logging.Logger, p pipelineFunc, ls file.LoadSaver, indexFilename string, errorFilename string) (swarm.Address, error) {
 	logger := tracing.NewLoggerWithTraceID(ctx, log)
 
-	dirManifest, err := manifest.NewDefaultManifest(ls)
+	dirManifest, err := manifest.NewDefaultManifest(ls, encrypt)
 	if err != nil {
 		return swarm.ZeroAddress, err
 	}

pkg/api/dirs_test.go

@@ -92,6 +92,8 @@ func TestDirs(t *testing.T) {
 	// valid tars
 	for _, tc := range []struct {
 		name                string
+		expectedReference   swarm.Address
+		encrypt             bool
 		wantIndexFilename   string
 		wantErrorFilename   string
 		indexFilenameOption jsonhttptest.Option
@@ -99,7 +101,8 @@ func TestDirs(t *testing.T) {
 		files               []f // files in dir for test case
 	}{
 		{
-			name: "non-nested files without extension",
+			name:              "non-nested files without extension",
+			expectedReference: swarm.MustParseHexAddress("126140bb0a33d62c4efb0523db2c26be849fcf458504618de785e2a219bad374"),
 			files: []f{
 				{
 					data:      []byte("first file data"),
@@ -122,7 +125,8 @@ func TestDirs(t *testing.T) {
 			},
 		},
 		{
-			name: "nested files with extension",
+			name:              "nested files with extension",
+			expectedReference: swarm.MustParseHexAddress("cad4b3847bd59532d9e73623d67c52e0c8d4e017d308bbaecb54f2866a91769d"),
 			files: []f{
 				{
 					data:      []byte("robots text"),
@@ -154,7 +158,8 @@ func TestDirs(t *testing.T) {
 			},
 		},
 		{
-			name: "no index filename",
+			name:              "no index filename",
+			expectedReference: swarm.MustParseHexAddress("a85aaea6a34a5c7127a3546196f2111f866fe369c6d6562ed5d3313a99388c03"),
 			files: []f{
 				{
 					data:      []byte("<h1>Swarm"),
@@ -169,6 +174,7 @@ func TestDirs(t *testing.T) {
 		},
 		{
 			name:                "explicit index filename",
+			expectedReference:   swarm.MustParseHexAddress("7d41402220f8e397ddf74d0cf4ac2055e753102bde0d622c45b03cea2b28b023"),
 			wantIndexFilename:   "index.html",
 			indexFilenameOption: jsonhttptest.WithRequestHeader(api.SwarmIndexDocumentHeader, "index.html"),
 			files: []f{
@@ -185,6 +191,7 @@ func TestDirs(t *testing.T) {
 		},
 		{
 			name:                "nested index filename",
+			expectedReference:   swarm.MustParseHexAddress("45249cf9caad842b31b29b831a1ff12aa2b711e7c282fa7a5f8c0fb544143421"),
 			wantIndexFilename:   "index.html",
 			indexFilenameOption: jsonhttptest.WithRequestHeader(api.SwarmIndexDocumentHeader, "index.html"),
 			files: []f{
@@ -201,6 +208,7 @@ func TestDirs(t *testing.T) {
 		},
 		{
 			name:                "explicit index and error filename",
+			expectedReference:   swarm.MustParseHexAddress("2046a4f758e2c0579ab923206a13fb041cec0925a6396f4f772c7ce859b8ca42"),
 			wantIndexFilename:   "index.html",
 			wantErrorFilename:   "error.html",
 			indexFilenameOption: jsonhttptest.WithRequestHeader(api.SwarmIndexDocumentHeader, "index.html"),
@@ -227,7 +235,8 @@ func TestDirs(t *testing.T) {
 			},
 		},
 		{
-			name: "invalid archive paths",
+			name:              "invalid archive paths",
+			expectedReference: swarm.MustParseHexAddress("6e6adb1ce936990cf1b7ecf8f01a8e3e8f939375b9bddb3d666151e0bdc08d4e"),
 			files: []f{
 				{
 					data:      []byte("<h1>Swarm"),
@@ -253,6 +262,18 @@ Disallow: /`),
 				},
 			},
 		},
+		{
+			name:    "encrypted",
+			encrypt: true,
+			files: []f{
+				{
+					data:     []byte("<h1>Swarm"),
+					name:     "index.html",
+					dir:      "",
+					filePath: "./index.html",
+				},
+			},
+		},
 	} {
 		t.Run(tc.name, func(t *testing.T) {
 			// tar all the test case files
@@ -271,25 +292,33 @@ Disallow: /`),
 			if tc.errorFilenameOption != nil {
 				options = append(options, tc.errorFilenameOption)
 			}
+			if tc.encrypt {
+				options = append(options, jsonhttptest.WithRequestHeader(api.SwarmEncryptHeader, "true"))
+			}
 
 			// verify directory tar upload response
 			jsonhttptest.Request(t, client, http.MethodPost, dirUploadResource, http.StatusOK, options...)
 
 			read := bytes.NewReader(respBytes)
 
-			// get the reference as everytime it will change because of random encryption key
+			// get the reference
 			var resp api.FileUploadResponse
 			err := json.NewDecoder(read).Decode(&resp)
 			if err != nil {
 				t.Fatal(err)
 			}
 
-			// NOTE: reference will be different each time, due to manifest randomness
-
 			if resp.Reference.String() == "" {
 				t.Fatalf("expected file reference, did not got any")
 			}
 
+			// NOTE: reference will be different each time when encryption is enabled
+			if !tc.encrypt {
+				if !resp.Reference.Equal(tc.expectedReference) {
+					t.Fatalf("expected root reference to match %s, got %s", tc.expectedReference, resp.Reference)
+				}
+			}
+
 			// read manifest metadata
 			j, _, err := joiner.New(context.Background(), storer, resp.Reference)
 			if err != nil {
@@ -327,8 +356,10 @@ Disallow: /`),
 
 				fileReference := entry.Reference()
 
-				if !bytes.Equal(file.reference.Bytes(), fileReference.Bytes()) {
-					t.Fatalf("expected file reference to match %s, got %s", file.reference, fileReference)
+				if !tc.encrypt {
+					if !bytes.Equal(file.reference.Bytes(), fileReference.Bytes()) {
+						t.Fatalf("expected file reference to match %s, got %s", file.reference, fileReference)
+					}
 				}
 
 				jsonhttptest.Request(t, client, http.MethodGet, fileDownloadResource(fileReference.String()), http.StatusOK,

pkg/api/file_test.go

@@ -6,7 +6,6 @@ package api_test
 
 import (
 	"bytes"
-	"encoding/json"
 	"fmt"
 	"io"
 	"io/ioutil"
@@ -281,7 +280,7 @@ func TestRangeRequests(t *testing.T) {
 			uploadEndpoint:   "/dirs",
 			downloadEndpoint: "/bzz",
 			filepath:         "/ipsum/lorem.txt",
-			reference:        "",
+			reference:        "96c68b99304b0868189e5c1d6c10be1984d93e88aab0384907f6b8814f60150b",
 			reader: tarFiles(t, []f{
 				{
 					data:      data,
@@ -355,25 +354,6 @@ func TestRangeRequests(t *testing.T) {
 				jsonhttptest.WithPutResponseBody(&respBytes),
 			)
 
-			if uploadReference == "" {
-				// NOTE: reference will be different each time, due to manifest randomness
-
-				read := bytes.NewReader(respBytes)
-
-				// get the reference as everytime it will change because of random encryption key
-				var resp api.FileUploadResponse
-				err := json.NewDecoder(read).Decode(&resp)
-				if err != nil {
-					t.Fatal(err)
-				}
-
-				if resp.Reference.String() == "" {
-					t.Fatalf("expected file reference, did not got any")
-				}
-
-				uploadReference = resp.Reference.String()
-			}
-
 			for _, tc := range ranges {
 				t.Run(tc.name, func(t *testing.T) {
 					rangeHeader, want := createRangeHeader(data, tc.ranges)

pkg/api/pin_bzz_test.go

@@ -20,7 +20,6 @@ import (
 	"github.com/ethersphere/bee/pkg/swarm"
 	"github.com/ethersphere/bee/pkg/tags"
 	"github.com/ethersphere/bee/pkg/traversal"
-	"github.com/ethersphere/manifest/mantaray"
 )
 
 func TestPinBzzHandler(t *testing.T) {
@@ -41,15 +40,6 @@ func TestPinBzzHandler(t *testing.T) {
 		})
 	)
 
-	var (
-		obfuscationKey   = make([]byte, 32)
-		obfuscationKeyFn = func(p []byte) (n int, err error) {
-			n = copy(p, obfuscationKey)
-			return
-		}
-	)
-	mantaray.SetObfuscationKeyFn(obfuscationKeyFn)
-
 	t.Run("pin-bzz-1", func(t *testing.T) {
 		files := []f{
 			{
@@ -89,7 +79,6 @@ func TestPinBzzHandler(t *testing.T) {
 
 		read := bytes.NewReader(respBytes)
 
-		// get the reference as everytime it will change because of random encryption key
 		var resp api.ListPinnedChunksResponse
 		err := json.NewDecoder(read).Decode(&resp)
 		if err != nil {

pkg/api/tag_test.go

@@ -6,7 +6,6 @@ package api_test
 
 import (
 	"bytes"
-	"encoding/json"
 	"fmt"
 	"io/ioutil"
 	"net/http"
@@ -306,32 +305,23 @@ func TestTags(t *testing.T) {
 	t.Run("dir tags", func(t *testing.T) {
 		// upload a dir without supplying tag
 		tarReader := tarFiles(t, []f{{
-			data: []byte("some data"),
+			data: []byte("some dir data"),
 			name: "binary-file",
 		}})
-
-		var respBytes []byte
+		expectedHash := swarm.MustParseHexAddress("3dc643abeb3db60a4dfb72008b577dd9a573abaa74c6afe37a75c63ceea829f6")
+		expectedResponse := api.FileUploadResponse{Reference: expectedHash}
 
 		respHeaders := jsonhttptest.Request(t, client, http.MethodPost, dirResource, http.StatusOK,
 			jsonhttptest.WithRequestBody(tarReader),
+			jsonhttptest.WithExpectedJSONResponse(expectedResponse),
 			jsonhttptest.WithRequestHeader("Content-Type", api.ContentTypeTar),
-			jsonhttptest.WithPutResponseBody(&respBytes),
 		)
 
-		read := bytes.NewReader(respBytes)
-
-		// get the reference as everytime it will change because of random encryption key
-		var resp api.FileUploadResponse
-		err := json.NewDecoder(read).Decode(&resp)
-		if err != nil {
-			t.Fatal(err)
-		}
-
 		tagId, err := strconv.Atoi(respHeaders.Get(api.SwarmTagUidHeader))
 		if err != nil {
 			t.Fatal(err)
 		}
-		tagValueTest(t, uint32(tagId), 7, 7, 1, 0, 0, 7, resp.Reference, client)
+		tagValueTest(t, uint32(tagId), 7, 7, 0, 0, 0, 7, expectedHash, client)
 	})
 
 	t.Run("bytes tags", func(t *testing.T) {

pkg/manifest/manifest.go

@@ -59,20 +59,24 @@ type Entry interface {
 }
 
 // NewDefaultManifest creates a new manifest with default type.
-func NewDefaultManifest(ls file.LoadSaver) (Interface, error) {
-	return NewManifest(DefaultManifestType, ls)
+func NewDefaultManifest(
+	ls file.LoadSaver,
+	encrypted bool,
+) (Interface, error) {
+	return NewManifest(DefaultManifestType, ls, encrypted)
 }
 
 // NewManifest creates a new manifest.
 func NewManifest(
 	manifestType string,
 	ls file.LoadSaver,
+	encrypted bool,
 ) (Interface, error) {
 	switch manifestType {
 	case ManifestSimpleContentType:
 		return NewSimpleManifest(ls)
 	case ManifestMantarayContentType:
-		return NewMantarayManifest(ls)
+		return NewMantarayManifest(ls, encrypted)
 	default:
 		return nil, ErrInvalidManifestType
 	}

pkg/manifest/mantaray.go

@@ -27,26 +27,19 @@ type mantarayManifest struct {
 }
 
 // NewMantarayManifest creates a new mantaray-based manifest.
-func NewMantarayManifest(ls file.LoadSaver) (Interface, error) {
-	return &mantarayManifest{
-		trie: mantaray.New(),
-		ls:   ls,
-	}, nil
-}
-
-// NewMantarayManifestWithObfuscationKeyFn creates a new mantaray-based manifest
-// with configured obfuscation key
-//
-// NOTE: This should only be used in tests.
-func NewMantarayManifestWithObfuscationKeyFn(
+func NewMantarayManifest(
 	ls file.LoadSaver,
-	obfuscationKeyFn func([]byte) (int, error),
+	encrypted bool,
 ) (Interface, error) {
 	mm := &mantarayManifest{
 		trie: mantaray.New(),
 		ls:   ls,
 	}
-	mantaray.SetObfuscationKeyFn(obfuscationKeyFn)
+	// use empty obfuscation key if not encrypting
+	if !encrypted {
+		// NOTE: it will be copied to all trie nodes
+		mm.trie.SetObfuscationKey(mantaray.ZeroObfuscationKey)
+	}
 	return mm, nil
 }
 

pkg/traversal/traversal_test.go

@@ -265,14 +265,6 @@ func TestTraversalManifest(t *testing.T) {
 		return traversalService.TraverseManifestAddresses
 	}
 
-	var (
-		obfuscationKey   = make([]byte, 32)
-		obfuscationKeyFn = func(p []byte) (n int, err error) {
-			n = copy(p, obfuscationKey)
-			return
-		}
-	)
-
 	testCases := []struct {
 		manifestType        string
 		files               []file
@@ -470,7 +462,7 @@ func TestTraversalManifest(t *testing.T) {
 					t.Fatal(err)
 				}
 			case manifest.ManifestMantarayContentType:
-				dirManifest, err = manifest.NewMantarayManifestWithObfuscationKeyFn(ls, obfuscationKeyFn)
+				dirManifest, err = manifest.NewMantarayManifest(ls, false)
 				if err != nil {
 					t.Fatal(err)
 				}