Add support for 'mantaray' manifest, and make it default (#621)

go.mod

@@ -8,7 +8,7 @@ require (
 	github.com/coreos/go-semver v0.3.0
 	github.com/davidlazar/go-crypto v0.0.0-20200604182044-b73af7476f6c // indirect
 	github.com/ethersphere/bmt v0.1.2
-	github.com/ethersphere/manifest v0.1.0
+	github.com/ethersphere/manifest v0.2.0
 	github.com/gogo/protobuf v1.3.1
 	github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e // indirect
 	github.com/gopherjs/gopherjs v0.0.0-20200217142428-fce0ec30dd00 // indirect
@@ -51,7 +51,7 @@ require (
 	gitlab.com/nolash/go-mockbytes v0.0.7
 	go.opencensus.io v0.22.4 // indirect
 	go.uber.org/zap v1.15.0 // indirect
-	golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9
+	golang.org/x/crypto v0.0.0-20200728195943-123391ffb6de
 	golang.org/x/lint v0.0.0-20200302205851-738671d3881b // indirect
 	golang.org/x/mod v0.3.0 // indirect
 	golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208

go.sum

@@ -108,8 +108,8 @@ github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1m
 github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
 github.com/ethersphere/bmt v0.1.2 h1:FEuvQY9xuK+rDp3VwDVyde8T396Matv/u9PdtKa2r9Q=
 github.com/ethersphere/bmt v0.1.2/go.mod h1:fqRBDmYwn3lX2MH4lkImXQgFWeNP8ikLkS/hgi/HRws=
-github.com/ethersphere/manifest v0.1.0 h1:uVlFzAZk5SqyzjHzDgF3rNuDA4CdbJQ8fVHS4pN0iHY=
-github.com/ethersphere/manifest v0.1.0/go.mod h1:eV7hOz2c5R1ol+SpBYdS5EUG6ubh11CQe6lFAn5q+q4=
+github.com/ethersphere/manifest v0.2.0 h1:HD2ufiIaw/5Vgrl4XyeGduDJ5tn50wIhqMQoWdT2GME=
+github.com/ethersphere/manifest v0.2.0/go.mod h1:ygAx0KLhXYmKqsjUab95RCbXf8UcO7yMDjyfP0lY76Y=
 github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
 github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc=
 github.com/francoispqt/gojay v1.2.13 h1:d2m3sFjloqoIUQU3TsHBgj6qg/BVGlTBeHDUmyJnXKk=
@@ -887,6 +887,8 @@ golang.org/x/crypto v0.0.0-20200423211502-4bdfaf469ed5/go.mod h1:LzIPMQfyMNhhGPh
 golang.org/x/crypto v0.0.0-20200602180216-279210d13fed/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 h1:psW17arqaxU48Z5kZ0CQnkZWQJsqcURM6tKiBApRjXI=
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/crypto v0.0.0-20200728195943-123391ffb6de h1:ikNHVSjEfnvz6sxdSPCaPt572qowuyMDMJLLm3Db3ig=
+golang.org/x/crypto v0.0.0-20200728195943-123391ffb6de/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=

pkg/api/dirs_test.go

@@ -83,12 +83,10 @@ func TestDirs(t *testing.T) {
 	// valid tars
 	for _, tc := range []struct {
 		name  string
-		expectedHash string
 		files []f // files in dir for test case
 	}{
 		{
 			name: "non-nested files without extension",
-			expectedHash: "685f591d0482a57e172aecb7f58babd7eb50fcb8411f875cae5c7b96fa44ff82",
 			files: []f{
 				{
 					data:      []byte("first file data"),
@@ -112,7 +110,6 @@ func TestDirs(t *testing.T) {
 		},
 		{
 			name: "nested files with extension",
-			expectedHash: "9e4e53c1764f2379408ffe019c097cbfcb8a0ba93587b52126a4e3e9d5b8556f",
 			files: []f{
 				{
 					data:      []byte("robots text"),
@@ -166,8 +163,10 @@ func TestDirs(t *testing.T) {
 				t.Fatal(err)
 			}
 
-			if tc.expectedHash != resp.Reference.String() {
-				t.Fatalf("expected file reference to match %s, got %x", tc.expectedHash, resp.Reference)
+			// NOTE: reference will be different each time, due to manifest randomness
+
+			if resp.Reference.String() == "" {
+				t.Fatalf("expected file reference, did not got any")
 			}
 
 			// read manifest metadata

pkg/api/file_test.go

@@ -6,6 +6,7 @@ package api_test
 
 import (
 	"bytes"
+	"encoding/json"
 	"fmt"
 	"io"
 	"io/ioutil"
@@ -277,7 +278,7 @@ func TestRangeRequests(t *testing.T) {
 			uploadEndpoint:   "/dirs",
 			downloadEndpoint: "/bzz",
 			filepath:         "/ipsum/lorem.txt",
-			reference:        "d2b1ab6fb26c1570712ca33efb30f8cbbaa994d5b85e1cf6f782bcae430eabaf",
+			reference:        "",
 			reader: tarFiles(t, []f{
 				{
 					data:      data,
@@ -339,20 +340,41 @@ func TestRangeRequests(t *testing.T) {
 				Logger: logging.New(ioutil.Discard, 5),
 			})
 
+			uploadReference := upload.reference
+
+			var respBytes []byte
+
 			jsonhttptest.Request(t, client, http.MethodPost, upload.uploadEndpoint, http.StatusOK,
 				jsonhttptest.WithRequestBody(upload.reader),
-				jsonhttptest.WithExpectedJSONResponse(api.FileUploadResponse{
-					Reference: swarm.MustParseHexAddress(upload.reference),
-				}),
 				jsonhttptest.WithRequestHeader("Content-Type", upload.contentType),
+				jsonhttptest.WithPutResponseBody(&respBytes),
 			)
 
+			if uploadReference == "" {
+				// NOTE: reference will be different each time, due to manifest randomness
+
+				read := bytes.NewReader(respBytes)
+
+				// get the reference as everytime it will change because of random encryption key
+				var resp api.FileUploadResponse
+				err := json.NewDecoder(read).Decode(&resp)
+				if err != nil {
+					t.Fatal(err)
+				}
+
+				if resp.Reference.String() == "" {
+					t.Fatalf("expected file reference, did not got any")
+				}
+
+				uploadReference = resp.Reference.String()
+			}
+
 			for _, tc := range ranges {
 				t.Run(tc.name, func(t *testing.T) {
 					rangeHeader, want := createRangeHeader(data, tc.ranges)
 
 					var body []byte
-					respHeaders := jsonhttptest.Request(t, client, http.MethodGet, upload.downloadEndpoint+"/"+upload.reference+upload.filepath, http.StatusPartialContent,
+					respHeaders := jsonhttptest.Request(t, client, http.MethodGet, upload.downloadEndpoint+"/"+uploadReference+upload.filepath, http.StatusPartialContent,
 						jsonhttptest.WithRequestHeader("Range", rangeHeader),
 						jsonhttptest.WithPutResponseBody(&body),
 					)

pkg/api/tag_test.go

@@ -6,6 +6,7 @@ package api_test
 
 import (
 	"bytes"
+	"encoding/json"
 	"fmt"
 	"net/http"
 	"strconv"
@@ -460,15 +461,29 @@ func TestTags(t *testing.T) {
 			name: "binary-file",
 		}})
 
-		expectedHash := swarm.MustParseHexAddress("ebcfbfac0e9a4fa4483491875f9486107a799e54cd832d0aacc59b1125b4b71f")
-		expectedResponse := api.FileUploadResponse{Reference: expectedHash}
+		var respBytes []byte
 
 		respHeaders := jsonhttptest.Request(t, client, http.MethodPost, dirResource, http.StatusOK,
 			jsonhttptest.WithRequestBody(tarReader),
-			jsonhttptest.WithExpectedJSONResponse(expectedResponse),
 			jsonhttptest.WithRequestHeader("Content-Type", api.ContentTypeTar),
+			jsonhttptest.WithPutResponseBody(&respBytes),
 		)
 
+		read := bytes.NewReader(respBytes)
+
+		// get the reference as everytime it will change because of random encryption key
+		var resp api.FileUploadResponse
+		err := json.NewDecoder(read).Decode(&resp)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		// NOTE: reference will be different each time, due to manifest randomness
+
+		if resp.Reference.String() == "" {
+			t.Fatalf("expected file reference, did not got any")
+		}
+
 		tagId, err := strconv.Atoi(respHeaders.Get(api.SwarmTagUidHeader))
 		if err != nil {
 			t.Fatal(err)

pkg/manifest/manifest.go

@@ -12,7 +12,7 @@ import (
 	"github.com/ethersphere/bee/pkg/swarm"
 )
 
-const DefaultManifestType = ManifestSimpleContentType
+const DefaultManifestType = ManifestMantarayContentType
 
 var (
 	// ErrNotFound is returned when an Entry is not found in the manifest.
@@ -60,6 +60,8 @@ func NewManifest(
 	switch manifestType {
 	case ManifestSimpleContentType:
 		return NewSimpleManifest(encrypted, storer)
+	case ManifestMantarayContentType:
+		return NewMantarayManifest(encrypted, storer)
 	default:
 		return nil, ErrInvalidManifestType
 	}
@@ -76,6 +78,8 @@ func NewManifestReference(
 	switch manifestType {
 	case ManifestSimpleContentType:
 		return NewSimpleManifestReference(ctx, reference, encrypted, storer)
+	case ManifestMantarayContentType:
+		return NewMantarayManifestReference(ctx, reference, encrypted, storer)
 	default:
 		return nil, ErrInvalidManifestType
 	}

pkg/manifest/mantaray.go

+// Copyright 2020 The Swarm Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package manifest
+
+import (
+	"bytes"
+	"context"
+	"errors"
+	"fmt"
+
+	"github.com/ethersphere/bee/pkg/file"
+	"github.com/ethersphere/bee/pkg/file/joiner"
+	"github.com/ethersphere/bee/pkg/file/splitter"
+	"github.com/ethersphere/bee/pkg/storage"
+	"github.com/ethersphere/bee/pkg/swarm"
+	"github.com/ethersphere/manifest/mantaray"
+)
+
+const (
+	// ManifestMantarayContentType represents content type used for noting that
+	// specific file should be processed as mantaray manifest.
+	ManifestMantarayContentType = "application/bzz-manifest-mantaray+octet-stream"
+)
+
+type mantarayManifest struct {
+	trie *mantaray.Node
+
+	encrypted bool
+	storer    storage.Storer
+
+	loader mantaray.LoadSaver
+}
+
+// NewMantarayManifest creates a new mantaray-based manifest.
+func NewMantarayManifest(
+	encrypted bool,
+	storer storage.Storer,
+) (Interface, error) {
+	return &mantarayManifest{
+		trie:      mantaray.New(),
+		encrypted: encrypted,
+		storer:    storer,
+	}, nil
+}
+
+// NewMantarayManifestReference loads existing mantaray-based manifest.
+func NewMantarayManifestReference(
+	ctx context.Context,
+	reference swarm.Address,
+	encrypted bool,
+	storer storage.Storer,
+) (Interface, error) {
+	return &mantarayManifest{
+		trie:      mantaray.NewNodeRef(reference.Bytes()),
+		encrypted: encrypted,
+		storer:    storer,
+		loader:    newMantarayLoader(ctx, encrypted, storer),
+	}, nil
+}
+
+func (m *mantarayManifest) Type() string {
+	return ManifestMantarayContentType
+}
+
+func (m *mantarayManifest) Add(path string, entry Entry) error {
+	p := []byte(path)
+	e := entry.Reference().Bytes()
+
+	return m.trie.Add(p, e, m.loader)
+}
+
+func (m *mantarayManifest) Remove(path string) error {
+	p := []byte(path)
+
+	err := m.trie.Remove(p, m.loader)
+	if err != nil {
+		if errors.Is(err, mantaray.ErrNotFound) {
+			return ErrNotFound
+		}
+		return err
+	}
+
+	return nil
+}
+
+func (m *mantarayManifest) Lookup(path string) (Entry, error) {
+	p := []byte(path)
+
+	ref, err := m.trie.Lookup(p, m.loader)
+	if err != nil {
+		return nil, ErrNotFound
+	}
+
+	address := swarm.NewAddress(ref)
+
+	entry := NewEntry(address)
+
+	return entry, nil
+}
+
+func (m *mantarayManifest) Store(ctx context.Context, mode storage.ModePut) (swarm.Address, error) {
+
+	saver := newMantaraySaver(ctx, m.encrypted, m.storer, mode)
+
+	err := m.trie.Save(saver)
+	if err != nil {
+		return swarm.ZeroAddress, fmt.Errorf("manifest save error: %w", err)
+	}
+
+	address := swarm.NewAddress(m.trie.Reference())
+
+	return address, nil
+}
+
+// mantarayLoadSaver implements required interface 'mantaray.LoadSaver'
+type mantarayLoadSaver struct {
+	ctx       context.Context
+	encrypted bool
+	storer    storage.Storer
+	modePut   storage.ModePut
+}
+
+func newMantarayLoader(
+	ctx context.Context,
+	encrypted bool,
+	storer storage.Storer,
+) *mantarayLoadSaver {
+	return &mantarayLoadSaver{
+		ctx:       ctx,
+		encrypted: encrypted,
+		storer:    storer,
+	}
+}
+
+func newMantaraySaver(
+	ctx context.Context,
+	encrypted bool,
+	storer storage.Storer,
+	modePut storage.ModePut,
+) *mantarayLoadSaver {
+	return &mantarayLoadSaver{
+		ctx:       ctx,
+		encrypted: encrypted,
+		storer:    storer,
+		modePut:   modePut,
+	}
+}
+
+func (ls *mantarayLoadSaver) Load(ref []byte) ([]byte, error) {
+	ctx := ls.ctx
+
+	j := joiner.NewSimpleJoiner(ls.storer)
+
+	buf := bytes.NewBuffer(nil)
+	_, err := file.JoinReadAll(ctx, j, swarm.NewAddress(ref), buf, ls.encrypted)
+	if err != nil {
+		return nil, err
+	}
+
+	return buf.Bytes(), nil
+}
+
+func (ls *mantarayLoadSaver) Save(data []byte) ([]byte, error) {
+	ctx := ls.ctx
+
+	sp := splitter.NewSimpleSplitter(ls.storer, ls.modePut)
+
+	address, err := file.SplitWriteAll(ctx, sp, bytes.NewReader(data), int64(len(data)), ls.encrypted)
+	if err != nil {
+		return swarm.ZeroAddress.Bytes(), err
+	}
+
+	return address.Bytes(), nil
+}