Add support for multiple manifest types (#591)

go.mod

@@ -8,6 +8,7 @@ require (
 	github.com/coreos/go-semver v0.3.0
 	github.com/davidlazar/go-crypto v0.0.0-20200604182044-b73af7476f6c // indirect
 	github.com/ethersphere/bmt v0.1.2
+	github.com/ethersphere/manifest v0.1.0
 	github.com/gogo/protobuf v1.3.1
 	github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e // indirect
 	github.com/gopherjs/gopherjs v0.0.0-20200217142428-fce0ec30dd00 // indirect

go.sum

@@ -110,6 +110,8 @@ github.com/ethersphere/bmt v0.1.1 h1:vwHSJwnDyzJ0fqP3YQBDk+/vqdAfulfRGJesQ5kL2ps
 github.com/ethersphere/bmt v0.1.1/go.mod h1:fqRBDmYwn3lX2MH4lkImXQgFWeNP8ikLkS/hgi/HRws=
 github.com/ethersphere/bmt v0.1.2 h1:FEuvQY9xuK+rDp3VwDVyde8T396Matv/u9PdtKa2r9Q=
 github.com/ethersphere/bmt v0.1.2/go.mod h1:fqRBDmYwn3lX2MH4lkImXQgFWeNP8ikLkS/hgi/HRws=
+github.com/ethersphere/manifest v0.1.0 h1:uVlFzAZk5SqyzjHzDgF3rNuDA4CdbJQ8fVHS4pN0iHY=
+github.com/ethersphere/manifest v0.1.0/go.mod h1:eV7hOz2c5R1ol+SpBYdS5EUG6ubh11CQe6lFAn5q+q4=
 github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
 github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc=
 github.com/francoispqt/gojay v1.2.13 h1:d2m3sFjloqoIUQU3TsHBgj6qg/BVGlTBeHDUmyJnXKk=

pkg/api/bzz.go

@@ -7,6 +7,7 @@ package api
 import (
 	"bytes"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"net/http"
 
@@ -17,17 +18,11 @@ import (
 	"github.com/ethersphere/bee/pkg/file"
 	"github.com/ethersphere/bee/pkg/file/joiner"
 	"github.com/ethersphere/bee/pkg/jsonhttp"
-	"github.com/ethersphere/bee/pkg/manifest/jsonmanifest"
+	"github.com/ethersphere/bee/pkg/manifest"
 	"github.com/ethersphere/bee/pkg/sctx"
 	"github.com/ethersphere/bee/pkg/swarm"
 )
 
-const (
-	// ManifestContentType represents content type used for noting that specific
-	// file should be processed as manifest
-	ManifestContentType = "application/bzz-manifest+json"
-)
-
 func (s *server) bzzDownloadHandler(w http.ResponseWriter, r *http.Request) {
 	targets := r.URL.Query().Get("targets")
 	r = r.WithContext(sctx.SetTargets(r.Context(), targets))
@@ -74,8 +69,8 @@ func (s *server) bzzDownloadHandler(w http.ResponseWriter, r *http.Request) {
 		jsonhttp.NotFound(w, nil)
 		return
 	}
-	metadata := &entry.Metadata{}
-	err = json.Unmarshal(buf.Bytes(), metadata)
+	manifestMetadata := &entry.Metadata{}
+	err = json.Unmarshal(buf.Bytes(), manifestMetadata)
 	if err != nil {
 		s.Logger.Debugf("bzz download: unmarshal metadata %s: %v", address, err)
 		s.Logger.Errorf("bzz download: unmarshal metadata %s", address)
@@ -84,55 +79,35 @@ func (s *server) bzzDownloadHandler(w http.ResponseWriter, r *http.Request) {
 	}
 
 	// we are expecting manifest Mime type here
-	if ManifestContentType != metadata.MimeType {
+	m, err := manifest.NewManifestReference(
+		ctx,
+		manifestMetadata.MimeType,
+		e.Reference(),
+		toDecrypt,
+		s.Storer,
+	)
+	if err != nil {
 		s.Logger.Debugf("bzz download: not manifest %s: %v", address, err)
 		s.Logger.Error("bzz download: not manifest")
 		jsonhttp.BadRequest(w, "not manifest")
 		return
 	}
 
-	// read manifest content
-	buf = bytes.NewBuffer(nil)
-	_, err = file.JoinReadAll(ctx, j, e.Reference(), buf, toDecrypt)
-	if err != nil {
-		s.Logger.Debugf("bzz download: data join %s: %v", address, err)
-		s.Logger.Errorf("bzz download: data join %s", address)
-		jsonhttp.NotFound(w, nil)
-		return
-	}
-	manifest := jsonmanifest.NewManifest()
-	err = manifest.UnmarshalBinary(buf.Bytes())
-	if err != nil {
-		s.Logger.Debugf("bzz download: unmarshal manifest %s: %v", address, err)
-		s.Logger.Errorf("bzz download: unmarshal manifest %s", address)
-		jsonhttp.InternalServerError(w, "error unmarshaling manifest")
-		return
-	}
-
-	me, err := manifest.Entry(path)
+	me, err := m.Lookup(path)
 	if err != nil {
 		s.Logger.Debugf("bzz download: invalid path %s/%s: %v", address, path, err)
 		s.Logger.Error("bzz download: invalid path")
-		jsonhttp.BadRequest(w, "invalid path address")
+
+		if errors.Is(err, manifest.ErrNotFound) {
+			jsonhttp.NotFound(w, "path address not found")
+		} else {
+			jsonhttp.BadRequest(w, "invalid path address")
+		}
 		return
 	}
 
 	manifestEntryAddress := me.Reference()
 
-	var additionalHeaders http.Header
-
-	// copy header from manifest
-	if me.Header() != nil {
-		additionalHeaders = me.Header().Clone()
-	} else {
-		additionalHeaders = http.Header{}
-	}
-
-	// include filename
-	if me.Name() != "" {
-		additionalHeaders.Set("Content-Disposition", fmt.Sprintf("inline; filename=\"%s\"", me.Name()))
-	}
-
 	// read file entry
 	buf = bytes.NewBuffer(nil)
 	_, err = file.JoinReadAll(ctx, j, manifestEntryAddress, buf, toDecrypt)
@@ -151,6 +126,29 @@ func (s *server) bzzDownloadHandler(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
+	// read file metadata
+	buf = bytes.NewBuffer(nil)
+	_, err = file.JoinReadAll(ctx, j, fe.Metadata(), buf, toDecrypt)
+	if err != nil {
+		s.Logger.Debugf("bzz download: read file metadata %s: %v", address, err)
+		s.Logger.Errorf("bzz download: read file metadata %s", address)
+		jsonhttp.NotFound(w, nil)
+		return
+	}
+	fileMetadata := &entry.Metadata{}
+	err = json.Unmarshal(buf.Bytes(), fileMetadata)
+	if err != nil {
+		s.Logger.Debugf("bzz download: unmarshal metadata %s: %v", address, err)
+		s.Logger.Errorf("bzz download: unmarshal metadata %s", address)
+		jsonhttp.InternalServerError(w, "error unmarshaling metadata")
+		return
+	}
+
+	additionalHeaders := http.Header{
+		"Content-Disposition": {fmt.Sprintf("inline; filename=\"%s\"", fileMetadata.Filename)},
+		"Content-Type":        {fileMetadata.MimeType},
+	}
+
 	fileEntryAddress := fe.Reference()
 
 	s.downloadHandler(w, r, fileEntryAddress, additionalHeaders)

pkg/api/bzz_test.go

@@ -15,14 +15,13 @@ import (
 	"strings"
 	"testing"
 
-	"github.com/ethersphere/bee/pkg/api"
 	"github.com/ethersphere/bee/pkg/collection/entry"
 	"github.com/ethersphere/bee/pkg/file"
 	"github.com/ethersphere/bee/pkg/file/splitter"
 	"github.com/ethersphere/bee/pkg/jsonhttp"
 	"github.com/ethersphere/bee/pkg/jsonhttp/jsonhttptest"
 	"github.com/ethersphere/bee/pkg/logging"
-	"github.com/ethersphere/bee/pkg/manifest/jsonmanifest"
+	"github.com/ethersphere/bee/pkg/manifest"
 	"github.com/ethersphere/bee/pkg/storage"
 	smock "github.com/ethersphere/bee/pkg/storage/mock"
 	"github.com/ethersphere/bee/pkg/swarm"
@@ -69,6 +68,7 @@ func TestBzz(t *testing.T) {
 		}
 
 		fileMetadata := entry.NewMetadata(fileName)
+		fileMetadata.MimeType = "text/html; charset=utf-8"
 		fileMetadataBytes, err := json.Marshal(fileMetadata)
 		if err != nil {
 			t.Fatal(err)
@@ -91,24 +91,26 @@ func TestBzz(t *testing.T) {
 
 		// save manifest
 
-		jsonManifest := jsonmanifest.NewManifest()
+		m, err := manifest.NewDefaultManifest(false, storer)
+		if err != nil {
+			t.Fatal(err)
+		}
 
-		e := jsonmanifest.NewEntry(fileReference, fileName, http.Header{"Content-Type": {"text/html", "charset=utf-8"}})
-		jsonManifest.Add(filePath, e)
+		e := manifest.NewEntry(fileReference)
 
-		manifestFileBytes, err := jsonManifest.MarshalBinary()
+		err = m.Add(filePath, e)
 		if err != nil {
 			t.Fatal(err)
 		}
 
-		fr, err := file.SplitWriteAll(context.Background(), sp, bytes.NewReader(manifestFileBytes), int64(len(manifestFileBytes)), false)
+		manifestBytesReference, err := m.Store(context.Background(), storage.ModePutUpload)
 		if err != nil {
 			t.Fatal(err)
 		}
 
-		m := entry.NewMetadata(fileName)
-		m.MimeType = api.ManifestContentType
-		metadataBytes, err := json.Marshal(m)
+		metadata := entry.NewMetadata(manifestBytesReference.String())
+		metadata.MimeType = m.Type()
+		metadataBytes, err := json.Marshal(metadata)
 		if err != nil {
 			t.Fatal(err)
 		}
@@ -118,8 +120,8 @@ func TestBzz(t *testing.T) {
 			t.Fatal(err)
 		}
 
-		// now join both references (mr,fr) to create an entry and store it.
-		newEntry := entry.New(fr, mr)
+		// now join both references (fr,mr) to create an entry and store it.
+		newEntry := entry.New(manifestBytesReference, mr)
 		manifestFileEntryBytes, err := newEntry.MarshalBinary()
 		if err != nil {
 			t.Fatal(err)
@@ -152,10 +154,10 @@ func TestBzz(t *testing.T) {
 
 		// check on invalid path
 
-		jsonhttptest.Request(t, client, http.MethodGet, bzzDownloadResource(manifestFileReference.String(), missingFilePath), http.StatusBadRequest,
+		jsonhttptest.Request(t, client, http.MethodGet, bzzDownloadResource(manifestFileReference.String(), missingFilePath), http.StatusNotFound,
 			jsonhttptest.WithExpectedJSONResponse(jsonhttp.StatusResponse{
-				Message: "invalid path address",
-				Code:    http.StatusBadRequest,
+				Message: "path address not found",
+				Code:    http.StatusNotFound,
 			}),
 		)
 	})

pkg/api/dirs.go

@@ -22,7 +22,7 @@ import (
 	"github.com/ethersphere/bee/pkg/file/splitter"
 	"github.com/ethersphere/bee/pkg/jsonhttp"
 	"github.com/ethersphere/bee/pkg/logging"
-	"github.com/ethersphere/bee/pkg/manifest/jsonmanifest"
+	"github.com/ethersphere/bee/pkg/manifest"
 	"github.com/ethersphere/bee/pkg/sctx"
 	"github.com/ethersphere/bee/pkg/storage"
 	"github.com/ethersphere/bee/pkg/swarm"
@@ -94,12 +94,20 @@ func validateRequest(r *http.Request) (context.Context, error) {
 // storeDir stores all files recursively contained in the directory given as a tar
 // it returns the hash for the uploaded manifest corresponding to the uploaded dir
 func storeDir(ctx context.Context, reader io.ReadCloser, s storage.Storer, mode storage.ModePut, logger logging.Logger) (swarm.Address, error) {
-	dirManifest := jsonmanifest.NewManifest()
+	v := ctx.Value(toEncryptContextKey{})
+	toEncrypt, _ := v.(bool) // default is false
+
+	dirManifest, err := manifest.NewDefaultManifest(toEncrypt, s)
+	if err != nil {
+		return swarm.ZeroAddress, err
+	}
 
 	// set up HTTP body reader
 	tarReader := tar.NewReader(reader)
 	defer reader.Close()
 
+	filesAdded := 0
+
 	// iterate through the files in the supplied tar
 	for {
 		fileHeader, err := tarReader.Next()
@@ -133,42 +141,54 @@ func storeDir(ctx context.Context, reader io.ReadCloser, s storage.Storer, mode
 		}
 		logger.Tracef("uploaded dir file %v with reference %v", filePath, fileReference)
 
-		// create manifest entry for uploaded file
-		headers := http.Header{}
-		headers.Set("Content-Type", contentType)
-		fileEntry := jsonmanifest.NewEntry(fileReference, fileName, headers)
+		// add file entry to dir manifest
+		err = dirManifest.Add(filePath, manifest.NewEntry(fileReference))
+		if err != nil {
+			return swarm.ZeroAddress, fmt.Errorf("add to manifest: %w", err)
+		}
+
+		filesAdded++
+	}
 
-		// add entry to dir manifest
-		dirManifest.Add(filePath, fileEntry)
+	// check if files were uploaded through the manifest
+	if filesAdded == 0 {
+		return swarm.ZeroAddress, fmt.Errorf("no files in tar")
 	}
 
-	// check if files were uploaded by querying manifest length
-	if dirManifest.Length() == 0 {
-		return swarm.ZeroAddress, fmt.Errorf("no files added from tar")
+	// save manifest
+	manifestBytesReference, err := dirManifest.Store(ctx, mode)
+	if err != nil {
+		return swarm.ZeroAddress, fmt.Errorf("store manifest: %w", err)
 	}
 
-	// upload manifest
-	// first, serialize into byte array
-	b, err := dirManifest.MarshalBinary()
+	// store the manifest metadata and get its reference
+	m := entry.NewMetadata(manifestBytesReference.String())
+	m.MimeType = dirManifest.Type()
+	metadataBytes, err := json.Marshal(m)
 	if err != nil {
-		return swarm.ZeroAddress, fmt.Errorf("manifest serialize: %w", err)
+		return swarm.ZeroAddress, fmt.Errorf("metadata marshal: %w", err)
 	}
 
-	// set up reader for manifest file upload
-	r := bytes.NewReader(b)
+	sp := splitter.NewSimpleSplitter(s, mode)
+	mr, err := file.SplitWriteAll(ctx, sp, bytes.NewReader(metadataBytes), int64(len(metadataBytes)), toEncrypt)
+	if err != nil {
+		return swarm.ZeroAddress, fmt.Errorf("split metadata: %w", err)
+	}
 
-	// then, upload manifest
-	manifestFileInfo := &fileUploadInfo{
-		size:        r.Size(),
-		contentType: ManifestContentType,
-		reader:      r,
+	// now join both references (fr, mr) to create an entry and store it
+	e := entry.New(manifestBytesReference, mr)
+	fileEntryBytes, err := e.MarshalBinary()
+	if err != nil {
+		return swarm.ZeroAddress, fmt.Errorf("entry marshal: %w", err)
 	}
-	manifestReference, err := storeFile(ctx, manifestFileInfo, s, mode)
+
+	sp = splitter.NewSimpleSplitter(s, mode)
+	manifestFileReference, err := file.SplitWriteAll(ctx, sp, bytes.NewReader(fileEntryBytes), int64(len(fileEntryBytes)), toEncrypt)
 	if err != nil {
-		return swarm.ZeroAddress, fmt.Errorf("store manifest: %w", err)
+		return swarm.ZeroAddress, fmt.Errorf("split entry: %w", err)
 	}
 
-	return manifestReference, nil
+	return manifestFileReference, nil
 }
 
 // storeFile uploads the given file and returns its reference

pkg/api/dirs_test.go

@@ -7,16 +7,21 @@ package api_test
 import (
 	"archive/tar"
 	"bytes"
+	"context"
+	"encoding/json"
 	"io/ioutil"
 	"net/http"
 	"path"
 	"testing"
 
 	"github.com/ethersphere/bee/pkg/api"
+	"github.com/ethersphere/bee/pkg/collection/entry"
+	"github.com/ethersphere/bee/pkg/file"
+	"github.com/ethersphere/bee/pkg/file/joiner"
 	"github.com/ethersphere/bee/pkg/jsonhttp"
 	"github.com/ethersphere/bee/pkg/jsonhttp/jsonhttptest"
 	"github.com/ethersphere/bee/pkg/logging"
-	"github.com/ethersphere/bee/pkg/manifest/jsonmanifest"
+	"github.com/ethersphere/bee/pkg/manifest"
 	"github.com/ethersphere/bee/pkg/storage/mock"
 	"github.com/ethersphere/bee/pkg/swarm"
 	"github.com/ethersphere/bee/pkg/tags"
@@ -26,8 +31,9 @@ func TestDirs(t *testing.T) {
 	var (
 		dirUploadResource    = "/dirs"
 		fileDownloadResource = func(addr string) string { return "/files/" + addr }
+		storer               = mock.NewStorer()
 		client               = newTestServer(t, testServerOptions{
-			Storer: mock.NewStorer(),
+			Storer: storer,
 			Tags:   tags.NewTags(),
 			Logger: logging.New(ioutil.Discard, 5),
 		})
@@ -82,7 +88,7 @@ func TestDirs(t *testing.T) {
 	}{
 		{
 			name:         "non-nested files without extension",
-			expectedHash: "3609d0521d34469ecbffc1d2401ce7a34c7c54bb63e8d23933ef0073015aa9e7",
+			expectedHash: "685f591d0482a57e172aecb7f58babd7eb50fcb8411f875cae5c7b96fa44ff82",
 			files: []f{
 				{
 					data:      []byte("first file data"),
@@ -106,7 +112,7 @@ func TestDirs(t *testing.T) {
 		},
 		{
 			name:         "nested files with extension",
-			expectedHash: "983869d469f0eab1f1bb6c2daeac1fdf476968246410b3001e59e9f2e0236da0",
+			expectedHash: "9e4e53c1764f2379408ffe019c097cbfcb8a0ba93587b52126a4e3e9d5b8556f",
 			files: []f{
 				{
 					data:      []byte("robots text"),
@@ -142,31 +148,76 @@ func TestDirs(t *testing.T) {
 			// tar all the test case files
 			tarReader := tarFiles(t, tc.files)
 
+			var respBytes []byte
+
 			// verify directory tar upload response
 			jsonhttptest.Request(t, client, http.MethodPost, dirUploadResource, http.StatusOK,
 				jsonhttptest.WithRequestBody(tarReader),
-				jsonhttptest.WithExpectedJSONResponse(api.FileUploadResponse{
-					Reference: swarm.MustParseHexAddress(tc.expectedHash),
-				}),
 				jsonhttptest.WithRequestHeader("Content-Type", api.ContentTypeTar),
+				jsonhttptest.WithPutResponseBody(&respBytes),
 			)
 
-			// create expected manifest
-			expectedManifest := jsonmanifest.NewManifest()
-			for _, file := range tc.files {
-				e := jsonmanifest.NewEntry(file.reference, file.name, file.header)
-				expectedManifest.Add(path.Join(file.dir, file.name), e)
+			read := bytes.NewReader(respBytes)
+
+			// get the reference as everytime it will change because of random encryption key
+			var resp api.FileUploadResponse
+			err := json.NewDecoder(read).Decode(&resp)
+			if err != nil {
+				t.Fatal(err)
+			}
+
+			if tc.expectedHash != resp.Reference.String() {
+				t.Fatalf("expected file reference to match %s, got %x", tc.expectedHash, resp.Reference)
 			}
 
-			b, err := expectedManifest.MarshalBinary()
+			// read manifest metadata
+			j := joiner.NewSimpleJoiner(storer)
+
+			buf := bytes.NewBuffer(nil)
+			_, err = file.JoinReadAll(context.Background(), j, resp.Reference, buf, false)
+			if err != nil {
+				t.Fatal(err)
+			}
+			e := &entry.Entry{}
+			err = e.UnmarshalBinary(buf.Bytes())
 			if err != nil {
 				t.Fatal(err)
 			}
 
-			// verify directory upload manifest through files api
-			jsonhttptest.Request(t, client, http.MethodGet, fileDownloadResource(tc.expectedHash), http.StatusOK,
-				jsonhttptest.WithExpectedResponse(b),
+			// verify manifest content
+			verifyManifest, err := manifest.NewManifestReference(
+				context.Background(),
+				manifest.DefaultManifestType,
+				e.Reference(),
+				false,
+				storer,
 			)
+			if err != nil {
+				t.Fatal(err)
+			}
+
+			// check if each file can be located and read
+			for _, file := range tc.files {
+				filePath := path.Join(file.dir, file.name)
+
+				entry, err := verifyManifest.Lookup(filePath)
+				if err != nil {
+					t.Fatal(err)
+				}
+
+				fileReference := entry.Reference()
+
+				if !bytes.Equal(file.reference.Bytes(), fileReference.Bytes()) {
+					t.Fatalf("expected file reference to match %x, got %x", file.reference, fileReference)
+				}
+
+				jsonhttptest.Request(t, client, http.MethodGet, fileDownloadResource(fileReference.String()), http.StatusOK,
+					jsonhttptest.WithExpectedResponse(file.data),
+					jsonhttptest.WithRequestHeader("Content-Type", file.header.Get("Content-Type")),
+				)
+
+			}
+
 		})
 	}
 }

pkg/api/file_test.go

@@ -277,7 +277,7 @@ func TestRangeRequests(t *testing.T) {
 			uploadEndpoint:   "/dirs",
 			downloadEndpoint: "/bzz",
 			filepath:         "/ipsum/lorem.txt",
-			reference:        "c1e596eebc9b39fea8f790b6ede4a294bf336e17b0cb7cd64ec54edc5c4ec0e2",
+			reference:        "d2b1ab6fb26c1570712ca33efb30f8cbbaa994d5b85e1cf6f782bcae430eabaf",
 			reader: tarFiles(t, []f{
 				{
 					data:      data,

pkg/api/tag_test.go

@@ -460,7 +460,7 @@ func TestTags(t *testing.T) {
 			name: "binary-file",
 		}})
 
-		expectedHash := swarm.MustParseHexAddress("9e5acfbfeb7e074d4c79f5f9922e8a25990dad267d0ea7becaaad07b47fb2a87")
+		expectedHash := swarm.MustParseHexAddress("ebcfbfac0e9a4fa4483491875f9486107a799e54cd832d0aacc59b1125b4b71f")
 		expectedResponse := api.FileUploadResponse{Reference: expectedHash}
 
 		respHeaders := jsonhttptest.Request(t, client, http.MethodPost, dirResource, http.StatusOK,

pkg/manifest/jsonmanifest/entry.go

-// Copyright 2020 The Swarm Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-package jsonmanifest
-
-import (
-	"net/http"
-
-	"github.com/ethersphere/bee/pkg/manifest"
-	"github.com/ethersphere/bee/pkg/swarm"
-)
-
-// verify jsonEntry implements manifest.Entry.
-var _ manifest.Entry = (*jsonEntry)(nil)
-
-// jsonEntry is a JSON representation of a single manifest entry for a jsonManifest.
-type jsonEntry struct {
-	R swarm.Address `json:"reference"`
-	N string        `json:"name"`
-	H http.Header   `json:"header"`
-}
-
-// NewEntry creates a new jsonEntry struct and returns it.
-func NewEntry(reference swarm.Address, name string, headers http.Header) manifest.Entry {
-	return &jsonEntry{
-		R: reference,
-		N: name,
-		H: headers,
-	}
-}
-
-// Reference returns the address of the file in the entry.
-func (me *jsonEntry) Reference() swarm.Address {
-	return me.R
-}
-
-// Name returns the name of the file in the entry.
-func (me *jsonEntry) Name() string {
-	return me.N
-}
-
-// Header returns the HTTP header for the file in the manifest entry.
-func (me *jsonEntry) Header() http.Header {
-	return me.H
-}

pkg/manifest/jsonmanifest/jsonmanifest_test.go

-// Copyright 2020 The Swarm Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-package jsonmanifest_test
-
-import (
-	"net/http"
-	"path/filepath"
-	"reflect"
-	"testing"
-
-	"github.com/ethersphere/bee/pkg/manifest"
-	"github.com/ethersphere/bee/pkg/manifest/jsonmanifest"
-	"github.com/ethersphere/bee/pkg/swarm"
-	"github.com/ethersphere/bee/pkg/swarm/test"
-)
-
-var testCases = []testCase{
-	{
-		name:    "empty-manifest",
-		entries: nil,
-	},
-	{
-		name: "one-entry",
-		entries: []e{
-			{
-				reference: test.RandomAddress(),
-				path:      "entry-1",
-				header:    http.Header{},
-			},
-		},
-	},
-	{
-		name: "two-entries",
-		entries: []e{
-			{
-				reference: test.RandomAddress(),
-				path:      "entry-1.txt",
-				header:    http.Header{"Content-Type": {"text/plain; charset=utf-8"}},
-			},
-			{
-				reference: test.RandomAddress(),
-				path:      "entry-2.png",
-				header:    http.Header{"Content-Type": {"image/png"}},
-			},
-		},
-	},
-	{
-		name: "nested-entries",
-		entries: []e{
-			{
-				reference: test.RandomAddress(),
-				path:      "text/robots.txt",
-				header:    http.Header{"Content-Type": {"text/plain; charset=utf-8"}},
-			},
-			{
-				reference: test.RandomAddress(),
-				path:      "img/1.png",
-				header:    http.Header{"Content-Type": {"image/png"}},
-			},
-			{
-				reference: test.RandomAddress(),
-				path:      "img/2.jpg",
-				header:    http.Header{"Content-Type": {"image/jpg"}},
-			},
-			{
-				reference: test.RandomAddress(),
-				path:      "readme.md",
-				header:    http.Header{"Content-Type": {"text/markdown; charset=UTF-8"}},
-			},
-		},
-	},
-}
-
-// TestEntries tests the Add, Length and Entry functions.
-// This test will add multiple entries to a manifest, checking that they are correctly retrieved each time,
-// and that the length of the manifest is as expected.
-// It will verify that the manifest length remains unchanged when replacing entries or removing inexistent ones.
-// Finally, it will remove all entries in the manifest, checking that they are correctly not found each time,
-// and that the length of the manifest is as expected.
-func TestEntries(t *testing.T) {
-	tc := testCases[len(testCases)-1] // get non-trivial test case
-
-	m := jsonmanifest.NewManifest()
-	checkLength(t, m, 0)
-
-	// add entries
-	for i, e := range tc.entries {
-		_, name := filepath.Split(e.path)
-		entry := jsonmanifest.NewEntry(e.reference, name, e.header)
-		m.Add(e.path, entry)
-
-		checkLength(t, m, i+1)
-		checkEntry(t, m, entry, e.path)
-	}
-
-	manifestLen := m.Length()
-
-	// replace entry
-	lastEntry := tc.entries[len(tc.entries)-1]
-	_, name := filepath.Split(lastEntry.path)
-
-	newEntry := jsonmanifest.NewEntry(test.RandomAddress(), name, lastEntry.header)
-	m.Add(lastEntry.path, newEntry)
-
-	checkLength(t, m, manifestLen) // length should not have changed
-	checkEntry(t, m, newEntry, lastEntry.path)
-
-	// remove entries
-	m.Remove("invalid/path.ext")   // try removing inexistent entry
-	checkLength(t, m, manifestLen) // length should not have changed
-
-	for i, e := range tc.entries {
-		m.Remove(e.path)
-
-		entry, err := m.Entry(e.path)
-		if entry != nil || err != manifest.ErrNotFound {
-			t.Fatalf("expected path %v not to be present in the manifest, but it was found", e.path)
-		}
-
-		checkLength(t, m, manifestLen-i-1)
-	}
-}
-
-// checkLength verifies that the given manifest length and integer match.
-func checkLength(t *testing.T, m manifest.Interface, length int) {
-	if m.Length() != length {
-		t.Fatalf("expected length to be %d, but is %d instead", length, m.Length())
-	}
-}
-
-// checkEntry verifies that an entry is equal to the one retrieved from the given manifest and path.
-func checkEntry(t *testing.T, m manifest.Interface, entry manifest.Entry, path string) {
-	re, err := m.Entry(path)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if !reflect.DeepEqual(entry, re) {
-		t.Fatalf("original and retrieved entry are not equal: %v, %v", entry, re)
-	}
-}
-
-// TestEntryModification verifies that manifest entries are not modifiable from outside of the manifest.
-// This test will add a single entry to a manifest, retrieve it, and modify it.
-// After, it will re-retrieve that same entry from the manifest, and check that it has not changed.
-func TestEntryModification(t *testing.T) {
-	m := jsonmanifest.NewManifest()
-
-	e := jsonmanifest.NewEntry(test.RandomAddress(), "single_entry.png", http.Header{"Content-Type": {"image/png"}})
-	m.Add("", e)
-
-	re, err := m.Entry("")
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	re.Header().Add("Content-Type", "text/plain; charset=utf-8") // modify retrieved entry
-
-	rre, err := m.Entry("") // re-retrieve entry
-	if err != nil {
-		t.Fatal(err)
-	}
-	if reflect.DeepEqual(rre, re) {
-		t.Fatalf("manifest entry %v was unexpectedly modified externally", rre)
-	}
-}
-
-// TestMarshal verifies that created manifests are successfully marshalled and unmarshalled.
-// This function wil add all test case entries to a manifest and marshal it.
-// After, it will unmarshal the result, and verify that it is equal to the original manifest.
-func TestMarshal(t *testing.T) {
-	for _, tc := range testCases {
-		t.Run(tc.name, func(t *testing.T) {
-			m := jsonmanifest.NewManifest()
-
-			for _, e := range tc.entries {
-				_, name := filepath.Split(e.path)
-				entry := jsonmanifest.NewEntry(e.reference, name, e.header)
-				m.Add(e.path, entry)
-			}
-
-			b, err := m.MarshalBinary()
-			if err != nil {
-				t.Fatal(err)
-			}
-
-			um := jsonmanifest.NewManifest()
-			if err := um.UnmarshalBinary(b); err != nil {
-				t.Fatal(err)
-			}
-
-			if !reflect.DeepEqual(m, um) {
-				t.Fatalf("marshalled and unmarshalled manifests are not equal: %v, %v", m, um)
-			}
-		})
-	}
-}
-
-// struct for manifest test cases
-type testCase struct {
-	name    string
-	entries []e // entries to add to manifest
-}
-
-// struct for manifest entries for test cases
-type e struct {
-	reference swarm.Address
-	path      string
-	header    http.Header
-}

pkg/manifest/jsonmanifest/manifest.go

-// Copyright 2020 The Swarm Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-package jsonmanifest
-
-import (
-	"encoding/json"
-	"sync"
-
-	"github.com/ethersphere/bee/pkg/manifest"
-)
-
-// verify jsonManifest implements manifest.Interface.
-var _ manifest.Interface = (*jsonManifest)(nil)
-
-// jsonManifest is a JSON representation of a manifest.
-// It stores manifest entries in a map based on string keys.
-type jsonManifest struct {
-	entriesMu sync.RWMutex          // mutex for accessing the entries map
-	Entries   map[string]*jsonEntry `json:"entries,omitempty"`
-}
-
-// NewManifest creates a new jsonManifest struct and returns a pointer to it.
-func NewManifest() manifest.Interface {
-	return &jsonManifest{
-		Entries: make(map[string]*jsonEntry),
-	}
-}
-
-// Add adds a manifest entry to the specified path.
-func (m *jsonManifest) Add(path string, entry manifest.Entry) {
-	m.entriesMu.Lock()
-	defer m.entriesMu.Unlock()
-
-	m.Entries[path] = &jsonEntry{
-		R: entry.Reference(),
-		N: entry.Name(),
-		H: entry.Header(),
-	}
-}
-
-// Remove removes a manifest entry on the specified path.
-func (m *jsonManifest) Remove(path string) {
-	m.entriesMu.Lock()
-	defer m.entriesMu.Unlock()
-
-	delete(m.Entries, path)
-}
-
-// Entry returns a manifest entry if one is found in the specified path.
-func (m *jsonManifest) Entry(path string) (manifest.Entry, error) {
-	m.entriesMu.RLock()
-	defer m.entriesMu.RUnlock()
-
-	entry, ok := m.Entries[path]
-	if !ok {
-		return nil, manifest.ErrNotFound
-	}
-
-	// return a copy to prevent external modification
-	return NewEntry(entry.Reference(), entry.Name(), entry.Header().Clone()), nil
-}
-
-// Length returns an implementation-specific count of elements in the manifest.
-// For jsonManifest, this means the number of all the existing entries.
-func (m *jsonManifest) Length() int {
-	m.entriesMu.RLock()
-	defer m.entriesMu.RUnlock()
-
-	return len(m.Entries)
-}
-
-// MarshalBinary implements encoding.BinaryMarshaler.
-func (m *jsonManifest) MarshalBinary() ([]byte, error) {
-	m.entriesMu.RLock()
-	defer m.entriesMu.RUnlock()
-
-	return json.Marshal(m)
-}
-
-// UnmarshalBinary implements encoding.BinaryUnmarshaler.
-func (m *jsonManifest) UnmarshalBinary(b []byte) error {
-	m.entriesMu.Lock()
-	defer m.entriesMu.Unlock()
-
-	return json.Unmarshal(b, m)
-}

pkg/manifest/manifest.go

@@ -5,36 +5,93 @@
 package manifest
 
 import (
-	"encoding"
+	"context"
 	"errors"
-	"net/http"
 
+	"github.com/ethersphere/bee/pkg/storage"
 	"github.com/ethersphere/bee/pkg/swarm"
 )
 
-// ErrNotFound is returned when an Entry is not found in the manifest.
-var ErrNotFound = errors.New("manifest: not found")
+const DefaultManifestType = ManifestSimpleContentType
+
+var (
+	// ErrNotFound is returned when an Entry is not found in the manifest.
+	ErrNotFound = errors.New("manifest: not found")
+
+	// ErrInvalidManifestType is returned when an unknown manifest type
+	// is provided to the function.
+	ErrInvalidManifestType = errors.New("manifest: invalid type")
+)
 
 // Interface for operations with manifest.
 type Interface interface {
+	// Type returns manifest implementation type information
+	Type() string
 	// Add a manifest entry to the specified path.
-	Add(string, Entry)
+	Add(string, Entry) error
 	// Remove a manifest entry on the specified path.
-	Remove(string)
-	// Entry returns a manifest entry if one is found in the specified path.
-	Entry(string) (Entry, error)
-	// Length returns an implementation-specific count of elements in the manifest.
-	Length() int
-	encoding.BinaryMarshaler
-	encoding.BinaryUnmarshaler
+	Remove(string) error
+	// Lookup returns a manifest entry if one is found in the specified path.
+	Lookup(string) (Entry, error)
+	// Store stores the manifest, returning the resulting address.
+	Store(context.Context, storage.ModePut) (swarm.Address, error)
 }
 
 // Entry represents a single manifest entry.
 type Entry interface {
-	// Reference returns the address of the file in the entry.
+	// Reference returns the address of the file.
 	Reference() swarm.Address
-	// Name returns the name of the file in the entry.
-	Name() string
-	// Header returns the HTTP header for the file in the manifest entry.
-	Header() http.Header
+}
+
+// NewDefaultManifest creates a new manifest with default type.
+func NewDefaultManifest(
+	encrypted bool,
+	storer storage.Storer,
+) (Interface, error) {
+	return NewManifest(DefaultManifestType, encrypted, storer)
+}
+
+// NewManifest creates a new manifest.
+func NewManifest(
+	manifestType string,
+	encrypted bool,
+	storer storage.Storer,
+) (Interface, error) {
+	switch manifestType {
+	case ManifestSimpleContentType:
+		return NewSimpleManifest(encrypted, storer)
+	default:
+		return nil, ErrInvalidManifestType
+	}
+}
+
+// NewManifestReference loads existing manifest.
+func NewManifestReference(
+	ctx context.Context,
+	manifestType string,
+	reference swarm.Address,
+	encrypted bool,
+	storer storage.Storer,
+) (Interface, error) {
+	switch manifestType {
+	case ManifestSimpleContentType:
+		return NewSimpleManifestReference(ctx, reference, encrypted, storer)
+	default:
+		return nil, ErrInvalidManifestType
+	}
+}
+
+type manifestEntry struct {
+	reference swarm.Address
+}
+
+// NewEntry creates a new manifest entry.
+func NewEntry(reference swarm.Address) Entry {
+	return &manifestEntry{
+		reference: reference,
+	}
+}
+
+func (e *manifestEntry) Reference() swarm.Address {
+	return e.reference
 }

pkg/manifest/simple.go

+// Copyright 2020 The Swarm Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package manifest
+
+import (
+	"bytes"
+	"context"
+	"errors"
+	"fmt"
+
+	"github.com/ethersphere/bee/pkg/file"
+	"github.com/ethersphere/bee/pkg/file/joiner"
+	"github.com/ethersphere/bee/pkg/file/splitter"
+	"github.com/ethersphere/bee/pkg/storage"
+	"github.com/ethersphere/bee/pkg/swarm"
+	"github.com/ethersphere/manifest/simple"
+)
+
+const (
+	// ManifestSimpleContentType represents content type used for noting that
+	// specific file should be processed as 'simple' manifest
+	ManifestSimpleContentType = "application/bzz-manifest-simple+json"
+)
+
+type simpleManifest struct {
+	manifest simple.Manifest
+
+	encrypted bool
+	storer    storage.Storer
+}
+
+// NewSimpleManifest creates a new simple manifest.
+func NewSimpleManifest(
+	encrypted bool,
+	storer storage.Storer,
+) (Interface, error) {
+	return &simpleManifest{
+		manifest:  simple.NewManifest(),
+		encrypted: encrypted,
+		storer:    storer,
+	}, nil
+}
+
+// NewSimpleManifestReference loads existing simple manifest.
+func NewSimpleManifestReference(
+	ctx context.Context,
+	reference swarm.Address,
+	encrypted bool,
+	storer storage.Storer,
+) (Interface, error) {
+	m := &simpleManifest{
+		manifest:  simple.NewManifest(),
+		encrypted: encrypted,
+		storer:    storer,
+	}
+	err := m.load(ctx, reference)
+	return m, err
+}
+
+func (m *simpleManifest) Type() string {
+	return ManifestSimpleContentType
+}
+
+func (m *simpleManifest) Add(path string, entry Entry) error {
+	e := entry.Reference().String()
+
+	return m.manifest.Add(path, e)
+}
+
+func (m *simpleManifest) Remove(path string) error {
+
+	err := m.manifest.Remove(path)
+	if err != nil {
+		if errors.Is(err, simple.ErrNotFound) {
+			return ErrNotFound
+		}
+		return err
+	}
+
+	return nil
+}
+
+func (m *simpleManifest) Lookup(path string) (Entry, error) {
+
+	n, err := m.manifest.Lookup(path)
+	if err != nil {
+		return nil, ErrNotFound
+	}
+
+	address, err := swarm.ParseHexAddress(n.Reference())
+	if err != nil {
+		return nil, fmt.Errorf("parse swarm address: %w", err)
+	}
+
+	entry := NewEntry(address)
+
+	return entry, nil
+}
+
+func (m *simpleManifest) Store(ctx context.Context, mode storage.ModePut) (swarm.Address, error) {
+
+	data, err := m.manifest.MarshalBinary()
+	if err != nil {
+		return swarm.ZeroAddress, fmt.Errorf("manifest marshal error: %w", err)
+	}
+
+	sp := splitter.NewSimpleSplitter(m.storer, mode)
+
+	address, err := file.SplitWriteAll(ctx, sp, bytes.NewReader(data), int64(len(data)), m.encrypted)
+	if err != nil {
+		return swarm.ZeroAddress, fmt.Errorf("manifest save error: %w", err)
+	}
+
+	return address, nil
+}
+
+func (m *simpleManifest) load(ctx context.Context, reference swarm.Address) error {
+	j := joiner.NewSimpleJoiner(m.storer)
+
+	buf := bytes.NewBuffer(nil)
+	_, err := file.JoinReadAll(ctx, j, reference, buf, m.encrypted)
+	if err != nil {
+		return fmt.Errorf("manifest load error: %w", err)
+	}
+
+	err = m.manifest.UnmarshalBinary(buf.Bytes())
+	if err != nil {
+		return fmt.Errorf("manifest unmarshal error: %w", err)
+	}
+
+	return nil
+}