manifest, jsonmanifest, api, node: add encoding marshaling to manifest interface (#495)

manifest, jsonmanifest, api, node: remove parser from jsonmanifest and replace with binary marshalling

pkg/api/api.go

@@ -8,7 +8,6 @@ import (
 	"net/http"
 
 	"github.com/ethersphere/bee/pkg/logging"
-	"github.com/ethersphere/bee/pkg/manifest"
 	m "github.com/ethersphere/bee/pkg/metrics"
 	"github.com/ethersphere/bee/pkg/storage"
 	"github.com/ethersphere/bee/pkg/tags"
@@ -29,7 +28,6 @@ type server struct {
 type Options struct {
 	Tags               *tags.Tags
 	Storer             storage.Storer
-	ManifestParser     manifest.Parser
 	CORSAllowedOrigins []string
 	Logger             logging.Logger
 	Tracer             *tracing.Tracer

pkg/api/api_test.go

@@ -13,7 +13,6 @@ import (
 
 	"github.com/ethersphere/bee/pkg/api"
 	"github.com/ethersphere/bee/pkg/logging"
-	"github.com/ethersphere/bee/pkg/manifest"
 	"github.com/ethersphere/bee/pkg/pingpong"
 	"github.com/ethersphere/bee/pkg/storage"
 	"github.com/ethersphere/bee/pkg/tags"
@@ -21,11 +20,10 @@ import (
 )
 
 type testServerOptions struct {
-	Pingpong       pingpong.Interface
-	Storer         storage.Storer
-	ManifestParser manifest.Parser
-	Tags           *tags.Tags
-	Logger         logging.Logger
+	Pingpong pingpong.Interface
+	Storer   storage.Storer
+	Tags     *tags.Tags
+	Logger   logging.Logger
 }
 
 func newTestServer(t *testing.T, o testServerOptions) *http.Client {
@@ -33,10 +31,9 @@ func newTestServer(t *testing.T, o testServerOptions) *http.Client {
 		o.Logger = logging.New(ioutil.Discard, 0)
 	}
 	s := api.New(api.Options{
-		Tags:           o.Tags,
-		Storer:         o.Storer,
-		ManifestParser: o.ManifestParser,
-		Logger:         o.Logger,
+		Tags:   o.Tags,
+		Storer: o.Storer,
+		Logger: o.Logger,
 	})
 	ts := httptest.NewServer(s)
 	t.Cleanup(ts.Close)

pkg/api/bzz.go

@@ -16,6 +16,7 @@ import (
 	"github.com/ethersphere/bee/pkg/file"
 	"github.com/ethersphere/bee/pkg/file/joiner"
 	"github.com/ethersphere/bee/pkg/jsonhttp"
+	"github.com/ethersphere/bee/pkg/manifest/jsonmanifest"
 	"github.com/ethersphere/bee/pkg/swarm"
 	"github.com/gorilla/mux"
 )
@@ -98,7 +99,8 @@ func (s *server) bzzDownloadHandler(w http.ResponseWriter, r *http.Request) {
 		jsonhttp.NotFound(w, nil)
 		return
 	}
-	manifest, err := s.ManifestParser.Parse(buf.Bytes())
+	manifest := jsonmanifest.NewManifest()
+	err = manifest.UnmarshalBinary(buf.Bytes())
 	if err != nil {
 		s.Logger.Debugf("bzz download: unmarshal manifest %s: %v", address, err)
 		s.Logger.Errorf("bzz download: unmarshal manifest %s", address)
@@ -106,7 +108,7 @@ func (s *server) bzzDownloadHandler(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	me, err := manifest.FindEntry(path)
+	me, err := manifest.Entry(path)
 	if err != nil {
 		s.Logger.Debugf("bzz download: invalid path %s/%s: %v", address, path, err)
 		s.Logger.Error("bzz download: invalid path")
@@ -114,20 +116,20 @@ func (s *server) bzzDownloadHandler(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	manifestEntryAddress := me.GetReference()
+	manifestEntryAddress := me.Reference()
 
 	var additionalHeaders http.Header
 
 	// copy headers from manifest
-	if me.GetHeaders() != nil {
-		additionalHeaders = me.GetHeaders().Clone()
+	if me.Headers() != nil {
+		additionalHeaders = me.Headers().Clone()
 	} else {
 		additionalHeaders = http.Header{}
 	}
 
 	// include filename
-	if me.GetName() != "" {
-		additionalHeaders.Set("Content-Disposition", fmt.Sprintf("inline; filename=\"%s\"", me.GetName()))
+	if me.Name() != "" {
+		additionalHeaders.Set("Content-Disposition", fmt.Sprintf("inline; filename=\"%s\"", me.Name()))
 	}
 
 	// read file entry

pkg/api/bzz_test.go

@@ -34,10 +34,9 @@ func TestBzz(t *testing.T) {
 		storer              = smock.NewStorer()
 		sp                  = splitter.NewSimpleSplitter(storer)
 		client              = newTestServer(t, testServerOptions{
-			Storer:         storer,
-			ManifestParser: jsonmanifest.NewParser(),
-			Tags:           tags.NewTags(),
-			Logger:         logging.New(ioutil.Discard, 5),
+			Storer: storer,
+			Tags:   tags.NewTags(),
+			Logger: logging.New(ioutil.Discard, 5),
 		})
 	)
 
@@ -93,15 +92,10 @@ func TestBzz(t *testing.T) {
 
 		jsonManifest := jsonmanifest.NewManifest()
 
-		jsonManifest.Add(filePath, jsonmanifest.JSONEntry{
-			Reference: fileReference,
-			Name:      fileName,
-			Headers: http.Header{
-				"Content-Type": {"text/html", "charset=utf-8"},
-			},
-		})
+		e := jsonmanifest.NewEntry(fileReference, fileName, http.Header{"Content-Type": {"text/html", "charset=utf-8"}})
+		jsonManifest.Add(filePath, e)
 
-		manifestFileBytes, err := jsonManifest.Serialize()
+		manifestFileBytes, err := jsonManifest.MarshalBinary()
 		if err != nil {
 			t.Fatal(err)
 		}

pkg/api/dirs.go

@@ -117,11 +117,7 @@ func storeDir(ctx context.Context, reader io.ReadCloser, s storage.Storer, logge
 		// create manifest entry for uploaded file
 		headers := http.Header{}
 		headers.Set("Content-Type", contentType)
-		fileEntry := &jsonmanifest.JSONEntry{
-			Reference: fileReference,
-			Name:      fileName,
-			Headers:   headers,
-		}
+		fileEntry := jsonmanifest.NewEntry(fileReference, fileName, headers)
 
 		// add entry to dir manifest
 		dirManifest.Add(filePath, fileEntry)
@@ -134,7 +130,7 @@ func storeDir(ctx context.Context, reader io.ReadCloser, s storage.Storer, logge
 
 	// upload manifest
 	// first, serialize into byte array
-	b, err := dirManifest.Serialize()
+	b, err := dirManifest.MarshalBinary()
 	if err != nil {
 		return swarm.ZeroAddress, fmt.Errorf("manifest serialize error: %w", err)
 	}

pkg/api/dirs_test.go

@@ -146,15 +146,11 @@ func TestDirs(t *testing.T) {
 			// create expected manifest
 			expectedManifest := jsonmanifest.NewManifest()
 			for _, file := range tc.files {
-				e := &jsonmanifest.JSONEntry{
-					Reference: file.reference,
-					Name:      file.name,
-					Headers:   file.headers,
-				}
+				e := jsonmanifest.NewEntry(file.reference, file.name, file.headers)
 				expectedManifest.Add(path.Join(file.dir, file.name), e)
 			}
 
-			b, err := expectedManifest.Serialize()
+			b, err := expectedManifest.MarshalBinary()
 			if err != nil {
 				t.Fatal(err)
 			}

pkg/manifest/jsonmanifest/entry.go

+// Copyright 2020 The Swarm Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package jsonmanifest
+
+import (
+	"encoding/json"
+	"net/http"
+
+	"github.com/ethersphere/bee/pkg/manifest"
+	"github.com/ethersphere/bee/pkg/swarm"
+)
+
+// verify JSONEntry implements manifest.Entry.
+var _ manifest.Entry = (*JSONEntry)(nil)
+
+// JSONEntry is a JSON representation of a single manifest entry for a JSONManifest.
+type JSONEntry struct {
+	reference swarm.Address
+	name      string
+	headers   http.Header
+}
+
+// NewEntry creates a new JSONEntry struct and returns it.
+func NewEntry(reference swarm.Address, name string, headers http.Header) *JSONEntry {
+	return &JSONEntry{
+		reference: reference,
+		name:      name,
+		headers:   headers,
+	}
+}
+
+// Reference returns the address of the file in the entry.
+func (me *JSONEntry) Reference() swarm.Address {
+	return me.reference
+}
+
+// Name returns the name of the file in the entry.
+func (me *JSONEntry) Name() string {
+	return me.name
+}
+
+// Headers returns the headers for the file in the manifest entry.
+func (me *JSONEntry) Headers() http.Header {
+	return me.headers
+}
+
+// exportEntry is a struct used for marshaling and unmarshaling JSONEntry structs.
+type exportEntry struct {
+	Reference swarm.Address `json:"reference"`
+	Name      string        `json:"name"`
+	Headers   http.Header   `json:"headers"`
+}
+
+// MarshalJSON implements the json.Marshaler interface.
+func (me *JSONEntry) MarshalJSON() ([]byte, error) {
+	return json.Marshal(exportEntry{
+		Reference: me.reference,
+		Name:      me.name,
+		Headers:   me.headers,
+	})
+}
+
+// UnmarshalJSON implements the json.Unmarshaler interface.
+func (me *JSONEntry) UnmarshalJSON(b []byte) error {
+	e := exportEntry{}
+	if err := json.Unmarshal(b, &e); err != nil {
+		return err
+	}
+	me.reference = e.Reference
+	me.name = e.Name
+	me.headers = e.Headers
+	return nil
+}

pkg/manifest/jsonmanifest/json.go → pkg/manifest/jsonmanifest/manifest.go

@@ -6,52 +6,38 @@ package jsonmanifest
 
 import (
 	"encoding/json"
-	"net/http"
 
 	"github.com/ethersphere/bee/pkg/manifest"
-	"github.com/ethersphere/bee/pkg/swarm"
 )
 
-var _ manifest.Parser = (*JSONParser)(nil)
-
-type JSONParser struct{}
-
-func NewParser() *JSONParser {
-	return &JSONParser{}
-}
-
-func (m *JSONParser) Parse(bytes []byte) (manifest.Interface, error) {
-	mi := &JSONManifest{}
-	err := json.Unmarshal(bytes, mi)
-
-	return mi, err
-}
-
+// verify JSONManifest implements manifest.Interface.
 var _ manifest.Interface = (*JSONManifest)(nil)
 
+// JSONManifest is a JSON representation of a manifest.
+// It stores manifest entries in a map based on string keys.
 type JSONManifest struct {
-	Entries map[string]JSONEntry `json:"entries,omitempty"`
+	Entries map[string]*JSONEntry `json:"entries,omitempty"`
 }
 
+// NewManifest creates a new JSONManifest struct and returns a pointer to it.
 func NewManifest() *JSONManifest {
 	return &JSONManifest{
-		Entries: make(map[string]JSONEntry),
+		Entries: make(map[string]*JSONEntry),
 	}
 }
 
+// Add adds a manifest entry to the specified path.
 func (m *JSONManifest) Add(path string, entry manifest.Entry) {
-	m.Entries[path] = JSONEntry{
-		Reference: entry.GetReference(),
-		Name:      entry.GetName(),
-		Headers:   entry.GetHeaders(),
-	}
+	m.Entries[path] = NewEntry(entry.Reference(), entry.Name(), entry.Headers())
 }
 
+// Remove removes a manifest entry on the specified path.
 func (m *JSONManifest) Remove(path string) {
 	delete(m.Entries, path)
 }
 
-func (m *JSONManifest) FindEntry(path string) (manifest.Entry, error) {
+// Entry returns a manifest entry if one is found in the specified path.
+func (m *JSONManifest) Entry(path string) (manifest.Entry, error) {
 	if entry, ok := m.Entries[path]; ok {
 		return entry, nil
 	}
@@ -59,26 +45,12 @@ func (m *JSONManifest) FindEntry(path string) (manifest.Entry, error) {
 	return nil, manifest.ErrNotFound
 }
 
-func (m *JSONManifest) Serialize() ([]byte, error) {
+// MarshalBinary implements encoding.BinaryMarshaler.
+func (m *JSONManifest) MarshalBinary() (data []byte, err error) {
 	return json.Marshal(m)
 }
 
-var _ manifest.Entry = (*JSONEntry)(nil)
-
-type JSONEntry struct {
-	Reference swarm.Address `json:"reference"`
-	Name      string        `json:"name"`
-	Headers   http.Header   `json:"headers"`
-}
-
-func (me JSONEntry) GetReference() swarm.Address {
-	return me.Reference
-}
-
-func (me JSONEntry) GetName() string {
-	return me.Name
-}
-
-func (me JSONEntry) GetHeaders() http.Header {
-	return me.Headers
+// UnmarshalBinary implements encoding.BinaryUnmarshaler.
+func (m *JSONManifest) UnmarshalBinary(data []byte) error {
+	return json.Unmarshal(data, m)
 }

pkg/manifest/manifest.go

@@ -5,38 +5,34 @@
 package manifest
 
 import (
+	"encoding"
 	"errors"
 	"net/http"
 
 	"github.com/ethersphere/bee/pkg/swarm"
 )
 
+// ErrNotFound is returned when an Entry is not found in the manifest.
 var ErrNotFound = errors.New("manifest: not found")
 
-// Parser for manifest
-type Parser interface {
-	// Parse parses the encoded manifest data and returns the result
-	Parse(bytes []byte) (Interface, error)
-}
-
-// Interface for operations with manifest
+// Interface for operations with manifest.
 type Interface interface {
-	// Add a manifest entry to specified path
+	// Add a manifest entry to the specified path.
 	Add(string, Entry)
-	// Remove reference from file on specified path
+	// Remove a manifest entry on the specified path.
 	Remove(string)
-	// FindEntry returns manifest entry if one is found on specified path
-	FindEntry(string) (Entry, error)
-	// Serialize return encoded manifest
-	Serialize() ([]byte, error)
+	// Entry returns a manifest entry if one is found in the specified path.
+	Entry(string) (Entry, error)
+	encoding.BinaryMarshaler
+	encoding.BinaryUnmarshaler
 }
 
-// Entry represents single manifest entry
+// Entry represents a single manifest entry.
 type Entry interface {
-	// GetReference returns address of the entry file
-	GetReference() swarm.Address
-	// GetName returns the name of the file for the entry, if added
-	GetName() string
-	// GetHeaders returns the headers for manifest entry, if configured
-	GetHeaders() http.Header
+	// Reference returns the address of the file in the entry.
+	Reference() swarm.Address
+	// Name returns the name of the file in the entry.
+	Name() string
+	// Headers returns the headers for the file in the manifest entry.
+	Headers() http.Header
 }

pkg/node/node.go

@@ -29,7 +29,6 @@ import (
 	memkeystore "github.com/ethersphere/bee/pkg/keystore/mem"
 	"github.com/ethersphere/bee/pkg/localstore"
 	"github.com/ethersphere/bee/pkg/logging"
-	"github.com/ethersphere/bee/pkg/manifest/jsonmanifest"
 	"github.com/ethersphere/bee/pkg/metrics"
 	"github.com/ethersphere/bee/pkg/netstore"
 	"github.com/ethersphere/bee/pkg/p2p"
@@ -302,15 +301,12 @@ func NewBee(o Options) (*Bee, error) {
 
 	b.pullerCloser = puller
 
-	manifestParser := jsonmanifest.NewParser()
-
 	var apiService api.Service
 	if o.APIAddr != "" {
 		// API server
 		apiService = api.New(api.Options{
 			Tags:               tagg,
 			Storer:             ns,
-			ManifestParser:     manifestParser,
 			CORSAllowedOrigins: o.CORSAllowedOrigins,
 			Logger:             logger,
 			Tracer:             tracer,