Skip to content

M
mybee
Commit 370a0e97 authored by Nemanja Zbiljić's avatar Nemanja Zbiljić Committed by GitHub
Browse files
Options

Add support for multiple manifest types (#591)

parent 6177d16c
Hide whitespace changes
Inline Side-by-side
Showing with 384 additions and 464 deletions
go.mod
View file @ 370a0e97
......@@ -8,6 +8,7 @@ require (
github.com/coreos/go-semver v0.3.0
github.com/davidlazar/go-crypto v0.0.0-20200604182044-b73af7476f6c // indirect
github.com/ethersphere/bmt v0.1.2
github.com/ethersphere/manifest v0.1.0
github.com/gogo/protobuf v1.3.1
github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e // indirect
github.com/gopherjs/gopherjs v0.0.0-20200217142428-fce0ec30dd00 // indirect
......
go.sum
View file @ 370a0e97
......@@ -110,6 +110,8 @@ github.com/ethersphere/bmt v0.1.1 h1:vwHSJwnDyzJ0fqP3YQBDk+/vqdAfulfRGJesQ5kL2ps
github.com/ethersphere/bmt v0.1.1/go.mod h1:fqRBDmYwn3lX2MH4lkImXQgFWeNP8ikLkS/hgi/HRws=
github.com/ethersphere/bmt v0.1.2 h1:FEuvQY9xuK+rDp3VwDVyde8T396Matv/u9PdtKa2r9Q=
github.com/ethersphere/bmt v0.1.2/go.mod h1:fqRBDmYwn3lX2MH4lkImXQgFWeNP8ikLkS/hgi/HRws=
github.com/ethersphere/manifest v0.1.0 h1:uVlFzAZk5SqyzjHzDgF3rNuDA4CdbJQ8fVHS4pN0iHY=
github.com/ethersphere/manifest v0.1.0/go.mod h1:eV7hOz2c5R1ol+SpBYdS5EUG6ubh11CQe6lFAn5q+q4=
github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc=
github.com/francoispqt/gojay v1.2.13 h1:d2m3sFjloqoIUQU3TsHBgj6qg/BVGlTBeHDUmyJnXKk=
......
pkg/api/bzz.go
View file @ 370a0e97
......@@ -7,6 +7,7 @@ package api
import (
"bytes"
"encoding/json"
"errors"
"fmt"
"net/http"
......@@ -17,17 +18,11 @@ import (
"github.com/ethersphere/bee/pkg/file"
"github.com/ethersphere/bee/pkg/file/joiner"
"github.com/ethersphere/bee/pkg/jsonhttp"
"github.com/ethersphere/bee/pkg/manifest/jsonmanifest"
"github.com/ethersphere/bee/pkg/manifest"
"github.com/ethersphere/bee/pkg/sctx"
"github.com/ethersphere/bee/pkg/swarm"
)
const (
// ManifestContentType represents content type used for noting that specific
// file should be processed as manifest
ManifestContentType = "application/bzz-manifest+json"
)
func (s *server) bzzDownloadHandler(w http.ResponseWriter, r *http.Request) {
targets := r.URL.Query().Get("targets")
r = r.WithContext(sctx.SetTargets(r.Context(), targets))
......@@ -74,8 +69,8 @@ func (s *server) bzzDownloadHandler(w http.ResponseWriter, r *http.Request) {
jsonhttp.NotFound(w, nil)
return
}
metadata := &entry.Metadata{}
err = json.Unmarshal(buf.Bytes(), metadata)
manifestMetadata := &entry.Metadata{}
err = json.Unmarshal(buf.Bytes(), manifestMetadata)
if err != nil {
s.Logger.Debugf("bzz download: unmarshal metadata %s: %v", address, err)
s.Logger.Errorf("bzz download: unmarshal metadata %s", address)
......@@ -84,55 +79,35 @@ func (s *server) bzzDownloadHandler(w http.ResponseWriter, r *http.Request) {
}
// we are expecting manifest Mime type here
if ManifestContentType != metadata.MimeType {
m, err := manifest.NewManifestReference(
ctx,
manifestMetadata.MimeType,
e.Reference(),
toDecrypt,
s.Storer,
)
if err != nil {
s.Logger.Debugf("bzz download: not manifest %s: %v", address, err)
s.Logger.Error("bzz download: not manifest")
jsonhttp.BadRequest(w, "not manifest")
return
}
// read manifest content
buf = bytes.NewBuffer(nil)
_, err = file.JoinReadAll(ctx, j, e.Reference(), buf, toDecrypt)
if err != nil {
s.Logger.Debugf("bzz download: data join %s: %v", address, err)
s.Logger.Errorf("bzz download: data join %s", address)
jsonhttp.NotFound(w, nil)
return
}
manifest := jsonmanifest.NewManifest()
err = manifest.UnmarshalBinary(buf.Bytes())
if err != nil {
s.Logger.Debugf("bzz download: unmarshal manifest %s: %v", address, err)
s.Logger.Errorf("bzz download: unmarshal manifest %s", address)
jsonhttp.InternalServerError(w, "error unmarshaling manifest")
return
}
me, err := manifest.Entry(path)
me, err := m.Lookup(path)
if err != nil {
s.Logger.Debugf("bzz download: invalid path %s/%s: %v", address, path, err)
s.Logger.Error("bzz download: invalid path")
jsonhttp.BadRequest(w, "invalid path address")
if errors.Is(err, manifest.ErrNotFound) {
jsonhttp.NotFound(w, "path address not found")
} else {
jsonhttp.BadRequest(w, "invalid path address")
}
return
}
manifestEntryAddress := me.Reference()
var additionalHeaders http.Header
// copy header from manifest
if me.Header() != nil {
additionalHeaders = me.Header().Clone()
} else {
additionalHeaders = http.Header{}
}
// include filename
if me.Name() != "" {
additionalHeaders.Set("Content-Disposition", fmt.Sprintf("inline; filename=\"%s\"", me.Name()))
}
// read file entry
buf = bytes.NewBuffer(nil)
_, err = file.JoinReadAll(ctx, j, manifestEntryAddress, buf, toDecrypt)
......@@ -151,6 +126,29 @@ func (s *server) bzzDownloadHandler(w http.ResponseWriter, r *http.Request) {
return
}
// read file metadata
buf = bytes.NewBuffer(nil)
_, err = file.JoinReadAll(ctx, j, fe.Metadata(), buf, toDecrypt)
if err != nil {
s.Logger.Debugf("bzz download: read file metadata %s: %v", address, err)
s.Logger.Errorf("bzz download: read file metadata %s", address)
jsonhttp.NotFound(w, nil)
return
}
fileMetadata := &entry.Metadata{}
err = json.Unmarshal(buf.Bytes(), fileMetadata)
if err != nil {
s.Logger.Debugf("bzz download: unmarshal metadata %s: %v", address, err)
s.Logger.Errorf("bzz download: unmarshal metadata %s", address)
jsonhttp.InternalServerError(w, "error unmarshaling metadata")
return
}
additionalHeaders := http.Header{
"Content-Disposition": {fmt.Sprintf("inline; filename=\"%s\"", fileMetadata.Filename)},
"Content-Type": {fileMetadata.MimeType},
}
fileEntryAddress := fe.Reference()
s.downloadHandler(w, r, fileEntryAddress, additionalHeaders)
......
pkg/api/bzz_test.go
View file @ 370a0e97
......@@ -15,14 +15,13 @@ import (
"strings"
"testing"
"github.com/ethersphere/bee/pkg/api"
"github.com/ethersphere/bee/pkg/collection/entry"
"github.com/ethersphere/bee/pkg/file"
"github.com/ethersphere/bee/pkg/file/splitter"
"github.com/ethersphere/bee/pkg/jsonhttp"
"github.com/ethersphere/bee/pkg/jsonhttp/jsonhttptest"
"github.com/ethersphere/bee/pkg/logging"
"github.com/ethersphere/bee/pkg/manifest/jsonmanifest"
"github.com/ethersphere/bee/pkg/manifest"
"github.com/ethersphere/bee/pkg/storage"
smock "github.com/ethersphere/bee/pkg/storage/mock"
"github.com/ethersphere/bee/pkg/swarm"
......@@ -69,6 +68,7 @@ func TestBzz(t *testing.T) {
}
fileMetadata := entry.NewMetadata(fileName)
fileMetadata.MimeType = "text/html; charset=utf-8"
fileMetadataBytes, err := json.Marshal(fileMetadata)
if err != nil {
t.Fatal(err)
......@@ -91,24 +91,26 @@ func TestBzz(t *testing.T) {
// save manifest
jsonManifest := jsonmanifest.NewManifest()
m, err := manifest.NewDefaultManifest(false, storer)
if err != nil {
t.Fatal(err)
}
e := jsonmanifest.NewEntry(fileReference, fileName, http.Header{"Content-Type": {"text/html", "charset=utf-8"}})
jsonManifest.Add(filePath, e)
e := manifest.NewEntry(fileReference)
manifestFileBytes, err := jsonManifest.MarshalBinary()
err = m.Add(filePath, e)
if err != nil {
t.Fatal(err)
}
fr, err := file.SplitWriteAll(context.Background(), sp, bytes.NewReader(manifestFileBytes), int64(len(manifestFileBytes)), false)
manifestBytesReference, err := m.Store(context.Background(), storage.ModePutUpload)
if err != nil {
t.Fatal(err)
}
m := entry.NewMetadata(fileName)
m.MimeType = api.ManifestContentType
metadataBytes, err := json.Marshal(m)
metadata := entry.NewMetadata(manifestBytesReference.String())
metadata.MimeType = m.Type()
metadataBytes, err := json.Marshal(metadata)
if err != nil {
t.Fatal(err)
}
......@@ -118,8 +120,8 @@ func TestBzz(t *testing.T) {
t.Fatal(err)
}
// now join both references (mr,fr) to create an entry and store it.
newEntry := entry.New(fr, mr)
// now join both references (fr,mr) to create an entry and store it.
newEntry := entry.New(manifestBytesReference, mr)
manifestFileEntryBytes, err := newEntry.MarshalBinary()
if err != nil {
t.Fatal(err)
......@@ -152,10 +154,10 @@ func TestBzz(t *testing.T) {
// check on invalid path
jsonhttptest.Request(t, client, http.MethodGet, bzzDownloadResource(manifestFileReference.String(), missingFilePath), http.StatusBadRequest,
jsonhttptest.Request(t, client, http.MethodGet, bzzDownloadResource(manifestFileReference.String(), missingFilePath), http.StatusNotFound,
jsonhttptest.WithExpectedJSONResponse(jsonhttp.StatusResponse{
Message: "invalid path address",
Code: http.StatusBadRequest,
Message: "path address not found",
Code: http.StatusNotFound,
}),
)
})
......
pkg/api/dirs.go
View file @ 370a0e97
......@@ -22,7 +22,7 @@ import (
"github.com/ethersphere/bee/pkg/file/splitter"
"github.com/ethersphere/bee/pkg/jsonhttp"
"github.com/ethersphere/bee/pkg/logging"
"github.com/ethersphere/bee/pkg/manifest/jsonmanifest"
"github.com/ethersphere/bee/pkg/manifest"
"github.com/ethersphere/bee/pkg/sctx"
"github.com/ethersphere/bee/pkg/storage"
"github.com/ethersphere/bee/pkg/swarm"
......@@ -94,12 +94,20 @@ func validateRequest(r *http.Request) (context.Context, error) {
// storeDir stores all files recursively contained in the directory given as a tar
// it returns the hash for the uploaded manifest corresponding to the uploaded dir
func storeDir(ctx context.Context, reader io.ReadCloser, s storage.Storer, mode storage.ModePut, logger logging.Logger) (swarm.Address, error) {
dirManifest := jsonmanifest.NewManifest()
v := ctx.Value(toEncryptContextKey{})
toEncrypt, _ := v.(bool) // default is false
dirManifest, err := manifest.NewDefaultManifest(toEncrypt, s)
if err != nil {
return swarm.ZeroAddress, err
}
// set up HTTP body reader
tarReader := tar.NewReader(reader)
defer reader.Close()
filesAdded := 0
// iterate through the files in the supplied tar
for {
fileHeader, err := tarReader.Next()
......@@ -133,42 +141,54 @@ func storeDir(ctx context.Context, reader io.ReadCloser, s storage.Storer, mode
}
logger.Tracef("uploaded dir file %v with reference %v", filePath, fileReference)
// create manifest entry for uploaded file
headers := http.Header{}
headers.Set("Content-Type", contentType)
fileEntry := jsonmanifest.NewEntry(fileReference, fileName, headers)
// add file entry to dir manifest
err = dirManifest.Add(filePath, manifest.NewEntry(fileReference))
if err != nil {
return swarm.ZeroAddress, fmt.Errorf("add to manifest: %w", err)
}
filesAdded++
}
// add entry to dir manifest
dirManifest.Add(filePath, fileEntry)
// check if files were uploaded through the manifest
if filesAdded == 0 {
return swarm.ZeroAddress, fmt.Errorf("no files in tar")
}
// check if files were uploaded by querying manifest length
if dirManifest.Length() == 0 {
return swarm.ZeroAddress, fmt.Errorf("no files added from tar")
// save manifest
manifestBytesReference, err := dirManifest.Store(ctx, mode)
if err != nil {
return swarm.ZeroAddress, fmt.Errorf("store manifest: %w", err)
}
// upload manifest
// first, serialize into byte array
b, err := dirManifest.MarshalBinary()
// store the manifest metadata and get its reference
m := entry.NewMetadata(manifestBytesReference.String())
m.MimeType = dirManifest.Type()
metadataBytes, err := json.Marshal(m)
if err != nil {
return swarm.ZeroAddress, fmt.Errorf("manifest serialize: %w", err)
return swarm.ZeroAddress, fmt.Errorf("metadata marshal: %w", err)
}
// set up reader for manifest file upload
r := bytes.NewReader(b)
sp := splitter.NewSimpleSplitter(s, mode)
mr, err := file.SplitWriteAll(ctx, sp, bytes.NewReader(metadataBytes), int64(len(metadataBytes)), toEncrypt)
if err != nil {
return swarm.ZeroAddress, fmt.Errorf("split metadata: %w", err)
}
// then, upload manifest
manifestFileInfo := &fileUploadInfo{
size: r.Size(),
contentType: ManifestContentType,
reader: r,
// now join both references (fr, mr) to create an entry and store it
e := entry.New(manifestBytesReference, mr)
fileEntryBytes, err := e.MarshalBinary()
if err != nil {
return swarm.ZeroAddress, fmt.Errorf("entry marshal: %w", err)
}
manifestReference, err := storeFile(ctx, manifestFileInfo, s, mode)
sp = splitter.NewSimpleSplitter(s, mode)
manifestFileReference, err := file.SplitWriteAll(ctx, sp, bytes.NewReader(fileEntryBytes), int64(len(fileEntryBytes)), toEncrypt)
if err != nil {
return swarm.ZeroAddress, fmt.Errorf("store manifest: %w", err)
return swarm.ZeroAddress, fmt.Errorf("split entry: %w", err)
}
return manifestReference, nil
return manifestFileReference, nil
}
// storeFile uploads the given file and returns its reference
......
pkg/api/dirs_test.go
View file @ 370a0e97
......@@ -7,16 +7,21 @@ package api_test
import (
"archive/tar"
"bytes"
"context"
"encoding/json"
"io/ioutil"
"net/http"
"path"
"testing"
"github.com/ethersphere/bee/pkg/api"
"github.com/ethersphere/bee/pkg/collection/entry"
"github.com/ethersphere/bee/pkg/file"
"github.com/ethersphere/bee/pkg/file/joiner"
"github.com/ethersphere/bee/pkg/jsonhttp"
"github.com/ethersphere/bee/pkg/jsonhttp/jsonhttptest"
"github.com/ethersphere/bee/pkg/logging"
"github.com/ethersphere/bee/pkg/manifest/jsonmanifest"
"github.com/ethersphere/bee/pkg/manifest"
"github.com/ethersphere/bee/pkg/storage/mock"
"github.com/ethersphere/bee/pkg/swarm"
"github.com/ethersphere/bee/pkg/tags"
......@@ -26,8 +31,9 @@ func TestDirs(t *testing.T) {
var (
dirUploadResource = "/dirs"
fileDownloadResource = func(addr string) string { return "/files/" + addr }
storer = mock.NewStorer()
client = newTestServer(t, testServerOptions{
Storer: mock.NewStorer(),
Storer: storer,
Tags: tags.NewTags(),
Logger: logging.New(ioutil.Discard, 5),
})
......@@ -82,7 +88,7 @@ func TestDirs(t *testing.T) {
}{
{
name: "non-nested files without extension",
expectedHash: "3609d0521d34469ecbffc1d2401ce7a34c7c54bb63e8d23933ef0073015aa9e7",
expectedHash: "685f591d0482a57e172aecb7f58babd7eb50fcb8411f875cae5c7b96fa44ff82",
files: []f{
{
data: []byte("first file data"),
......@@ -106,7 +112,7 @@ func TestDirs(t *testing.T) {
},
{
name: "nested files with extension",
expectedHash: "983869d469f0eab1f1bb6c2daeac1fdf476968246410b3001e59e9f2e0236da0",
expectedHash: "9e4e53c1764f2379408ffe019c097cbfcb8a0ba93587b52126a4e3e9d5b8556f",
files: []f{
{
data: []byte("robots text"),
......@@ -142,31 +148,76 @@ func TestDirs(t *testing.T) {
// tar all the test case files
tarReader := tarFiles(t, tc.files)
var respBytes []byte
// verify directory tar upload response
jsonhttptest.Request(t, client, http.MethodPost, dirUploadResource, http.StatusOK,
jsonhttptest.WithRequestBody(tarReader),
jsonhttptest.WithExpectedJSONResponse(api.FileUploadResponse{
Reference: swarm.MustParseHexAddress(tc.expectedHash),
}),
jsonhttptest.WithRequestHeader("Content-Type", api.ContentTypeTar),
jsonhttptest.WithPutResponseBody(&respBytes),
)
// create expected manifest
expectedManifest := jsonmanifest.NewManifest()
for _, file := range tc.files {
e := jsonmanifest.NewEntry(file.reference, file.name, file.header)
expectedManifest.Add(path.Join(file.dir, file.name), e)
read := bytes.NewReader(respBytes)
// get the reference as everytime it will change because of random encryption key
var resp api.FileUploadResponse
err := json.NewDecoder(read).Decode(&resp)
if err != nil {
t.Fatal(err)
}
if tc.expectedHash != resp.Reference.String() {
t.Fatalf("expected file reference to match %s, got %x", tc.expectedHash, resp.Reference)
}
b, err := expectedManifest.MarshalBinary()
// read manifest metadata
j := joiner.NewSimpleJoiner(storer)
buf := bytes.NewBuffer(nil)
_, err = file.JoinReadAll(context.Background(), j, resp.Reference, buf, false)
if err != nil {
t.Fatal(err)
}
e := &entry.Entry{}
err = e.UnmarshalBinary(buf.Bytes())
if err != nil {
t.Fatal(err)
}
// verify directory upload manifest through files api
jsonhttptest.Request(t, client, http.MethodGet, fileDownloadResource(tc.expectedHash), http.StatusOK,
jsonhttptest.WithExpectedResponse(b),
// verify manifest content
verifyManifest, err := manifest.NewManifestReference(
context.Background(),
manifest.DefaultManifestType,
e.Reference(),
false,
storer,
)
if err != nil {
t.Fatal(err)
}
// check if each file can be located and read
for _, file := range tc.files {
filePath := path.Join(file.dir, file.name)
entry, err := verifyManifest.Lookup(filePath)
if err != nil {
t.Fatal(err)
}
fileReference := entry.Reference()
if !bytes.Equal(file.reference.Bytes(), fileReference.Bytes()) {
t.Fatalf("expected file reference to match %x, got %x", file.reference, fileReference)
}
jsonhttptest.Request(t, client, http.MethodGet, fileDownloadResource(fileReference.String()), http.StatusOK,
jsonhttptest.WithExpectedResponse(file.data),
jsonhttptest.WithRequestHeader("Content-Type", file.header.Get("Content-Type")),
)
}
})
}
}
......
pkg/api/file_test.go
View file @ 370a0e97
......@@ -277,7 +277,7 @@ func TestRangeRequests(t *testing.T) {
uploadEndpoint: "/dirs",
downloadEndpoint: "/bzz",
filepath: "/ipsum/lorem.txt",
reference: "c1e596eebc9b39fea8f790b6ede4a294bf336e17b0cb7cd64ec54edc5c4ec0e2",
reference: "d2b1ab6fb26c1570712ca33efb30f8cbbaa994d5b85e1cf6f782bcae430eabaf",
reader: tarFiles(t, []f{
{
data: data,
......
pkg/api/tag_test.go
View file @ 370a0e97
......@@ -460,7 +460,7 @@ func TestTags(t *testing.T) {
name: "binary-file",
}})
expectedHash := swarm.MustParseHexAddress("9e5acfbfeb7e074d4c79f5f9922e8a25990dad267d0ea7becaaad07b47fb2a87")
expectedHash := swarm.MustParseHexAddress("ebcfbfac0e9a4fa4483491875f9486107a799e54cd832d0aacc59b1125b4b71f")
expectedResponse := api.FileUploadResponse{Reference: expectedHash}
respHeaders := jsonhttptest.Request(t, client, http.MethodPost, dirResource, http.StatusOK,
......
pkg/manifest/jsonmanifest/entry.go deleted 100644 → 0
View file @ 6177d16c
// Copyright 2020 The Swarm Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package jsonmanifest
import (
"net/http"
"github.com/ethersphere/bee/pkg/manifest"
"github.com/ethersphere/bee/pkg/swarm"
)
// verify jsonEntry implements manifest.Entry.
var _ manifest.Entry = (*jsonEntry)(nil)
// jsonEntry is a JSON representation of a single manifest entry for a jsonManifest.
type jsonEntry struct {
R swarm.Address `json:"reference"`
N string `json:"name"`
H http.Header `json:"header"`
}
// NewEntry creates a new jsonEntry struct and returns it.
func NewEntry(reference swarm.Address, name string, headers http.Header) manifest.Entry {
return &jsonEntry{
R: reference,
N: name,
H: headers,
}
}
// Reference returns the address of the file in the entry.
func (me *jsonEntry) Reference() swarm.Address {
return me.R
}
// Name returns the name of the file in the entry.
func (me *jsonEntry) Name() string {
return me.N
}
// Header returns the HTTP header for the file in the manifest entry.
func (me *jsonEntry) Header() http.Header {
return me.H
}
pkg/manifest/jsonmanifest/jsonmanifest_test.go deleted 100644 → 0
View file @ 6177d16c
// Copyright 2020 The Swarm Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package jsonmanifest_test
import (
"net/http"
"path/filepath"
"reflect"
"testing"
"github.com/ethersphere/bee/pkg/manifest"
"github.com/ethersphere/bee/pkg/manifest/jsonmanifest"
"github.com/ethersphere/bee/pkg/swarm"
"github.com/ethersphere/bee/pkg/swarm/test"
)
var testCases = []testCase{
{
name: "empty-manifest",
entries: nil,
},
{
name: "one-entry",
entries: []e{
{
reference: test.RandomAddress(),
path: "entry-1",
header: http.Header{},
},
},
},
{
name: "two-entries",
entries: []e{
{
reference: test.RandomAddress(),
path: "entry-1.txt",
header: http.Header{"Content-Type": {"text/plain; charset=utf-8"}},
},
{
reference: test.RandomAddress(),
path: "entry-2.png",
header: http.Header{"Content-Type": {"image/png"}},
},
},
},
{
name: "nested-entries",
entries: []e{
{
reference: test.RandomAddress(),
path: "text/robots.txt",
header: http.Header{"Content-Type": {"text/plain; charset=utf-8"}},
},
{
reference: test.RandomAddress(),
path: "img/1.png",
header: http.Header{"Content-Type": {"image/png"}},
},
{
reference: test.RandomAddress(),
path: "img/2.jpg",
header: http.Header{"Content-Type": {"image/jpg"}},
},
{
reference: test.RandomAddress(),
path: "readme.md",
header: http.Header{"Content-Type": {"text/markdown; charset=UTF-8"}},
},
},
},
}
// TestEntries tests the Add, Length and Entry functions.
// This test will add multiple entries to a manifest, checking that they are correctly retrieved each time,
// and that the length of the manifest is as expected.
// It will verify that the manifest length remains unchanged when replacing entries or removing inexistent ones.
// Finally, it will remove all entries in the manifest, checking that they are correctly not found each time,
// and that the length of the manifest is as expected.
func TestEntries(t *testing.T) {
tc := testCases[len(testCases)-1] // get non-trivial test case
m := jsonmanifest.NewManifest()
checkLength(t, m, 0)
// add entries
for i, e := range tc.entries {
_, name := filepath.Split(e.path)
entry := jsonmanifest.NewEntry(e.reference, name, e.header)
m.Add(e.path, entry)
checkLength(t, m, i+1)
checkEntry(t, m, entry, e.path)
}
manifestLen := m.Length()
// replace entry
lastEntry := tc.entries[len(tc.entries)-1]
_, name := filepath.Split(lastEntry.path)
newEntry := jsonmanifest.NewEntry(test.RandomAddress(), name, lastEntry.header)
m.Add(lastEntry.path, newEntry)
checkLength(t, m, manifestLen) // length should not have changed
checkEntry(t, m, newEntry, lastEntry.path)
// remove entries
m.Remove("invalid/path.ext") // try removing inexistent entry
checkLength(t, m, manifestLen) // length should not have changed
for i, e := range tc.entries {
m.Remove(e.path)
entry, err := m.Entry(e.path)
if entry != nil || err != manifest.ErrNotFound {
t.Fatalf("expected path %v not to be present in the manifest, but it was found", e.path)
}
checkLength(t, m, manifestLen-i-1)
}
}
// checkLength verifies that the given manifest length and integer match.
func checkLength(t *testing.T, m manifest.Interface, length int) {
if m.Length() != length {
t.Fatalf("expected length to be %d, but is %d instead", length, m.Length())
}
}
// checkEntry verifies that an entry is equal to the one retrieved from the given manifest and path.
func checkEntry(t *testing.T, m manifest.Interface, entry manifest.Entry, path string) {
re, err := m.Entry(path)
if err != nil {
t.Fatal(err)
}
if !reflect.DeepEqual(entry, re) {
t.Fatalf("original and retrieved entry are not equal: %v, %v", entry, re)
}
}
// TestEntryModification verifies that manifest entries are not modifiable from outside of the manifest.
// This test will add a single entry to a manifest, retrieve it, and modify it.
// After, it will re-retrieve that same entry from the manifest, and check that it has not changed.
func TestEntryModification(t *testing.T) {
m := jsonmanifest.NewManifest()
e := jsonmanifest.NewEntry(test.RandomAddress(), "single_entry.png", http.Header{"Content-Type": {"image/png"}})
m.Add("", e)
re, err := m.Entry("")
if err != nil {
t.Fatal(err)
}
re.Header().Add("Content-Type", "text/plain; charset=utf-8") // modify retrieved entry
rre, err := m.Entry("") // re-retrieve entry
if err != nil {
t.Fatal(err)
}
if reflect.DeepEqual(rre, re) {
t.Fatalf("manifest entry %v was unexpectedly modified externally", rre)
}
}
// TestMarshal verifies that created manifests are successfully marshalled and unmarshalled.
// This function wil add all test case entries to a manifest and marshal it.
// After, it will unmarshal the result, and verify that it is equal to the original manifest.
func TestMarshal(t *testing.T) {
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
m := jsonmanifest.NewManifest()
for _, e := range tc.entries {
_, name := filepath.Split(e.path)
entry := jsonmanifest.NewEntry(e.reference, name, e.header)
m.Add(e.path, entry)
}
b, err := m.MarshalBinary()
if err != nil {
t.Fatal(err)
}
um := jsonmanifest.NewManifest()
if err := um.UnmarshalBinary(b); err != nil {
t.Fatal(err)
}
if !reflect.DeepEqual(m, um) {
t.Fatalf("marshalled and unmarshalled manifests are not equal: %v, %v", m, um)
}
})
}
}
// struct for manifest test cases
type testCase struct {
name string
entries []e // entries to add to manifest
}
// struct for manifest entries for test cases
type e struct {
reference swarm.Address
path string
header http.Header
}
pkg/manifest/jsonmanifest/manifest.go deleted 100644 → 0
View file @ 6177d16c
// Copyright 2020 The Swarm Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package jsonmanifest
import (
"encoding/json"
"sync"
"github.com/ethersphere/bee/pkg/manifest"
)
// verify jsonManifest implements manifest.Interface.
var _ manifest.Interface = (*jsonManifest)(nil)
// jsonManifest is a JSON representation of a manifest.
// It stores manifest entries in a map based on string keys.
type jsonManifest struct {
entriesMu sync.RWMutex // mutex for accessing the entries map
Entries map[string]*jsonEntry `json:"entries,omitempty"`
}
// NewManifest creates a new jsonManifest struct and returns a pointer to it.
func NewManifest() manifest.Interface {
return &jsonManifest{
Entries: make(map[string]*jsonEntry),
}
}
// Add adds a manifest entry to the specified path.
func (m *jsonManifest) Add(path string, entry manifest.Entry) {
m.entriesMu.Lock()
defer m.entriesMu.Unlock()
m.Entries[path] = &jsonEntry{
R: entry.Reference(),
N: entry.Name(),
H: entry.Header(),
}
}
// Remove removes a manifest entry on the specified path.
func (m *jsonManifest) Remove(path string) {
m.entriesMu.Lock()
defer m.entriesMu.Unlock()
delete(m.Entries, path)
}
// Entry returns a manifest entry if one is found in the specified path.
func (m *jsonManifest) Entry(path string) (manifest.Entry, error) {
m.entriesMu.RLock()
defer m.entriesMu.RUnlock()
entry, ok := m.Entries[path]
if !ok {
return nil, manifest.ErrNotFound
}
// return a copy to prevent external modification
return NewEntry(entry.Reference(), entry.Name(), entry.Header().Clone()), nil
}
// Length returns an implementation-specific count of elements in the manifest.
// For jsonManifest, this means the number of all the existing entries.
func (m *jsonManifest) Length() int {
m.entriesMu.RLock()
defer m.entriesMu.RUnlock()
return len(m.Entries)
}
// MarshalBinary implements encoding.BinaryMarshaler.
func (m *jsonManifest) MarshalBinary() ([]byte, error) {
m.entriesMu.RLock()
defer m.entriesMu.RUnlock()
return json.Marshal(m)
}
// UnmarshalBinary implements encoding.BinaryUnmarshaler.
func (m *jsonManifest) UnmarshalBinary(b []byte) error {
m.entriesMu.Lock()
defer m.entriesMu.Unlock()
return json.Unmarshal(b, m)
}
pkg/manifest/manifest.go
View file @ 370a0e97
......@@ -5,36 +5,93 @@
package manifest
import (
"encoding"
"context"
"errors"
"net/http"
"github.com/ethersphere/bee/pkg/storage"
"github.com/ethersphere/bee/pkg/swarm"
)
// ErrNotFound is returned when an Entry is not found in the manifest.
var ErrNotFound = errors.New("manifest: not found")
const DefaultManifestType = ManifestSimpleContentType
var (
// ErrNotFound is returned when an Entry is not found in the manifest.
ErrNotFound = errors.New("manifest: not found")
// ErrInvalidManifestType is returned when an unknown manifest type
// is provided to the function.
ErrInvalidManifestType = errors.New("manifest: invalid type")
)
// Interface for operations with manifest.
type Interface interface {
// Type returns manifest implementation type information
Type() string
// Add a manifest entry to the specified path.
Add(string, Entry)
Add(string, Entry) error
// Remove a manifest entry on the specified path.
Remove(string)
// Entry returns a manifest entry if one is found in the specified path.
Entry(string) (Entry, error)
// Length returns an implementation-specific count of elements in the manifest.
Length() int
encoding.BinaryMarshaler
encoding.BinaryUnmarshaler
Remove(string) error
// Lookup returns a manifest entry if one is found in the specified path.
Lookup(string) (Entry, error)
// Store stores the manifest, returning the resulting address.
Store(context.Context, storage.ModePut) (swarm.Address, error)
}
// Entry represents a single manifest entry.
type Entry interface {
// Reference returns the address of the file in the entry.
// Reference returns the address of the file.
Reference() swarm.Address
// Name returns the name of the file in the entry.
Name() string
// Header returns the HTTP header for the file in the manifest entry.
Header() http.Header
}
// NewDefaultManifest creates a new manifest with default type.
func NewDefaultManifest(
encrypted bool,
storer storage.Storer,
) (Interface, error) {
return NewManifest(DefaultManifestType, encrypted, storer)
}
// NewManifest creates a new manifest.
func NewManifest(
manifestType string,
encrypted bool,
storer storage.Storer,
) (Interface, error) {
switch manifestType {
case ManifestSimpleContentType:
return NewSimpleManifest(encrypted, storer)
default:
return nil, ErrInvalidManifestType
}
}
// NewManifestReference loads existing manifest.
func NewManifestReference(
ctx context.Context,
manifestType string,
reference swarm.Address,
encrypted bool,
storer storage.Storer,
) (Interface, error) {
switch manifestType {
case ManifestSimpleContentType:
return NewSimpleManifestReference(ctx, reference, encrypted, storer)
default:
return nil, ErrInvalidManifestType
}
}
type manifestEntry struct {
reference swarm.Address
}
// NewEntry creates a new manifest entry.
func NewEntry(reference swarm.Address) Entry {
return &manifestEntry{
reference: reference,
}
}
func (e *manifestEntry) Reference() swarm.Address {
return e.reference
}
pkg/manifest/simple.go 0 → 100644
View file @ 370a0e97
// Copyright 2020 The Swarm Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package manifest
import (
"bytes"
"context"
"errors"
"fmt"
"github.com/ethersphere/bee/pkg/file"
"github.com/ethersphere/bee/pkg/file/joiner"
"github.com/ethersphere/bee/pkg/file/splitter"
"github.com/ethersphere/bee/pkg/storage"
"github.com/ethersphere/bee/pkg/swarm"
"github.com/ethersphere/manifest/simple"
)
const (
// ManifestSimpleContentType represents content type used for noting that
// specific file should be processed as 'simple' manifest
ManifestSimpleContentType = "application/bzz-manifest-simple+json"
)
type simpleManifest struct {
manifest simple.Manifest
encrypted bool
storer storage.Storer
}
// NewSimpleManifest creates a new simple manifest.
func NewSimpleManifest(
encrypted bool,
storer storage.Storer,
) (Interface, error) {
return &simpleManifest{
manifest: simple.NewManifest(),
encrypted: encrypted,
storer: storer,
}, nil
}
// NewSimpleManifestReference loads existing simple manifest.
func NewSimpleManifestReference(
ctx context.Context,
reference swarm.Address,
encrypted bool,
storer storage.Storer,
) (Interface, error) {
m := &simpleManifest{
manifest: simple.NewManifest(),
encrypted: encrypted,
storer: storer,
}
err := m.load(ctx, reference)
return m, err
}
func (m *simpleManifest) Type() string {
return ManifestSimpleContentType
}
func (m *simpleManifest) Add(path string, entry Entry) error {
e := entry.Reference().String()
return m.manifest.Add(path, e)
}
func (m *simpleManifest) Remove(path string) error {
err := m.manifest.Remove(path)
if err != nil {
if errors.Is(err, simple.ErrNotFound) {
return ErrNotFound
}
return err
}
return nil
}
func (m *simpleManifest) Lookup(path string) (Entry, error) {
n, err := m.manifest.Lookup(path)
if err != nil {
return nil, ErrNotFound
}
address, err := swarm.ParseHexAddress(n.Reference())
if err != nil {
return nil, fmt.Errorf("parse swarm address: %w", err)
}
entry := NewEntry(address)
return entry, nil
}
func (m *simpleManifest) Store(ctx context.Context, mode storage.ModePut) (swarm.Address, error) {
data, err := m.manifest.MarshalBinary()
if err != nil {
return swarm.ZeroAddress, fmt.Errorf("manifest marshal error: %w", err)
}
sp := splitter.NewSimpleSplitter(m.storer, mode)
address, err := file.SplitWriteAll(ctx, sp, bytes.NewReader(data), int64(len(data)), m.encrypted)
if err != nil {
return swarm.ZeroAddress, fmt.Errorf("manifest save error: %w", err)
}
return address, nil
}
func (m *simpleManifest) load(ctx context.Context, reference swarm.Address) error {
j := joiner.NewSimpleJoiner(m.storer)
buf := bytes.NewBuffer(nil)
_, err := file.JoinReadAll(ctx, j, reference, buf, m.encrypted)
if err != nil {
return fmt.Errorf("manifest load error: %w", err)
}
err = m.manifest.UnmarshalBinary(buf.Bytes())
if err != nil {
return fmt.Errorf("manifest unmarshal error: %w", err)
}
return nil
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment