Commit 62e65995 authored by Nemanja Zbiljić's avatar Nemanja Zbiljić Committed by GitHub

Add support for 'mantaray' manifest, and make it default (#621)

parent f8258c89
......@@ -8,7 +8,7 @@ require (
github.com/coreos/go-semver v0.3.0
github.com/davidlazar/go-crypto v0.0.0-20200604182044-b73af7476f6c // indirect
github.com/ethersphere/bmt v0.1.2
github.com/ethersphere/manifest v0.1.0
github.com/ethersphere/manifest v0.2.0
github.com/gogo/protobuf v1.3.1
github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e // indirect
github.com/gopherjs/gopherjs v0.0.0-20200217142428-fce0ec30dd00 // indirect
......@@ -51,7 +51,7 @@ require (
gitlab.com/nolash/go-mockbytes v0.0.7
go.opencensus.io v0.22.4 // indirect
go.uber.org/zap v1.15.0 // indirect
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9
golang.org/x/crypto v0.0.0-20200728195943-123391ffb6de
golang.org/x/lint v0.0.0-20200302205851-738671d3881b // indirect
golang.org/x/mod v0.3.0 // indirect
golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208
......
......@@ -108,8 +108,8 @@ github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1m
github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
github.com/ethersphere/bmt v0.1.2 h1:FEuvQY9xuK+rDp3VwDVyde8T396Matv/u9PdtKa2r9Q=
github.com/ethersphere/bmt v0.1.2/go.mod h1:fqRBDmYwn3lX2MH4lkImXQgFWeNP8ikLkS/hgi/HRws=
github.com/ethersphere/manifest v0.1.0 h1:uVlFzAZk5SqyzjHzDgF3rNuDA4CdbJQ8fVHS4pN0iHY=
github.com/ethersphere/manifest v0.1.0/go.mod h1:eV7hOz2c5R1ol+SpBYdS5EUG6ubh11CQe6lFAn5q+q4=
github.com/ethersphere/manifest v0.2.0 h1:HD2ufiIaw/5Vgrl4XyeGduDJ5tn50wIhqMQoWdT2GME=
github.com/ethersphere/manifest v0.2.0/go.mod h1:ygAx0KLhXYmKqsjUab95RCbXf8UcO7yMDjyfP0lY76Y=
github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc=
github.com/francoispqt/gojay v1.2.13 h1:d2m3sFjloqoIUQU3TsHBgj6qg/BVGlTBeHDUmyJnXKk=
......@@ -887,6 +887,8 @@ golang.org/x/crypto v0.0.0-20200423211502-4bdfaf469ed5/go.mod h1:LzIPMQfyMNhhGPh
golang.org/x/crypto v0.0.0-20200602180216-279210d13fed/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 h1:psW17arqaxU48Z5kZ0CQnkZWQJsqcURM6tKiBApRjXI=
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/crypto v0.0.0-20200728195943-123391ffb6de h1:ikNHVSjEfnvz6sxdSPCaPt572qowuyMDMJLLm3Db3ig=
golang.org/x/crypto v0.0.0-20200728195943-123391ffb6de/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
......
......@@ -82,13 +82,11 @@ func TestDirs(t *testing.T) {
// valid tars
for _, tc := range []struct {
name string
expectedHash string
files []f // files in dir for test case
name string
files []f // files in dir for test case
}{
{
name: "non-nested files without extension",
expectedHash: "685f591d0482a57e172aecb7f58babd7eb50fcb8411f875cae5c7b96fa44ff82",
name: "non-nested files without extension",
files: []f{
{
data: []byte("first file data"),
......@@ -111,8 +109,7 @@ func TestDirs(t *testing.T) {
},
},
{
name: "nested files with extension",
expectedHash: "9e4e53c1764f2379408ffe019c097cbfcb8a0ba93587b52126a4e3e9d5b8556f",
name: "nested files with extension",
files: []f{
{
data: []byte("robots text"),
......@@ -166,8 +163,10 @@ func TestDirs(t *testing.T) {
t.Fatal(err)
}
if tc.expectedHash != resp.Reference.String() {
t.Fatalf("expected file reference to match %s, got %x", tc.expectedHash, resp.Reference)
// NOTE: reference will be different each time, due to manifest randomness
if resp.Reference.String() == "" {
t.Fatalf("expected file reference, did not got any")
}
// read manifest metadata
......
......@@ -6,6 +6,7 @@ package api_test
import (
"bytes"
"encoding/json"
"fmt"
"io"
"io/ioutil"
......@@ -277,7 +278,7 @@ func TestRangeRequests(t *testing.T) {
uploadEndpoint: "/dirs",
downloadEndpoint: "/bzz",
filepath: "/ipsum/lorem.txt",
reference: "d2b1ab6fb26c1570712ca33efb30f8cbbaa994d5b85e1cf6f782bcae430eabaf",
reference: "",
reader: tarFiles(t, []f{
{
data: data,
......@@ -339,20 +340,41 @@ func TestRangeRequests(t *testing.T) {
Logger: logging.New(ioutil.Discard, 5),
})
uploadReference := upload.reference
var respBytes []byte
jsonhttptest.Request(t, client, http.MethodPost, upload.uploadEndpoint, http.StatusOK,
jsonhttptest.WithRequestBody(upload.reader),
jsonhttptest.WithExpectedJSONResponse(api.FileUploadResponse{
Reference: swarm.MustParseHexAddress(upload.reference),
}),
jsonhttptest.WithRequestHeader("Content-Type", upload.contentType),
jsonhttptest.WithPutResponseBody(&respBytes),
)
if uploadReference == "" {
// NOTE: reference will be different each time, due to manifest randomness
read := bytes.NewReader(respBytes)
// get the reference as everytime it will change because of random encryption key
var resp api.FileUploadResponse
err := json.NewDecoder(read).Decode(&resp)
if err != nil {
t.Fatal(err)
}
if resp.Reference.String() == "" {
t.Fatalf("expected file reference, did not got any")
}
uploadReference = resp.Reference.String()
}
for _, tc := range ranges {
t.Run(tc.name, func(t *testing.T) {
rangeHeader, want := createRangeHeader(data, tc.ranges)
var body []byte
respHeaders := jsonhttptest.Request(t, client, http.MethodGet, upload.downloadEndpoint+"/"+upload.reference+upload.filepath, http.StatusPartialContent,
respHeaders := jsonhttptest.Request(t, client, http.MethodGet, upload.downloadEndpoint+"/"+uploadReference+upload.filepath, http.StatusPartialContent,
jsonhttptest.WithRequestHeader("Range", rangeHeader),
jsonhttptest.WithPutResponseBody(&body),
)
......
......@@ -6,6 +6,7 @@ package api_test
import (
"bytes"
"encoding/json"
"fmt"
"net/http"
"strconv"
......@@ -460,15 +461,29 @@ func TestTags(t *testing.T) {
name: "binary-file",
}})
expectedHash := swarm.MustParseHexAddress("ebcfbfac0e9a4fa4483491875f9486107a799e54cd832d0aacc59b1125b4b71f")
expectedResponse := api.FileUploadResponse{Reference: expectedHash}
var respBytes []byte
respHeaders := jsonhttptest.Request(t, client, http.MethodPost, dirResource, http.StatusOK,
jsonhttptest.WithRequestBody(tarReader),
jsonhttptest.WithExpectedJSONResponse(expectedResponse),
jsonhttptest.WithRequestHeader("Content-Type", api.ContentTypeTar),
jsonhttptest.WithPutResponseBody(&respBytes),
)
read := bytes.NewReader(respBytes)
// get the reference as everytime it will change because of random encryption key
var resp api.FileUploadResponse
err := json.NewDecoder(read).Decode(&resp)
if err != nil {
t.Fatal(err)
}
// NOTE: reference will be different each time, due to manifest randomness
if resp.Reference.String() == "" {
t.Fatalf("expected file reference, did not got any")
}
tagId, err := strconv.Atoi(respHeaders.Get(api.SwarmTagUidHeader))
if err != nil {
t.Fatal(err)
......
......@@ -12,7 +12,7 @@ import (
"github.com/ethersphere/bee/pkg/swarm"
)
const DefaultManifestType = ManifestSimpleContentType
const DefaultManifestType = ManifestMantarayContentType
var (
// ErrNotFound is returned when an Entry is not found in the manifest.
......@@ -60,6 +60,8 @@ func NewManifest(
switch manifestType {
case ManifestSimpleContentType:
return NewSimpleManifest(encrypted, storer)
case ManifestMantarayContentType:
return NewMantarayManifest(encrypted, storer)
default:
return nil, ErrInvalidManifestType
}
......@@ -76,6 +78,8 @@ func NewManifestReference(
switch manifestType {
case ManifestSimpleContentType:
return NewSimpleManifestReference(ctx, reference, encrypted, storer)
case ManifestMantarayContentType:
return NewMantarayManifestReference(ctx, reference, encrypted, storer)
default:
return nil, ErrInvalidManifestType
}
......
// Copyright 2020 The Swarm Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package manifest
import (
"bytes"
"context"
"errors"
"fmt"
"github.com/ethersphere/bee/pkg/file"
"github.com/ethersphere/bee/pkg/file/joiner"
"github.com/ethersphere/bee/pkg/file/splitter"
"github.com/ethersphere/bee/pkg/storage"
"github.com/ethersphere/bee/pkg/swarm"
"github.com/ethersphere/manifest/mantaray"
)
const (
// ManifestMantarayContentType represents content type used for noting that
// specific file should be processed as mantaray manifest.
ManifestMantarayContentType = "application/bzz-manifest-mantaray+octet-stream"
)
type mantarayManifest struct {
trie *mantaray.Node
encrypted bool
storer storage.Storer
loader mantaray.LoadSaver
}
// NewMantarayManifest creates a new mantaray-based manifest.
func NewMantarayManifest(
encrypted bool,
storer storage.Storer,
) (Interface, error) {
return &mantarayManifest{
trie: mantaray.New(),
encrypted: encrypted,
storer: storer,
}, nil
}
// NewMantarayManifestReference loads existing mantaray-based manifest.
func NewMantarayManifestReference(
ctx context.Context,
reference swarm.Address,
encrypted bool,
storer storage.Storer,
) (Interface, error) {
return &mantarayManifest{
trie: mantaray.NewNodeRef(reference.Bytes()),
encrypted: encrypted,
storer: storer,
loader: newMantarayLoader(ctx, encrypted, storer),
}, nil
}
func (m *mantarayManifest) Type() string {
return ManifestMantarayContentType
}
func (m *mantarayManifest) Add(path string, entry Entry) error {
p := []byte(path)
e := entry.Reference().Bytes()
return m.trie.Add(p, e, m.loader)
}
func (m *mantarayManifest) Remove(path string) error {
p := []byte(path)
err := m.trie.Remove(p, m.loader)
if err != nil {
if errors.Is(err, mantaray.ErrNotFound) {
return ErrNotFound
}
return err
}
return nil
}
func (m *mantarayManifest) Lookup(path string) (Entry, error) {
p := []byte(path)
ref, err := m.trie.Lookup(p, m.loader)
if err != nil {
return nil, ErrNotFound
}
address := swarm.NewAddress(ref)
entry := NewEntry(address)
return entry, nil
}
func (m *mantarayManifest) Store(ctx context.Context, mode storage.ModePut) (swarm.Address, error) {
saver := newMantaraySaver(ctx, m.encrypted, m.storer, mode)
err := m.trie.Save(saver)
if err != nil {
return swarm.ZeroAddress, fmt.Errorf("manifest save error: %w", err)
}
address := swarm.NewAddress(m.trie.Reference())
return address, nil
}
// mantarayLoadSaver implements required interface 'mantaray.LoadSaver'
type mantarayLoadSaver struct {
ctx context.Context
encrypted bool
storer storage.Storer
modePut storage.ModePut
}
func newMantarayLoader(
ctx context.Context,
encrypted bool,
storer storage.Storer,
) *mantarayLoadSaver {
return &mantarayLoadSaver{
ctx: ctx,
encrypted: encrypted,
storer: storer,
}
}
func newMantaraySaver(
ctx context.Context,
encrypted bool,
storer storage.Storer,
modePut storage.ModePut,
) *mantarayLoadSaver {
return &mantarayLoadSaver{
ctx: ctx,
encrypted: encrypted,
storer: storer,
modePut: modePut,
}
}
func (ls *mantarayLoadSaver) Load(ref []byte) ([]byte, error) {
ctx := ls.ctx
j := joiner.NewSimpleJoiner(ls.storer)
buf := bytes.NewBuffer(nil)
_, err := file.JoinReadAll(ctx, j, swarm.NewAddress(ref), buf, ls.encrypted)
if err != nil {
return nil, err
}
return buf.Bytes(), nil
}
func (ls *mantarayLoadSaver) Save(data []byte) ([]byte, error) {
ctx := ls.ctx
sp := splitter.NewSimpleSplitter(ls.storer, ls.modePut)
address, err := file.SplitWriteAll(ctx, sp, bytes.NewReader(data), int64(len(data)), ls.encrypted)
if err != nil {
return swarm.ZeroAddress.Bytes(), err
}
return address.Bytes(), nil
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment