Skip to content
Projects
Groups
Snippets
Help
Loading...
Help
Support
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
M
mybee
Project
Project
Details
Activity
Releases
Cycle Analytics
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Charts
Issues
0
Issues
0
List
Boards
Labels
Milestones
Merge Requests
0
Merge Requests
0
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Charts
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Charts
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
vicotor
mybee
Commits
62e65995
Unverified
Commit
62e65995
authored
Aug 27, 2020
by
Nemanja Zbiljić
Committed by
GitHub
Aug 27, 2020
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
Add support for 'mantaray' manifest, and make it default (#621)
parent
f8258c89
Changes
7
Hide whitespace changes
Inline
Side-by-side
Showing
7 changed files
with
240 additions
and
22 deletions
+240
-22
go.mod
go.mod
+2
-2
go.sum
go.sum
+4
-2
dirs_test.go
pkg/api/dirs_test.go
+8
-9
file_test.go
pkg/api/file_test.go
+27
-5
tag_test.go
pkg/api/tag_test.go
+18
-3
manifest.go
pkg/manifest/manifest.go
+5
-1
mantaray.go
pkg/manifest/mantaray.go
+176
-0
No files found.
go.mod
View file @
62e65995
...
...
@@ -8,7 +8,7 @@ require (
github.com/coreos/go-semver v0.3.0
github.com/davidlazar/go-crypto v0.0.0-20200604182044-b73af7476f6c // indirect
github.com/ethersphere/bmt v0.1.2
github.com/ethersphere/manifest v0.
1
.0
github.com/ethersphere/manifest v0.
2
.0
github.com/gogo/protobuf v1.3.1
github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e // indirect
github.com/gopherjs/gopherjs v0.0.0-20200217142428-fce0ec30dd00 // indirect
...
...
@@ -51,7 +51,7 @@ require (
gitlab.com/nolash/go-mockbytes v0.0.7
go.opencensus.io v0.22.4 // indirect
go.uber.org/zap v1.15.0 // indirect
golang.org/x/crypto v0.0.0-20200
622213623-75b288015ac9
golang.org/x/crypto v0.0.0-20200
728195943-123391ffb6de
golang.org/x/lint v0.0.0-20200302205851-738671d3881b // indirect
golang.org/x/mod v0.3.0 // indirect
golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208
...
...
go.sum
View file @
62e65995
...
...
@@ -108,8 +108,8 @@ github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1m
github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
github.com/ethersphere/bmt v0.1.2 h1:FEuvQY9xuK+rDp3VwDVyde8T396Matv/u9PdtKa2r9Q=
github.com/ethersphere/bmt v0.1.2/go.mod h1:fqRBDmYwn3lX2MH4lkImXQgFWeNP8ikLkS/hgi/HRws=
github.com/ethersphere/manifest v0.
1.0 h1:uVlFzAZk5SqyzjHzDgF3rNuDA4CdbJQ8fVHS4pN0iHY
=
github.com/ethersphere/manifest v0.
1.0/go.mod h1:eV7hOz2c5R1ol+SpBYdS5EUG6ubh11CQe6lFAn5q+q4
=
github.com/ethersphere/manifest v0.
2.0 h1:HD2ufiIaw/5Vgrl4XyeGduDJ5tn50wIhqMQoWdT2GME
=
github.com/ethersphere/manifest v0.
2.0/go.mod h1:ygAx0KLhXYmKqsjUab95RCbXf8UcO7yMDjyfP0lY76Y
=
github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc=
github.com/francoispqt/gojay v1.2.13 h1:d2m3sFjloqoIUQU3TsHBgj6qg/BVGlTBeHDUmyJnXKk=
...
...
@@ -887,6 +887,8 @@ golang.org/x/crypto v0.0.0-20200423211502-4bdfaf469ed5/go.mod h1:LzIPMQfyMNhhGPh
golang.org/x/crypto v0.0.0-20200602180216-279210d13fed/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 h1:psW17arqaxU48Z5kZ0CQnkZWQJsqcURM6tKiBApRjXI=
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/crypto v0.0.0-20200728195943-123391ffb6de h1:ikNHVSjEfnvz6sxdSPCaPt572qowuyMDMJLLm3Db3ig=
golang.org/x/crypto v0.0.0-20200728195943-123391ffb6de/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
...
...
pkg/api/dirs_test.go
View file @
62e65995
...
...
@@ -82,13 +82,11 @@ func TestDirs(t *testing.T) {
// valid tars
for
_
,
tc
:=
range
[]
struct
{
name
string
expectedHash
string
files
[]
f
// files in dir for test case
name
string
files
[]
f
// files in dir for test case
}{
{
name
:
"non-nested files without extension"
,
expectedHash
:
"685f591d0482a57e172aecb7f58babd7eb50fcb8411f875cae5c7b96fa44ff82"
,
name
:
"non-nested files without extension"
,
files
:
[]
f
{
{
data
:
[]
byte
(
"first file data"
),
...
...
@@ -111,8 +109,7 @@ func TestDirs(t *testing.T) {
},
},
{
name
:
"nested files with extension"
,
expectedHash
:
"9e4e53c1764f2379408ffe019c097cbfcb8a0ba93587b52126a4e3e9d5b8556f"
,
name
:
"nested files with extension"
,
files
:
[]
f
{
{
data
:
[]
byte
(
"robots text"
),
...
...
@@ -166,8 +163,10 @@ func TestDirs(t *testing.T) {
t
.
Fatal
(
err
)
}
if
tc
.
expectedHash
!=
resp
.
Reference
.
String
()
{
t
.
Fatalf
(
"expected file reference to match %s, got %x"
,
tc
.
expectedHash
,
resp
.
Reference
)
// NOTE: reference will be different each time, due to manifest randomness
if
resp
.
Reference
.
String
()
==
""
{
t
.
Fatalf
(
"expected file reference, did not got any"
)
}
// read manifest metadata
...
...
pkg/api/file_test.go
View file @
62e65995
...
...
@@ -6,6 +6,7 @@ package api_test
import
(
"bytes"
"encoding/json"
"fmt"
"io"
"io/ioutil"
...
...
@@ -277,7 +278,7 @@ func TestRangeRequests(t *testing.T) {
uploadEndpoint
:
"/dirs"
,
downloadEndpoint
:
"/bzz"
,
filepath
:
"/ipsum/lorem.txt"
,
reference
:
"
d2b1ab6fb26c1570712ca33efb30f8cbbaa994d5b85e1cf6f782bcae430eabaf
"
,
reference
:
""
,
reader
:
tarFiles
(
t
,
[]
f
{
{
data
:
data
,
...
...
@@ -339,20 +340,41 @@ func TestRangeRequests(t *testing.T) {
Logger
:
logging
.
New
(
ioutil
.
Discard
,
5
),
})
uploadReference
:=
upload
.
reference
var
respBytes
[]
byte
jsonhttptest
.
Request
(
t
,
client
,
http
.
MethodPost
,
upload
.
uploadEndpoint
,
http
.
StatusOK
,
jsonhttptest
.
WithRequestBody
(
upload
.
reader
),
jsonhttptest
.
WithExpectedJSONResponse
(
api
.
FileUploadResponse
{
Reference
:
swarm
.
MustParseHexAddress
(
upload
.
reference
),
}),
jsonhttptest
.
WithRequestHeader
(
"Content-Type"
,
upload
.
contentType
),
jsonhttptest
.
WithPutResponseBody
(
&
respBytes
),
)
if
uploadReference
==
""
{
// NOTE: reference will be different each time, due to manifest randomness
read
:=
bytes
.
NewReader
(
respBytes
)
// get the reference as everytime it will change because of random encryption key
var
resp
api
.
FileUploadResponse
err
:=
json
.
NewDecoder
(
read
)
.
Decode
(
&
resp
)
if
err
!=
nil
{
t
.
Fatal
(
err
)
}
if
resp
.
Reference
.
String
()
==
""
{
t
.
Fatalf
(
"expected file reference, did not got any"
)
}
uploadReference
=
resp
.
Reference
.
String
()
}
for
_
,
tc
:=
range
ranges
{
t
.
Run
(
tc
.
name
,
func
(
t
*
testing
.
T
)
{
rangeHeader
,
want
:=
createRangeHeader
(
data
,
tc
.
ranges
)
var
body
[]
byte
respHeaders
:=
jsonhttptest
.
Request
(
t
,
client
,
http
.
MethodGet
,
upload
.
downloadEndpoint
+
"/"
+
upload
.
r
eference
+
upload
.
filepath
,
http
.
StatusPartialContent
,
respHeaders
:=
jsonhttptest
.
Request
(
t
,
client
,
http
.
MethodGet
,
upload
.
downloadEndpoint
+
"/"
+
upload
R
eference
+
upload
.
filepath
,
http
.
StatusPartialContent
,
jsonhttptest
.
WithRequestHeader
(
"Range"
,
rangeHeader
),
jsonhttptest
.
WithPutResponseBody
(
&
body
),
)
...
...
pkg/api/tag_test.go
View file @
62e65995
...
...
@@ -6,6 +6,7 @@ package api_test
import
(
"bytes"
"encoding/json"
"fmt"
"net/http"
"strconv"
...
...
@@ -460,15 +461,29 @@ func TestTags(t *testing.T) {
name
:
"binary-file"
,
}})
expectedHash
:=
swarm
.
MustParseHexAddress
(
"ebcfbfac0e9a4fa4483491875f9486107a799e54cd832d0aacc59b1125b4b71f"
)
expectedResponse
:=
api
.
FileUploadResponse
{
Reference
:
expectedHash
}
var
respBytes
[]
byte
respHeaders
:=
jsonhttptest
.
Request
(
t
,
client
,
http
.
MethodPost
,
dirResource
,
http
.
StatusOK
,
jsonhttptest
.
WithRequestBody
(
tarReader
),
jsonhttptest
.
WithExpectedJSONResponse
(
expectedResponse
),
jsonhttptest
.
WithRequestHeader
(
"Content-Type"
,
api
.
ContentTypeTar
),
jsonhttptest
.
WithPutResponseBody
(
&
respBytes
),
)
read
:=
bytes
.
NewReader
(
respBytes
)
// get the reference as everytime it will change because of random encryption key
var
resp
api
.
FileUploadResponse
err
:=
json
.
NewDecoder
(
read
)
.
Decode
(
&
resp
)
if
err
!=
nil
{
t
.
Fatal
(
err
)
}
// NOTE: reference will be different each time, due to manifest randomness
if
resp
.
Reference
.
String
()
==
""
{
t
.
Fatalf
(
"expected file reference, did not got any"
)
}
tagId
,
err
:=
strconv
.
Atoi
(
respHeaders
.
Get
(
api
.
SwarmTagUidHeader
))
if
err
!=
nil
{
t
.
Fatal
(
err
)
...
...
pkg/manifest/manifest.go
View file @
62e65995
...
...
@@ -12,7 +12,7 @@ import (
"github.com/ethersphere/bee/pkg/swarm"
)
const
DefaultManifestType
=
Manifest
Simple
ContentType
const
DefaultManifestType
=
Manifest
Mantaray
ContentType
var
(
// ErrNotFound is returned when an Entry is not found in the manifest.
...
...
@@ -60,6 +60,8 @@ func NewManifest(
switch
manifestType
{
case
ManifestSimpleContentType
:
return
NewSimpleManifest
(
encrypted
,
storer
)
case
ManifestMantarayContentType
:
return
NewMantarayManifest
(
encrypted
,
storer
)
default
:
return
nil
,
ErrInvalidManifestType
}
...
...
@@ -76,6 +78,8 @@ func NewManifestReference(
switch
manifestType
{
case
ManifestSimpleContentType
:
return
NewSimpleManifestReference
(
ctx
,
reference
,
encrypted
,
storer
)
case
ManifestMantarayContentType
:
return
NewMantarayManifestReference
(
ctx
,
reference
,
encrypted
,
storer
)
default
:
return
nil
,
ErrInvalidManifestType
}
...
...
pkg/manifest/mantaray.go
0 → 100644
View file @
62e65995
// Copyright 2020 The Swarm Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package
manifest
import
(
"bytes"
"context"
"errors"
"fmt"
"github.com/ethersphere/bee/pkg/file"
"github.com/ethersphere/bee/pkg/file/joiner"
"github.com/ethersphere/bee/pkg/file/splitter"
"github.com/ethersphere/bee/pkg/storage"
"github.com/ethersphere/bee/pkg/swarm"
"github.com/ethersphere/manifest/mantaray"
)
const
(
// ManifestMantarayContentType represents content type used for noting that
// specific file should be processed as mantaray manifest.
ManifestMantarayContentType
=
"application/bzz-manifest-mantaray+octet-stream"
)
type
mantarayManifest
struct
{
trie
*
mantaray
.
Node
encrypted
bool
storer
storage
.
Storer
loader
mantaray
.
LoadSaver
}
// NewMantarayManifest creates a new mantaray-based manifest.
func
NewMantarayManifest
(
encrypted
bool
,
storer
storage
.
Storer
,
)
(
Interface
,
error
)
{
return
&
mantarayManifest
{
trie
:
mantaray
.
New
(),
encrypted
:
encrypted
,
storer
:
storer
,
},
nil
}
// NewMantarayManifestReference loads existing mantaray-based manifest.
func
NewMantarayManifestReference
(
ctx
context
.
Context
,
reference
swarm
.
Address
,
encrypted
bool
,
storer
storage
.
Storer
,
)
(
Interface
,
error
)
{
return
&
mantarayManifest
{
trie
:
mantaray
.
NewNodeRef
(
reference
.
Bytes
()),
encrypted
:
encrypted
,
storer
:
storer
,
loader
:
newMantarayLoader
(
ctx
,
encrypted
,
storer
),
},
nil
}
func
(
m
*
mantarayManifest
)
Type
()
string
{
return
ManifestMantarayContentType
}
func
(
m
*
mantarayManifest
)
Add
(
path
string
,
entry
Entry
)
error
{
p
:=
[]
byte
(
path
)
e
:=
entry
.
Reference
()
.
Bytes
()
return
m
.
trie
.
Add
(
p
,
e
,
m
.
loader
)
}
func
(
m
*
mantarayManifest
)
Remove
(
path
string
)
error
{
p
:=
[]
byte
(
path
)
err
:=
m
.
trie
.
Remove
(
p
,
m
.
loader
)
if
err
!=
nil
{
if
errors
.
Is
(
err
,
mantaray
.
ErrNotFound
)
{
return
ErrNotFound
}
return
err
}
return
nil
}
func
(
m
*
mantarayManifest
)
Lookup
(
path
string
)
(
Entry
,
error
)
{
p
:=
[]
byte
(
path
)
ref
,
err
:=
m
.
trie
.
Lookup
(
p
,
m
.
loader
)
if
err
!=
nil
{
return
nil
,
ErrNotFound
}
address
:=
swarm
.
NewAddress
(
ref
)
entry
:=
NewEntry
(
address
)
return
entry
,
nil
}
func
(
m
*
mantarayManifest
)
Store
(
ctx
context
.
Context
,
mode
storage
.
ModePut
)
(
swarm
.
Address
,
error
)
{
saver
:=
newMantaraySaver
(
ctx
,
m
.
encrypted
,
m
.
storer
,
mode
)
err
:=
m
.
trie
.
Save
(
saver
)
if
err
!=
nil
{
return
swarm
.
ZeroAddress
,
fmt
.
Errorf
(
"manifest save error: %w"
,
err
)
}
address
:=
swarm
.
NewAddress
(
m
.
trie
.
Reference
())
return
address
,
nil
}
// mantarayLoadSaver implements required interface 'mantaray.LoadSaver'
type
mantarayLoadSaver
struct
{
ctx
context
.
Context
encrypted
bool
storer
storage
.
Storer
modePut
storage
.
ModePut
}
func
newMantarayLoader
(
ctx
context
.
Context
,
encrypted
bool
,
storer
storage
.
Storer
,
)
*
mantarayLoadSaver
{
return
&
mantarayLoadSaver
{
ctx
:
ctx
,
encrypted
:
encrypted
,
storer
:
storer
,
}
}
func
newMantaraySaver
(
ctx
context
.
Context
,
encrypted
bool
,
storer
storage
.
Storer
,
modePut
storage
.
ModePut
,
)
*
mantarayLoadSaver
{
return
&
mantarayLoadSaver
{
ctx
:
ctx
,
encrypted
:
encrypted
,
storer
:
storer
,
modePut
:
modePut
,
}
}
func
(
ls
*
mantarayLoadSaver
)
Load
(
ref
[]
byte
)
([]
byte
,
error
)
{
ctx
:=
ls
.
ctx
j
:=
joiner
.
NewSimpleJoiner
(
ls
.
storer
)
buf
:=
bytes
.
NewBuffer
(
nil
)
_
,
err
:=
file
.
JoinReadAll
(
ctx
,
j
,
swarm
.
NewAddress
(
ref
),
buf
,
ls
.
encrypted
)
if
err
!=
nil
{
return
nil
,
err
}
return
buf
.
Bytes
(),
nil
}
func
(
ls
*
mantarayLoadSaver
)
Save
(
data
[]
byte
)
([]
byte
,
error
)
{
ctx
:=
ls
.
ctx
sp
:=
splitter
.
NewSimpleSplitter
(
ls
.
storer
,
ls
.
modePut
)
address
,
err
:=
file
.
SplitWriteAll
(
ctx
,
sp
,
bytes
.
NewReader
(
data
),
int64
(
len
(
data
)),
ls
.
encrypted
)
if
err
!=
nil
{
return
swarm
.
ZeroAddress
.
Bytes
(),
err
}
return
address
.
Bytes
(),
nil
}
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment