Skip to content
Projects
Groups
Snippets
Help
Loading...
Help
Support
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
M
mybee
Project
Project
Details
Activity
Releases
Cycle Analytics
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Charts
Issues
0
Issues
0
List
Boards
Labels
Milestones
Merge Requests
0
Merge Requests
0
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Charts
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Charts
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
vicotor
mybee
Commits
a96c4c23
Unverified
Commit
a96c4c23
authored
Jan 06, 2021
by
Nemanja Zbiljić
Committed by
GitHub
Jan 06, 2021
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
Random manifest obfuscation key only when encrypting (#1065)
parent
96734961
Changes
11
Hide whitespace changes
Inline
Side-by-side
Showing
11 changed files
with
70 additions
and
90 deletions
+70
-90
go.mod
go.mod
+1
-1
go.sum
go.sum
+2
-2
bzz_test.go
pkg/api/bzz_test.go
+1
-1
dirs.go
pkg/api/dirs.go
+5
-4
dirs_test.go
pkg/api/dirs_test.go
+40
-9
file_test.go
pkg/api/file_test.go
+1
-21
pin_bzz_test.go
pkg/api/pin_bzz_test.go
+0
-11
tag_test.go
pkg/api/tag_test.go
+5
-15
manifest.go
pkg/manifest/manifest.go
+7
-3
mantaray.go
pkg/manifest/mantaray.go
+7
-14
traversal_test.go
pkg/traversal/traversal_test.go
+1
-9
No files found.
go.mod
View file @
a96c4c23
...
...
@@ -10,7 +10,7 @@ require (
github.com/ethereum/go-ethereum v1.9.20
github.com/ethersphere/bmt v0.1.4
github.com/ethersphere/langos v1.0.0
github.com/ethersphere/manifest v0.3.
5
github.com/ethersphere/manifest v0.3.
6
github.com/ethersphere/sw3-bindings/v2 v2.1.0
github.com/gogo/protobuf v1.3.1
github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e // indirect
...
...
go.sum
View file @
a96c4c23
...
...
@@ -167,8 +167,8 @@ github.com/ethersphere/bmt v0.1.4 h1:+rkWYNtMgDx6bkNqGdWu+U9DgGI1rRZplpSW3YhBr1Q
github.com/ethersphere/bmt v0.1.4/go.mod h1:Yd8ft1U69WDuHevZc/rwPxUv1rzPSMpMnS6xbU53aY8=
github.com/ethersphere/langos v1.0.0 h1:NBtNKzXTTRSue95uOlzPN4py7Aofs0xWPzyj4AI1Vcc=
github.com/ethersphere/langos v1.0.0/go.mod h1:dlcN2j4O8sQ+BlCaxeBu43bgr4RQ+inJ+pHwLeZg5Tw=
github.com/ethersphere/manifest v0.3.
5 h1:/UMN4X4eKyTCARS9dv2HqqdFCJI2Emu09tivYsp5FZM
=
github.com/ethersphere/manifest v0.3.
5
/go.mod h1:frSxQFT67hQvmTN5CBtgVuqHzGQpg0V0oIIm/B3Am+U=
github.com/ethersphere/manifest v0.3.
6 h1:38WgYoXAQyC2lrSTArj+HM62AecX8JfUn1oVr1q+CVg
=
github.com/ethersphere/manifest v0.3.
6
/go.mod h1:frSxQFT67hQvmTN5CBtgVuqHzGQpg0V0oIIm/B3Am+U=
github.com/ethersphere/sw3-bindings/v2 v2.1.0 h1:QefDtzU94UelICMPXWr7m52E2oj6r018Yc0XLoCWOxw=
github.com/ethersphere/sw3-bindings/v2 v2.1.0/go.mod h1:ozMVBZZlAirS/FcUpFwzV60v8gC0nVbA/5ZXtCX3xCc=
github.com/fatih/color v1.3.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
...
...
pkg/api/bzz_test.go
View file @
a96c4c23
...
...
@@ -99,7 +99,7 @@ func TestBzz(t *testing.T) {
}
// save manifest
m
,
err
:=
manifest
.
NewDefaultManifest
(
loadsave
.
New
(
storer
,
storage
.
ModePutRequest
,
false
))
m
,
err
:=
manifest
.
NewDefaultManifest
(
loadsave
.
New
(
storer
,
storage
.
ModePutRequest
,
false
)
,
false
)
if
err
!=
nil
{
t
.
Fatal
(
err
)
}
...
...
pkg/api/dirs.go
View file @
a96c4c23
...
...
@@ -62,8 +62,9 @@ func (s *server) dirUploadHandler(w http.ResponseWriter, r *http.Request) {
// Add the tag to the context
ctx
:=
sctx
.
SetTag
(
r
.
Context
(),
tag
)
p
:=
requestPipelineFn
(
s
.
Storer
,
r
)
l
:=
loadsave
.
New
(
s
.
Storer
,
requestModePut
(
r
),
requestEncrypt
(
r
))
reference
,
err
:=
storeDir
(
ctx
,
r
.
Body
,
s
.
Logger
,
p
,
l
,
r
.
Header
.
Get
(
SwarmIndexDocumentHeader
),
r
.
Header
.
Get
(
SwarmErrorDocumentHeader
))
encrypt
:=
requestEncrypt
(
r
)
l
:=
loadsave
.
New
(
s
.
Storer
,
requestModePut
(
r
),
encrypt
)
reference
,
err
:=
storeDir
(
ctx
,
encrypt
,
r
.
Body
,
s
.
Logger
,
p
,
l
,
r
.
Header
.
Get
(
SwarmIndexDocumentHeader
),
r
.
Header
.
Get
(
SwarmErrorDocumentHeader
))
if
err
!=
nil
{
logger
.
Debugf
(
"dir upload: store dir err: %v"
,
err
)
logger
.
Errorf
(
"dir upload: store dir"
)
...
...
@@ -103,10 +104,10 @@ func validateRequest(r *http.Request) error {
// storeDir stores all files recursively contained in the directory given as a tar
// it returns the hash for the uploaded manifest corresponding to the uploaded dir
func
storeDir
(
ctx
context
.
Context
,
reader
io
.
ReadCloser
,
log
logging
.
Logger
,
p
pipelineFunc
,
ls
file
.
LoadSaver
,
indexFilename
string
,
errorFilename
string
)
(
swarm
.
Address
,
error
)
{
func
storeDir
(
ctx
context
.
Context
,
encrypt
bool
,
reader
io
.
ReadCloser
,
log
logging
.
Logger
,
p
pipelineFunc
,
ls
file
.
LoadSaver
,
indexFilename
string
,
errorFilename
string
)
(
swarm
.
Address
,
error
)
{
logger
:=
tracing
.
NewLoggerWithTraceID
(
ctx
,
log
)
dirManifest
,
err
:=
manifest
.
NewDefaultManifest
(
ls
)
dirManifest
,
err
:=
manifest
.
NewDefaultManifest
(
ls
,
encrypt
)
if
err
!=
nil
{
return
swarm
.
ZeroAddress
,
err
}
...
...
pkg/api/dirs_test.go
View file @
a96c4c23
...
...
@@ -92,6 +92,8 @@ func TestDirs(t *testing.T) {
// valid tars
for
_
,
tc
:=
range
[]
struct
{
name
string
expectedReference
swarm
.
Address
encrypt
bool
wantIndexFilename
string
wantErrorFilename
string
indexFilenameOption
jsonhttptest
.
Option
...
...
@@ -99,7 +101,8 @@ func TestDirs(t *testing.T) {
files
[]
f
// files in dir for test case
}{
{
name
:
"non-nested files without extension"
,
name
:
"non-nested files without extension"
,
expectedReference
:
swarm
.
MustParseHexAddress
(
"126140bb0a33d62c4efb0523db2c26be849fcf458504618de785e2a219bad374"
),
files
:
[]
f
{
{
data
:
[]
byte
(
"first file data"
),
...
...
@@ -122,7 +125,8 @@ func TestDirs(t *testing.T) {
},
},
{
name
:
"nested files with extension"
,
name
:
"nested files with extension"
,
expectedReference
:
swarm
.
MustParseHexAddress
(
"cad4b3847bd59532d9e73623d67c52e0c8d4e017d308bbaecb54f2866a91769d"
),
files
:
[]
f
{
{
data
:
[]
byte
(
"robots text"
),
...
...
@@ -154,7 +158,8 @@ func TestDirs(t *testing.T) {
},
},
{
name
:
"no index filename"
,
name
:
"no index filename"
,
expectedReference
:
swarm
.
MustParseHexAddress
(
"a85aaea6a34a5c7127a3546196f2111f866fe369c6d6562ed5d3313a99388c03"
),
files
:
[]
f
{
{
data
:
[]
byte
(
"<h1>Swarm"
),
...
...
@@ -169,6 +174,7 @@ func TestDirs(t *testing.T) {
},
{
name
:
"explicit index filename"
,
expectedReference
:
swarm
.
MustParseHexAddress
(
"7d41402220f8e397ddf74d0cf4ac2055e753102bde0d622c45b03cea2b28b023"
),
wantIndexFilename
:
"index.html"
,
indexFilenameOption
:
jsonhttptest
.
WithRequestHeader
(
api
.
SwarmIndexDocumentHeader
,
"index.html"
),
files
:
[]
f
{
...
...
@@ -185,6 +191,7 @@ func TestDirs(t *testing.T) {
},
{
name
:
"nested index filename"
,
expectedReference
:
swarm
.
MustParseHexAddress
(
"45249cf9caad842b31b29b831a1ff12aa2b711e7c282fa7a5f8c0fb544143421"
),
wantIndexFilename
:
"index.html"
,
indexFilenameOption
:
jsonhttptest
.
WithRequestHeader
(
api
.
SwarmIndexDocumentHeader
,
"index.html"
),
files
:
[]
f
{
...
...
@@ -201,6 +208,7 @@ func TestDirs(t *testing.T) {
},
{
name
:
"explicit index and error filename"
,
expectedReference
:
swarm
.
MustParseHexAddress
(
"2046a4f758e2c0579ab923206a13fb041cec0925a6396f4f772c7ce859b8ca42"
),
wantIndexFilename
:
"index.html"
,
wantErrorFilename
:
"error.html"
,
indexFilenameOption
:
jsonhttptest
.
WithRequestHeader
(
api
.
SwarmIndexDocumentHeader
,
"index.html"
),
...
...
@@ -227,7 +235,8 @@ func TestDirs(t *testing.T) {
},
},
{
name
:
"invalid archive paths"
,
name
:
"invalid archive paths"
,
expectedReference
:
swarm
.
MustParseHexAddress
(
"6e6adb1ce936990cf1b7ecf8f01a8e3e8f939375b9bddb3d666151e0bdc08d4e"
),
files
:
[]
f
{
{
data
:
[]
byte
(
"<h1>Swarm"
),
...
...
@@ -253,6 +262,18 @@ Disallow: /`),
},
},
},
{
name
:
"encrypted"
,
encrypt
:
true
,
files
:
[]
f
{
{
data
:
[]
byte
(
"<h1>Swarm"
),
name
:
"index.html"
,
dir
:
""
,
filePath
:
"./index.html"
,
},
},
},
}
{
t
.
Run
(
tc
.
name
,
func
(
t
*
testing
.
T
)
{
// tar all the test case files
...
...
@@ -271,25 +292,33 @@ Disallow: /`),
if
tc
.
errorFilenameOption
!=
nil
{
options
=
append
(
options
,
tc
.
errorFilenameOption
)
}
if
tc
.
encrypt
{
options
=
append
(
options
,
jsonhttptest
.
WithRequestHeader
(
api
.
SwarmEncryptHeader
,
"true"
))
}
// verify directory tar upload response
jsonhttptest
.
Request
(
t
,
client
,
http
.
MethodPost
,
dirUploadResource
,
http
.
StatusOK
,
options
...
)
read
:=
bytes
.
NewReader
(
respBytes
)
// get the reference
as everytime it will change because of random encryption key
// get the reference
var
resp
api
.
FileUploadResponse
err
:=
json
.
NewDecoder
(
read
)
.
Decode
(
&
resp
)
if
err
!=
nil
{
t
.
Fatal
(
err
)
}
// NOTE: reference will be different each time, due to manifest randomness
if
resp
.
Reference
.
String
()
==
""
{
t
.
Fatalf
(
"expected file reference, did not got any"
)
}
// NOTE: reference will be different each time when encryption is enabled
if
!
tc
.
encrypt
{
if
!
resp
.
Reference
.
Equal
(
tc
.
expectedReference
)
{
t
.
Fatalf
(
"expected root reference to match %s, got %s"
,
tc
.
expectedReference
,
resp
.
Reference
)
}
}
// read manifest metadata
j
,
_
,
err
:=
joiner
.
New
(
context
.
Background
(),
storer
,
resp
.
Reference
)
if
err
!=
nil
{
...
...
@@ -327,8 +356,10 @@ Disallow: /`),
fileReference
:=
entry
.
Reference
()
if
!
bytes
.
Equal
(
file
.
reference
.
Bytes
(),
fileReference
.
Bytes
())
{
t
.
Fatalf
(
"expected file reference to match %s, got %s"
,
file
.
reference
,
fileReference
)
if
!
tc
.
encrypt
{
if
!
bytes
.
Equal
(
file
.
reference
.
Bytes
(),
fileReference
.
Bytes
())
{
t
.
Fatalf
(
"expected file reference to match %s, got %s"
,
file
.
reference
,
fileReference
)
}
}
jsonhttptest
.
Request
(
t
,
client
,
http
.
MethodGet
,
fileDownloadResource
(
fileReference
.
String
()),
http
.
StatusOK
,
...
...
pkg/api/file_test.go
View file @
a96c4c23
...
...
@@ -6,7 +6,6 @@ package api_test
import
(
"bytes"
"encoding/json"
"fmt"
"io"
"io/ioutil"
...
...
@@ -281,7 +280,7 @@ func TestRangeRequests(t *testing.T) {
uploadEndpoint
:
"/dirs"
,
downloadEndpoint
:
"/bzz"
,
filepath
:
"/ipsum/lorem.txt"
,
reference
:
""
,
reference
:
"
96c68b99304b0868189e5c1d6c10be1984d93e88aab0384907f6b8814f60150b
"
,
reader
:
tarFiles
(
t
,
[]
f
{
{
data
:
data
,
...
...
@@ -355,25 +354,6 @@ func TestRangeRequests(t *testing.T) {
jsonhttptest
.
WithPutResponseBody
(
&
respBytes
),
)
if
uploadReference
==
""
{
// NOTE: reference will be different each time, due to manifest randomness
read
:=
bytes
.
NewReader
(
respBytes
)
// get the reference as everytime it will change because of random encryption key
var
resp
api
.
FileUploadResponse
err
:=
json
.
NewDecoder
(
read
)
.
Decode
(
&
resp
)
if
err
!=
nil
{
t
.
Fatal
(
err
)
}
if
resp
.
Reference
.
String
()
==
""
{
t
.
Fatalf
(
"expected file reference, did not got any"
)
}
uploadReference
=
resp
.
Reference
.
String
()
}
for
_
,
tc
:=
range
ranges
{
t
.
Run
(
tc
.
name
,
func
(
t
*
testing
.
T
)
{
rangeHeader
,
want
:=
createRangeHeader
(
data
,
tc
.
ranges
)
...
...
pkg/api/pin_bzz_test.go
View file @
a96c4c23
...
...
@@ -20,7 +20,6 @@ import (
"github.com/ethersphere/bee/pkg/swarm"
"github.com/ethersphere/bee/pkg/tags"
"github.com/ethersphere/bee/pkg/traversal"
"github.com/ethersphere/manifest/mantaray"
)
func
TestPinBzzHandler
(
t
*
testing
.
T
)
{
...
...
@@ -41,15 +40,6 @@ func TestPinBzzHandler(t *testing.T) {
})
)
var
(
obfuscationKey
=
make
([]
byte
,
32
)
obfuscationKeyFn
=
func
(
p
[]
byte
)
(
n
int
,
err
error
)
{
n
=
copy
(
p
,
obfuscationKey
)
return
}
)
mantaray
.
SetObfuscationKeyFn
(
obfuscationKeyFn
)
t
.
Run
(
"pin-bzz-1"
,
func
(
t
*
testing
.
T
)
{
files
:=
[]
f
{
{
...
...
@@ -89,7 +79,6 @@ func TestPinBzzHandler(t *testing.T) {
read
:=
bytes
.
NewReader
(
respBytes
)
// get the reference as everytime it will change because of random encryption key
var
resp
api
.
ListPinnedChunksResponse
err
:=
json
.
NewDecoder
(
read
)
.
Decode
(
&
resp
)
if
err
!=
nil
{
...
...
pkg/api/tag_test.go
View file @
a96c4c23
...
...
@@ -6,7 +6,6 @@ package api_test
import
(
"bytes"
"encoding/json"
"fmt"
"io/ioutil"
"net/http"
...
...
@@ -306,32 +305,23 @@ func TestTags(t *testing.T) {
t
.
Run
(
"dir tags"
,
func
(
t
*
testing
.
T
)
{
// upload a dir without supplying tag
tarReader
:=
tarFiles
(
t
,
[]
f
{{
data
:
[]
byte
(
"some data"
),
data
:
[]
byte
(
"some d
ir d
ata"
),
name
:
"binary-file"
,
}})
var
respBytes
[]
byte
expectedHash
:=
swarm
.
MustParseHexAddress
(
"3dc643abeb3db60a4dfb72008b577dd9a573abaa74c6afe37a75c63ceea829f6"
)
expectedResponse
:=
api
.
FileUploadResponse
{
Reference
:
expectedHash
}
respHeaders
:=
jsonhttptest
.
Request
(
t
,
client
,
http
.
MethodPost
,
dirResource
,
http
.
StatusOK
,
jsonhttptest
.
WithRequestBody
(
tarReader
),
jsonhttptest
.
WithExpectedJSONResponse
(
expectedResponse
),
jsonhttptest
.
WithRequestHeader
(
"Content-Type"
,
api
.
ContentTypeTar
),
jsonhttptest
.
WithPutResponseBody
(
&
respBytes
),
)
read
:=
bytes
.
NewReader
(
respBytes
)
// get the reference as everytime it will change because of random encryption key
var
resp
api
.
FileUploadResponse
err
:=
json
.
NewDecoder
(
read
)
.
Decode
(
&
resp
)
if
err
!=
nil
{
t
.
Fatal
(
err
)
}
tagId
,
err
:=
strconv
.
Atoi
(
respHeaders
.
Get
(
api
.
SwarmTagUidHeader
))
if
err
!=
nil
{
t
.
Fatal
(
err
)
}
tagValueTest
(
t
,
uint32
(
tagId
),
7
,
7
,
1
,
0
,
0
,
7
,
resp
.
Reference
,
client
)
tagValueTest
(
t
,
uint32
(
tagId
),
7
,
7
,
0
,
0
,
0
,
7
,
expectedHash
,
client
)
})
t
.
Run
(
"bytes tags"
,
func
(
t
*
testing
.
T
)
{
...
...
pkg/manifest/manifest.go
View file @
a96c4c23
...
...
@@ -59,20 +59,24 @@ type Entry interface {
}
// NewDefaultManifest creates a new manifest with default type.
func
NewDefaultManifest
(
ls
file
.
LoadSaver
)
(
Interface
,
error
)
{
return
NewManifest
(
DefaultManifestType
,
ls
)
func
NewDefaultManifest
(
ls
file
.
LoadSaver
,
encrypted
bool
,
)
(
Interface
,
error
)
{
return
NewManifest
(
DefaultManifestType
,
ls
,
encrypted
)
}
// NewManifest creates a new manifest.
func
NewManifest
(
manifestType
string
,
ls
file
.
LoadSaver
,
encrypted
bool
,
)
(
Interface
,
error
)
{
switch
manifestType
{
case
ManifestSimpleContentType
:
return
NewSimpleManifest
(
ls
)
case
ManifestMantarayContentType
:
return
NewMantarayManifest
(
ls
)
return
NewMantarayManifest
(
ls
,
encrypted
)
default
:
return
nil
,
ErrInvalidManifestType
}
...
...
pkg/manifest/mantaray.go
View file @
a96c4c23
...
...
@@ -27,26 +27,19 @@ type mantarayManifest struct {
}
// NewMantarayManifest creates a new mantaray-based manifest.
func
NewMantarayManifest
(
ls
file
.
LoadSaver
)
(
Interface
,
error
)
{
return
&
mantarayManifest
{
trie
:
mantaray
.
New
(),
ls
:
ls
,
},
nil
}
// NewMantarayManifestWithObfuscationKeyFn creates a new mantaray-based manifest
// with configured obfuscation key
//
// NOTE: This should only be used in tests.
func
NewMantarayManifestWithObfuscationKeyFn
(
func
NewMantarayManifest
(
ls
file
.
LoadSaver
,
obfuscationKeyFn
func
([]
byte
)
(
int
,
error
)
,
encrypted
bool
,
)
(
Interface
,
error
)
{
mm
:=
&
mantarayManifest
{
trie
:
mantaray
.
New
(),
ls
:
ls
,
}
mantaray
.
SetObfuscationKeyFn
(
obfuscationKeyFn
)
// use empty obfuscation key if not encrypting
if
!
encrypted
{
// NOTE: it will be copied to all trie nodes
mm
.
trie
.
SetObfuscationKey
(
mantaray
.
ZeroObfuscationKey
)
}
return
mm
,
nil
}
...
...
pkg/traversal/traversal_test.go
View file @
a96c4c23
...
...
@@ -265,14 +265,6 @@ func TestTraversalManifest(t *testing.T) {
return
traversalService
.
TraverseManifestAddresses
}
var
(
obfuscationKey
=
make
([]
byte
,
32
)
obfuscationKeyFn
=
func
(
p
[]
byte
)
(
n
int
,
err
error
)
{
n
=
copy
(
p
,
obfuscationKey
)
return
}
)
testCases
:=
[]
struct
{
manifestType
string
files
[]
file
...
...
@@ -470,7 +462,7 @@ func TestTraversalManifest(t *testing.T) {
t
.
Fatal
(
err
)
}
case
manifest
.
ManifestMantarayContentType
:
dirManifest
,
err
=
manifest
.
NewMantarayManifest
WithObfuscationKeyFn
(
ls
,
obfuscationKeyFn
)
dirManifest
,
err
=
manifest
.
NewMantarayManifest
(
ls
,
false
)
if
err
!=
nil
{
t
.
Fatal
(
err
)
}
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment