Skip to content
Projects
Groups
Snippets
Help
Loading...
Help
Support
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
M
mybee
Project
Project
Details
Activity
Releases
Cycle Analytics
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Charts
Issues
0
Issues
0
List
Boards
Labels
Milestones
Merge Requests
0
Merge Requests
0
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Charts
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Charts
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
vicotor
mybee
Commits
3cdcdc62
Unverified
Commit
3cdcdc62
authored
Jul 27, 2020
by
Zahoor Mohamed
Committed by
GitHub
Jul 27, 2020
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
Fix file upload/download during encryption (#472)
* Fix file upload/download during encryption
parent
2b82b44a
Changes
7
Expand all
Hide whitespace changes
Inline
Side-by-side
Showing
7 changed files
with
102 additions
and
54 deletions
+102
-54
file_test.go
pkg/api/file_test.go
+12
-5
entry.go
pkg/collection/entry/entry.go
+11
-5
encryption_test.go
pkg/encryption/encryption_test.go
+17
-16
joiner.go
pkg/file/joiner/joiner.go
+20
-1
job.go
pkg/file/splitter/internal/job.go
+15
-19
jsonhttptest.go
pkg/jsonhttp/jsonhttptest/jsonhttptest.go
+17
-0
swarm.go
pkg/swarm/swarm.go
+10
-8
No files found.
pkg/api/file_test.go
View file @
3cdcdc62
...
...
@@ -6,6 +6,7 @@ package api_test
import
(
"bytes"
"encoding/json"
"fmt"
"io/ioutil"
"mime"
...
...
@@ -51,15 +52,21 @@ func TestFiles(t *testing.T) {
t
.
Run
(
"encrypt-decrypt"
,
func
(
t
*
testing
.
T
)
{
fileName
:=
"my-pictures.jpeg"
rootHash
:=
"f2e761160deda91c1fbfab065a5abf530b0766b3e102b51fbd626ba37c3bc581"
headers
:=
make
(
http
.
Header
)
headers
.
Add
(
"EncryptHeader"
,
"True"
)
headers
.
Add
(
api
.
EncryptHeader
,
"True"
)
headers
.
Add
(
"Content-Type"
,
"image/jpeg; charset=utf-8"
)
_
=
jsonhttptest
.
ResponseDirectSendHeadersAndReceiveHeaders
(
t
,
client
,
http
.
MethodPost
,
fileUploadResource
+
"?name="
+
fileName
,
bytes
.
NewReader
(
simpleData
),
http
.
StatusOK
,
api
.
FileUploadResponse
{
Reference
:
swarm
.
MustParseHexAddress
(
rootHash
),
},
headers
)
_
,
respBytes
:=
jsonhttptest
.
ResponseDirectSendHeadersAndDontCheckResponse
(
t
,
client
,
http
.
MethodPost
,
fileUploadResource
+
"?name="
+
fileName
,
bytes
.
NewReader
(
simpleData
),
http
.
StatusOK
,
headers
)
read
:=
bytes
.
NewReader
(
respBytes
)
// get the reference as everytime it will change because of random encryption key
var
resp
api
.
FileUploadResponse
err
:=
json
.
NewDecoder
(
read
)
.
Decode
(
&
resp
)
if
err
!=
nil
{
t
.
Fatal
(
err
)
}
rootHash
:=
resp
.
Reference
.
String
()
rcvdHeader
:=
jsonhttptest
.
ResponseDirectCheckBinaryResponse
(
t
,
client
,
http
.
MethodGet
,
fileDownloadResource
(
rootHash
),
nil
,
http
.
StatusOK
,
simpleData
,
nil
)
cd
:=
rcvdHeader
.
Get
(
"Content-Disposition"
)
_
,
params
,
err
:=
mime
.
ParseMediaType
(
cd
)
...
...
pkg/collection/entry/entry.go
View file @
3cdcdc62
...
...
@@ -12,8 +12,9 @@ import (
)
var
(
_
=
collection
.
Entry
(
&
Entry
{})
serializedDataSize
=
swarm
.
SectionSize
*
2
_
=
collection
.
Entry
(
&
Entry
{})
serializedDataSize
=
swarm
.
SectionSize
*
2
encryptedSerializedDataSize
=
swarm
.
EncryptedReferenceSize
*
2
)
// Entry provides addition of metadata to a data reference.
...
...
@@ -51,10 +52,15 @@ func (e *Entry) MarshalBinary() ([]byte, error) {
// UnmarshalBinary implements encoding.BinaryUnmarshaler
func
(
e
*
Entry
)
UnmarshalBinary
(
b
[]
byte
)
error
{
if
len
(
b
)
!=
serializedDataSize
{
var
size
int
if
len
(
b
)
==
serializedDataSize
{
size
=
serializedDataSize
}
else
if
len
(
b
)
==
encryptedSerializedDataSize
{
size
=
encryptedSerializedDataSize
}
else
{
return
errors
.
New
(
"invalid data length"
)
}
e
.
reference
=
swarm
.
NewAddress
(
b
[
:
s
warm
.
SectionSize
])
e
.
metadata
=
swarm
.
NewAddress
(
b
[
s
warm
.
SectionSize
:
])
e
.
reference
=
swarm
.
NewAddress
(
b
[
:
s
ize
/
2
])
e
.
metadata
=
swarm
.
NewAddress
(
b
[
s
ize
/
2
:
])
return
nil
}
pkg/encryption/encryption_test.go
View file @
3cdcdc62
This diff is collapsed.
Click to expand it.
pkg/file/joiner/joiner.go
View file @
3cdcdc62
...
...
@@ -31,6 +31,17 @@ func NewSimpleJoiner(getter storage.Getter) file.Joiner {
}
func
(
s
*
simpleJoiner
)
Size
(
ctx
context
.
Context
,
address
swarm
.
Address
)
(
dataSize
int64
,
err
error
)
{
// Handle size based on whether the root chunk is encrypted or not
toDecrypt
:=
len
(
address
.
Bytes
())
==
swarm
.
EncryptedReferenceSize
var
key
encryption
.
Key
addrBytes
:=
address
.
Bytes
()
if
toDecrypt
{
addrBytes
=
address
.
Bytes
()[
:
swarm
.
HashSize
]
key
=
address
.
Bytes
()[
swarm
.
HashSize
:
swarm
.
HashSize
+
encryption
.
KeyLength
]
}
address
=
swarm
.
NewAddress
(
addrBytes
)
// retrieve the root chunk to read the total data length the be retrieved
rootChunk
,
err
:=
s
.
getter
.
Get
(
ctx
,
storage
.
ModeGetRequest
,
address
)
if
err
!=
nil
{
...
...
@@ -42,7 +53,15 @@ func (s *simpleJoiner) Size(ctx context.Context, address swarm.Address) (dataSiz
return
0
,
fmt
.
Errorf
(
"invalid chunk content of %d bytes"
,
chunkLength
)
}
dataLength
:=
binary
.
LittleEndian
.
Uint64
(
rootChunk
.
Data
())
chunkData
:=
rootChunk
.
Data
()
if
toDecrypt
{
originalData
,
err
:=
internal
.
DecryptChunkData
(
rootChunk
.
Data
(),
key
)
if
err
!=
nil
{
return
0
,
err
}
chunkData
=
originalData
}
dataLength
:=
binary
.
LittleEndian
.
Uint64
(
chunkData
[
:
8
])
return
int64
(
dataLength
),
nil
}
...
...
pkg/file/splitter/internal/job.go
View file @
3cdcdc62
...
...
@@ -136,41 +136,37 @@ func (s *SimpleSplitterJob) sumLevel(lvl int) ([]byte, error) {
s
.
sumCounts
[
lvl
]
++
spanSize
:=
file
.
Spans
[
lvl
]
*
swarm
.
ChunkSize
span
:=
(
s
.
length
-
1
)
%
spanSize
+
1
sizeToSum
:=
s
.
cursors
[
lvl
]
-
s
.
cursors
[
lvl
+
1
]
//perform hashing
s
.
hasher
.
Reset
()
err
:=
s
.
hasher
.
SetSpan
(
span
)
if
err
!=
nil
{
return
nil
,
err
}
var
ref
encryption
.
Key
var
chunkData
[]
byte
data
:=
s
.
buffer
[
s
.
cursors
[
lvl
+
1
]
:
s
.
cursors
[
lvl
+
1
]
+
sizeToSum
]
var
addr
swarm
.
Address
_
,
err
=
s
.
hasher
.
Write
(
data
)
if
err
!=
nil
{
return
nil
,
err
}
ref
=
s
.
hasher
.
Sum
(
nil
)
head
:=
make
([]
byte
,
8
)
binary
.
LittleEndian
.
PutUint64
(
head
,
uint64
(
span
))
tail
:=
s
.
buffer
[
s
.
cursors
[
lvl
+
1
]
:
s
.
cursors
[
lvl
]]
chunkData
=
append
(
head
,
tail
...
)
// assemble chunk and put in store
addr
:=
swarm
.
NewAddress
(
ref
)
c
:=
chunkData
var
encryptionKey
encryption
.
Key
if
s
.
toEncrypt
{
var
err
error
c
,
encryptionKey
,
err
=
s
.
encryptChunkData
(
chunkData
)
if
err
!=
nil
{
return
nil
,
err
}
}
s
.
hasher
.
Reset
()
err
:=
s
.
hasher
.
SetSpanBytes
(
c
[
:
8
])
if
err
!=
nil
{
return
nil
,
err
}
_
,
err
=
s
.
hasher
.
Write
(
c
[
8
:
])
if
err
!=
nil
{
return
nil
,
err
}
ref
:=
s
.
hasher
.
Sum
(
nil
)
addr
=
swarm
.
NewAddress
(
ref
)
ch
:=
swarm
.
NewChunk
(
addr
,
c
)
_
,
err
=
s
.
putter
.
Put
(
s
.
ctx
,
storage
.
ModePutUpload
,
ch
)
if
err
!=
nil
{
...
...
pkg/jsonhttp/jsonhttptest/jsonhttptest.go
View file @
3cdcdc62
...
...
@@ -137,6 +137,23 @@ func ResponseDirectSendHeadersAndReceiveHeaders(t *testing.T, client *http.Clien
return
resp
.
Header
}
// ResponseDirectSendHeadersAndDontCheckResponse sends a request with the given headers and does not check for the returned reference.
// this is useful in tests which does not know the return reference, for ex: when encryption flag is set
func
ResponseDirectSendHeadersAndDontCheckResponse
(
t
*
testing
.
T
,
client
*
http
.
Client
,
method
,
url
string
,
body
io
.
Reader
,
responseCode
int
,
headers
http
.
Header
)
(
http
.
Header
,
[]
byte
)
{
t
.
Helper
()
resp
:=
request
(
t
,
client
,
method
,
url
,
body
,
responseCode
,
headers
)
defer
resp
.
Body
.
Close
()
got
,
err
:=
ioutil
.
ReadAll
(
resp
.
Body
)
if
err
!=
nil
{
t
.
Fatal
(
err
)
}
got
=
bytes
.
TrimSpace
(
got
)
return
resp
.
Header
,
got
}
func
ResponseUnmarshal
(
t
*
testing
.
T
,
client
*
http
.
Client
,
method
,
url
string
,
body
io
.
Reader
,
responseCode
int
,
response
interface
{})
{
t
.
Helper
()
...
...
pkg/swarm/swarm.go
View file @
3cdcdc62
...
...
@@ -11,18 +11,20 @@ import (
"encoding/json"
"fmt"
"github.com/ethersphere/bee/pkg/encryption"
"golang.org/x/crypto/sha3"
)
const
(
SpanSize
=
8
SectionSize
=
32
Branches
=
128
ChunkSize
=
SectionSize
*
Branches
HashSize
=
32
MaxPO
uint8
=
15
MaxBins
=
MaxPO
+
1
ChunkWithSpanSize
=
ChunkSize
+
SpanSize
SpanSize
=
8
SectionSize
=
32
Branches
=
128
ChunkSize
=
SectionSize
*
Branches
HashSize
=
32
EncryptedReferenceSize
=
HashSize
+
encryption
.
KeyLength
MaxPO
uint8
=
15
MaxBins
=
MaxPO
+
1
ChunkWithSpanSize
=
ChunkSize
+
SpanSize
)
var
(
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment