Skip to content
Projects
Groups
Snippets
Help
Loading...
Help
Support
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
M
mybee
Project
Project
Details
Activity
Releases
Cycle Analytics
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Charts
Issues
0
Issues
0
List
Boards
Labels
Milestones
Merge Requests
0
Merge Requests
0
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Charts
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Charts
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
vicotor
mybee
Commits
53885e61
Unverified
Commit
53885e61
authored
Jun 16, 2020
by
Janoš Guljaš
Committed by
GitHub
Jun 16, 2020
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
support binary upload to api files endpoint (#306)
parent
77c1f634
Changes
5
Show whitespace changes
Inline
Side-by-side
Showing
5 changed files
with
256 additions
and
220 deletions
+256
-220
bytes.go
pkg/api/bytes.go
+12
-17
export_test.go
pkg/api/export_test.go
+2
-1
file.go
pkg/api/file.go
+132
-148
file_test.go
pkg/api/file_test.go
+105
-47
router.go
pkg/api/router.go
+5
-7
No files found.
pkg/api/bytes.go
View file @
53885e61
...
@@ -27,45 +27,40 @@ type bytesPostResponse struct {
...
@@ -27,45 +27,40 @@ type bytesPostResponse struct {
// bytesUploadHandler handles upload of raw binary data of arbitrary length.
// bytesUploadHandler handles upload of raw binary data of arbitrary length.
func
(
s
*
server
)
bytesUploadHandler
(
w
http
.
ResponseWriter
,
r
*
http
.
Request
)
{
func
(
s
*
server
)
bytesUploadHandler
(
w
http
.
ResponseWriter
,
r
*
http
.
Request
)
{
ctx
:=
r
.
Context
()
ctx
:=
r
.
Context
()
responseObject
,
err
:=
s
.
splitUpload
(
ctx
,
r
.
Body
,
r
.
ContentLength
)
address
,
err
:=
s
.
splitUpload
(
ctx
,
r
.
Body
,
r
.
ContentLength
)
if
err
!=
nil
{
if
err
!=
nil
{
s
.
Logger
.
Debugf
(
"bytes upload: %v"
,
err
)
s
.
Logger
.
Debugf
(
"bytes upload: %v"
,
err
)
var
response
jsonhttp
.
StatusResponse
jsonhttp
.
InternalServerError
(
w
,
nil
)
response
.
Message
=
"upload error"
return
response
.
Code
=
http
.
StatusInternalServerError
jsonhttp
.
Respond
(
w
,
response
.
Code
,
response
)
}
else
{
jsonhttp
.
OK
(
w
,
responseObject
)
}
}
jsonhttp
.
OK
(
w
,
bytesPostResponse
{
Reference
:
address
,
})
}
}
func
(
s
*
server
)
splitUpload
(
ctx
context
.
Context
,
r
io
.
Read
Closer
,
l
int64
)
(
interface
{}
,
error
)
{
func
(
s
*
server
)
splitUpload
(
ctx
context
.
Context
,
r
io
.
Read
er
,
l
int64
)
(
swarm
.
Address
,
error
)
{
chunkPipe
:=
file
.
NewChunkPipe
()
chunkPipe
:=
file
.
NewChunkPipe
()
go
func
()
{
go
func
()
{
buf
:=
make
([]
byte
,
swarm
.
ChunkSize
)
buf
:=
make
([]
byte
,
swarm
.
ChunkSize
)
c
,
err
:=
io
.
CopyBuffer
(
chunkPipe
,
r
,
buf
)
c
,
err
:=
io
.
CopyBuffer
(
chunkPipe
,
r
,
buf
)
if
err
!=
nil
{
if
err
!=
nil
{
s
.
Logger
.
Debugf
(
"split upload: io error %d: %v"
,
c
,
err
)
s
.
Logger
.
Debugf
(
"split upload: io error %d: %v"
,
c
,
err
)
s
.
Logger
.
Error
(
"io error"
)
s
.
Logger
.
Error
(
"
split upload:
io error"
)
return
return
}
}
if
c
!=
l
{
if
c
!=
l
{
s
.
Logger
.
Debugf
(
"split upload: read count mismatch %d: %v"
,
c
,
err
)
s
.
Logger
.
Debugf
(
"split upload: read count mismatch %d: %v"
,
c
,
err
)
s
.
Logger
.
Error
(
"read count mismatch"
)
s
.
Logger
.
Error
(
"
split upload:
read count mismatch"
)
return
return
}
}
err
=
chunkPipe
.
Close
()
err
=
chunkPipe
.
Close
()
if
err
!=
nil
{
if
err
!=
nil
{
s
.
Logger
.
Error
f
(
"split upload: incomplete file write close %v"
,
err
)
s
.
Logger
.
Debug
f
(
"split upload: incomplete file write close %v"
,
err
)
s
.
Logger
.
Error
(
"incomplete file write close"
)
s
.
Logger
.
Error
(
"
split upload:
incomplete file write close"
)
}
}
}()
}()
sp
:=
splitter
.
NewSimpleSplitter
(
s
.
Storer
)
sp
:=
splitter
.
NewSimpleSplitter
(
s
.
Storer
)
address
,
err
:=
sp
.
Split
(
ctx
,
chunkPipe
,
l
)
return
sp
.
Split
(
ctx
,
chunkPipe
,
l
)
if
err
!=
nil
{
return
swarm
.
ZeroAddress
,
err
}
return
bytesPostResponse
{
Reference
:
address
},
nil
}
}
// bytesGetHandler handles retrieval of raw binary data of arbitrary length.
// bytesGetHandler handles retrieval of raw binary data of arbitrary length.
...
...
pkg/api/export_test.go
View file @
53885e61
...
@@ -6,4 +6,5 @@ package api
...
@@ -6,4 +6,5 @@ package api
type
(
type
(
BytesPostResponse
=
bytesPostResponse
BytesPostResponse
=
bytesPostResponse
FileUploadResponse
=
fileUploadResponse
)
)
pkg/api/file.go
View file @
53885e61
...
@@ -7,7 +7,6 @@ package api
...
@@ -7,7 +7,6 @@ package api
import
(
import
(
"bufio"
"bufio"
"bytes"
"bytes"
"context"
"encoding/json"
"encoding/json"
"errors"
"errors"
"fmt"
"fmt"
...
@@ -28,72 +27,78 @@ import (
...
@@ -28,72 +27,78 @@ import (
"github.com/gorilla/mux"
"github.com/gorilla/mux"
)
)
const
(
const
multipartFormDataMediaType
=
"multipart/form-data"
MultiPartFormData
=
"multipart/form-data"
)
type
F
ileUploadResponse
struct
{
type
f
ileUploadResponse
struct
{
Reference
swarm
.
Address
`json:"reference"`
Reference
swarm
.
Address
`json:"reference"`
}
}
// bzzFileUploadHandler uploads the file and its metadata supplied as a multipart http message.
// fileUploadHandler uploads the file and its metadata supplied as:
func
(
s
*
server
)
bzzFileUploadHandler
(
w
http
.
ResponseWriter
,
r
*
http
.
Request
)
{
// - multipart http message
contentType
,
params
,
err
:=
mime
.
ParseMediaType
(
r
.
Header
.
Get
(
"Content-Type"
))
// - other content types as complete file body
if
contentType
!=
MultiPartFormData
{
func
(
s
*
server
)
fileUploadHandler
(
w
http
.
ResponseWriter
,
r
*
http
.
Request
)
{
s
.
Logger
.
Debugf
(
"file: no mutlipart: %v"
,
err
)
contentType
:=
r
.
Header
.
Get
(
"Content-Type"
)
s
.
Logger
.
Error
(
"file: no mutlipart"
)
mediaType
,
params
,
err
:=
mime
.
ParseMediaType
(
contentType
)
jsonhttp
.
BadRequest
(
w
,
"not a mutlipart/form-data"
)
if
err
!=
nil
{
s
.
Logger
.
Debugf
(
"file upload: parse content type header %q: %v"
,
contentType
,
err
)
s
.
Logger
.
Errorf
(
"file upload: parse content type header %q"
,
contentType
)
jsonhttp
.
BadRequest
(
w
,
"invalid content-type header"
)
return
return
}
}
ctx
:=
r
.
Context
()
var
reader
io
.
Reader
var
fileName
,
contentLength
string
var
fileSize
uint64
if
mediaType
==
multipartFormDataMediaType
{
mr
:=
multipart
.
NewReader
(
r
.
Body
,
params
[
"boundary"
])
mr
:=
multipart
.
NewReader
(
r
.
Body
,
params
[
"boundary"
])
for
{
// read only the first part, as only one file upload is supported
part
,
err
:=
mr
.
NextPart
()
part
,
err
:=
mr
.
NextPart
()
if
err
==
io
.
EOF
{
if
err
!=
nil
{
break
s
.
Logger
.
Debugf
(
"file upload: read multipart: %v"
,
err
)
}
else
if
err
!=
nil
{
s
.
Logger
.
Error
(
"file upload: read multipart"
)
s
.
Logger
.
Debugf
(
"file: read mutlipart: %v"
,
err
)
jsonhttp
.
BadRequest
(
w
,
"invalid multipart/form-data"
)
s
.
Logger
.
Error
(
"file: read mutlipart"
)
jsonhttp
.
BadRequest
(
w
,
"error reading a mutlipart/form-data"
)
return
return
}
}
ctx
:=
r
.
Context
()
// try to find filename
// try to find filename
// 1) in part header params
// 1) in part header params
// 2) as formname
// 2) as formname
// 3) file reference hash (after uploading the file)
// 3) file reference hash (after uploading the file)
fileName
:=
part
.
FileName
()
if
fileName
=
part
.
FileName
();
fileName
==
""
{
if
fileName
==
""
{
fileName
=
part
.
FormName
()
fileName
=
part
.
FormName
()
}
}
var
reader
io
.
ReadCloser
// then find out content type
// then find out content type
contentType
:
=
part
.
Header
.
Get
(
"Content-Type"
)
contentType
=
part
.
Header
.
Get
(
"Content-Type"
)
if
contentType
==
""
{
if
contentType
==
""
{
br
:=
bufio
.
NewReader
(
part
)
br
:=
bufio
.
NewReader
(
part
)
buf
,
err
:=
br
.
Peek
(
512
)
buf
,
err
:=
br
.
Peek
(
512
)
if
err
!=
nil
&&
err
!=
io
.
EOF
{
if
err
!=
nil
&&
err
!=
io
.
EOF
{
s
.
Logger
.
Debugf
(
"file
: read content type: %v, file name %s"
,
err
,
fileName
)
s
.
Logger
.
Debugf
(
"file
upload: read content type, file %q: %v"
,
fileName
,
err
)
s
.
Logger
.
Error
(
"file: read content type"
)
s
.
Logger
.
Error
f
(
"file upload: read content type, file %q"
,
fileName
)
jsonhttp
.
BadRequest
(
w
,
"error reading content type"
)
jsonhttp
.
BadRequest
(
w
,
"error reading content type"
)
return
return
}
}
contentType
=
http
.
DetectContentType
(
buf
)
contentType
=
http
.
DetectContentType
(
buf
)
reader
=
ioutil
.
NopCloser
(
br
)
reader
=
br
}
else
{
}
else
{
reader
=
part
reader
=
part
}
}
contentLength
=
part
.
Header
.
Get
(
"Content-Length"
)
}
else
{
fileName
=
r
.
URL
.
Query
()
.
Get
(
"name"
)
contentLength
=
r
.
Header
.
Get
(
"Content-Length"
)
reader
=
r
.
Body
}
var
fileSize
uint64
if
contentLength
!=
""
{
if
contentLength
:=
part
.
Header
.
Get
(
"Content-Length"
);
contentLength
!=
""
{
fileSize
,
err
=
strconv
.
ParseUint
(
contentLength
,
10
,
64
)
fileSize
,
err
=
strconv
.
ParseUint
(
contentLength
,
10
,
64
)
if
err
!=
nil
{
if
err
!=
nil
{
s
.
Logger
.
Debugf
(
"file: content length: %v"
,
err
)
s
.
Logger
.
Debugf
(
"file upload: content length, file %q: %v"
,
fileName
,
err
)
s
.
Logger
.
Error
(
"file: content length"
)
s
.
Logger
.
Errorf
(
"file upload: content length, file %q"
,
fileName
)
jsonhttp
.
BadRequest
(
w
,
"invalid content length header"
)
jsonhttp
.
BadRequest
(
w
,
"invalid content length header"
)
return
return
}
}
...
@@ -101,23 +106,23 @@ func (s *server) bzzFileUploadHandler(w http.ResponseWriter, r *http.Request) {
...
@@ -101,23 +106,23 @@ func (s *server) bzzFileUploadHandler(w http.ResponseWriter, r *http.Request) {
// copy the part to a tmp file to get its size
// copy the part to a tmp file to get its size
tmp
,
err
:=
ioutil
.
TempFile
(
""
,
"bee-multipart"
)
tmp
,
err
:=
ioutil
.
TempFile
(
""
,
"bee-multipart"
)
if
err
!=
nil
{
if
err
!=
nil
{
s
.
Logger
.
Debugf
(
"file
: create temporary file: %v"
,
err
)
s
.
Logger
.
Debugf
(
"file upload
: create temporary file: %v"
,
err
)
s
.
Logger
.
Error
(
"file
: create temporary file"
)
s
.
Logger
.
Errorf
(
"file upload
: create temporary file"
)
jsonhttp
.
InternalServerError
(
w
,
nil
)
jsonhttp
.
InternalServerError
(
w
,
nil
)
return
return
}
}
defer
os
.
Remove
(
tmp
.
Name
())
defer
os
.
Remove
(
tmp
.
Name
())
defer
tmp
.
Close
()
defer
tmp
.
Close
()
n
,
err
:=
io
.
Copy
(
tmp
,
part
)
n
,
err
:=
io
.
Copy
(
tmp
,
reader
)
if
err
!=
nil
{
if
err
!=
nil
{
s
.
Logger
.
Debugf
(
"file
: write temporary file: %v"
,
err
)
s
.
Logger
.
Debugf
(
"file upload
: write temporary file: %v"
,
err
)
s
.
Logger
.
Error
(
"file
: write temporary file"
)
s
.
Logger
.
Error
(
"file upload
: write temporary file"
)
jsonhttp
.
InternalServerError
(
w
,
nil
)
jsonhttp
.
InternalServerError
(
w
,
nil
)
return
return
}
}
if
_
,
err
:=
tmp
.
Seek
(
0
,
io
.
SeekStart
);
err
!=
nil
{
if
_
,
err
:=
tmp
.
Seek
(
0
,
io
.
SeekStart
);
err
!=
nil
{
s
.
Logger
.
Debugf
(
"file
: seek to beginning of temporary file: %v"
,
err
)
s
.
Logger
.
Debugf
(
"file upload
: seek to beginning of temporary file: %v"
,
err
)
s
.
Logger
.
Error
(
"file
: seek to beginning of temporary file"
)
s
.
Logger
.
Error
(
"file upload
: seek to beginning of temporary file"
)
jsonhttp
.
InternalServerError
(
w
,
nil
)
jsonhttp
.
InternalServerError
(
w
,
nil
)
return
return
}
}
...
@@ -126,15 +131,15 @@ func (s *server) bzzFileUploadHandler(w http.ResponseWriter, r *http.Request) {
...
@@ -126,15 +131,15 @@ func (s *server) bzzFileUploadHandler(w http.ResponseWriter, r *http.Request) {
}
}
// first store the file and get its reference
// first store the file and get its reference
fr
,
err
:=
s
.
storePartData
(
ctx
,
reader
,
fileSize
)
fr
,
err
:=
s
.
splitUpload
(
ctx
,
reader
,
int64
(
fileSize
)
)
if
err
!=
nil
{
if
err
!=
nil
{
s
.
Logger
.
Debugf
(
"file: file store: %v,"
,
err
)
s
.
Logger
.
Debugf
(
"file upload: file store, file %q: %v"
,
fileName
,
err
)
s
.
Logger
.
Error
(
"file: file store"
)
s
.
Logger
.
Errorf
(
"file upload: file store, file %q"
,
fileName
)
jsonhttp
.
InternalServerError
(
w
,
"could not store file data"
)
jsonhttp
.
InternalServerError
(
w
,
"could not store file data"
)
return
return
}
}
// If filename is still empty, use the file hash
the filename
// If filename is still empty, use the file hash as
the filename
if
fileName
==
""
{
if
fileName
==
""
{
fileName
=
fr
.
String
()
fileName
=
fr
.
String
()
}
}
...
@@ -144,15 +149,15 @@ func (s *server) bzzFileUploadHandler(w http.ResponseWriter, r *http.Request) {
...
@@ -144,15 +149,15 @@ func (s *server) bzzFileUploadHandler(w http.ResponseWriter, r *http.Request) {
m
.
MimeType
=
contentType
m
.
MimeType
=
contentType
metadataBytes
,
err
:=
json
.
Marshal
(
m
)
metadataBytes
,
err
:=
json
.
Marshal
(
m
)
if
err
!=
nil
{
if
err
!=
nil
{
s
.
Logger
.
Debugf
(
"file: metadata marshall: %v, file name %s"
,
err
,
fileName
)
s
.
Logger
.
Debugf
(
"file upload: metadata marshal, file %q: %v"
,
fileName
,
err
)
s
.
Logger
.
Error
(
"file: metadata marshall"
)
s
.
Logger
.
Errorf
(
"file upload: metadata marshal, file %q"
,
fileName
)
jsonhttp
.
InternalServerError
(
w
,
"metadata marshal
l error"
)
jsonhttp
.
InternalServerError
(
w
,
"metadata marsha
l error"
)
return
return
}
}
mr
,
err
:=
s
.
storeMeta
(
ctx
,
metadataBytes
)
mr
,
err
:=
s
.
splitUpload
(
ctx
,
bytes
.
NewReader
(
metadataBytes
),
int64
(
len
(
metadataBytes
))
)
if
err
!=
nil
{
if
err
!=
nil
{
s
.
Logger
.
Debugf
(
"file: metadata store: %v, file name %s"
,
err
,
fileName
)
s
.
Logger
.
Debugf
(
"file upload: metadata store, file %q: %v"
,
fileName
,
err
)
s
.
Logger
.
Error
(
"file: metadata store"
)
s
.
Logger
.
Errorf
(
"file upload: metadata store, file %q"
,
fileName
)
jsonhttp
.
InternalServerError
(
w
,
"could not store metadata"
)
jsonhttp
.
InternalServerError
(
w
,
"could not store metadata"
)
return
return
}
}
...
@@ -161,30 +166,31 @@ func (s *server) bzzFileUploadHandler(w http.ResponseWriter, r *http.Request) {
...
@@ -161,30 +166,31 @@ func (s *server) bzzFileUploadHandler(w http.ResponseWriter, r *http.Request) {
entrie
:=
entry
.
New
(
fr
,
mr
)
entrie
:=
entry
.
New
(
fr
,
mr
)
fileEntryBytes
,
err
:=
entrie
.
MarshalBinary
()
fileEntryBytes
,
err
:=
entrie
.
MarshalBinary
()
if
err
!=
nil
{
if
err
!=
nil
{
s
.
Logger
.
Debugf
(
"file: entry marshall: %v, file name %s"
,
err
,
fileName
)
s
.
Logger
.
Debugf
(
"file upload: entry marshal, file %q: %v"
,
fileName
,
err
)
s
.
Logger
.
Error
(
"file: entry marshall"
)
s
.
Logger
.
Errorf
(
"file upload: entry marshal, file %q"
,
fileName
)
jsonhttp
.
InternalServerError
(
w
,
"entry marshal
l error"
)
jsonhttp
.
InternalServerError
(
w
,
"entry marsha
l error"
)
return
return
}
}
er
,
err
:=
s
.
storeMeta
(
ctx
,
fileEntryBytes
)
reference
,
err
:=
s
.
splitUpload
(
ctx
,
bytes
.
NewReader
(
fileEntryBytes
),
int64
(
len
(
fileEntryBytes
))
)
if
err
!=
nil
{
if
err
!=
nil
{
s
.
Logger
.
Debugf
(
"file: entry store: %v, file name %s"
,
err
,
fileName
)
s
.
Logger
.
Debugf
(
"file upload: entry store, file %q: %v"
,
fileName
,
err
)
s
.
Logger
.
Error
(
"file: entry store"
)
s
.
Logger
.
Errorf
(
"file upload: entry store, file %q"
,
fileName
)
jsonhttp
.
InternalServerError
(
w
,
"could not store entry"
)
jsonhttp
.
InternalServerError
(
w
,
"could not store entry"
)
return
return
}
}
w
.
Header
()
.
Set
(
"ETag"
,
fmt
.
Sprintf
(
"%q"
,
er
.
String
()))
w
.
Header
()
.
Set
(
"ETag"
,
fmt
.
Sprintf
(
"%q"
,
reference
.
String
()))
jsonhttp
.
OK
(
w
,
&
FileUploadResponse
{
Reference
:
er
})
jsonhttp
.
OK
(
w
,
fileUploadResponse
{
}
Reference
:
reference
,
})
}
}
//
bzzF
ileDownloadHandler downloads the file given the entry's reference.
//
f
ileDownloadHandler downloads the file given the entry's reference.
func
(
s
*
server
)
bzzF
ileDownloadHandler
(
w
http
.
ResponseWriter
,
r
*
http
.
Request
)
{
func
(
s
*
server
)
f
ileDownloadHandler
(
w
http
.
ResponseWriter
,
r
*
http
.
Request
)
{
addr
:=
mux
.
Vars
(
r
)[
"addr"
]
addr
:=
mux
.
Vars
(
r
)[
"addr"
]
address
,
err
:=
swarm
.
ParseHexAddress
(
addr
)
address
,
err
:=
swarm
.
ParseHexAddress
(
addr
)
if
err
!=
nil
{
if
err
!=
nil
{
s
.
Logger
.
Debugf
(
"file: parse file address %s: %v"
,
addr
,
err
)
s
.
Logger
.
Debugf
(
"file
download
: parse file address %s: %v"
,
addr
,
err
)
s
.
Logger
.
Error
(
"file: parse file address"
)
s
.
Logger
.
Error
f
(
"file download: parse file address %s"
,
addr
)
jsonhttp
.
BadRequest
(
w
,
"invalid file address"
)
jsonhttp
.
BadRequest
(
w
,
"invalid file address"
)
return
return
}
}
...
@@ -194,17 +200,17 @@ func (s *server) bzzFileDownloadHandler(w http.ResponseWriter, r *http.Request)
...
@@ -194,17 +200,17 @@ func (s *server) bzzFileDownloadHandler(w http.ResponseWriter, r *http.Request)
buf
:=
bytes
.
NewBuffer
(
nil
)
buf
:=
bytes
.
NewBuffer
(
nil
)
_
,
err
=
file
.
JoinReadAll
(
j
,
address
,
buf
)
_
,
err
=
file
.
JoinReadAll
(
j
,
address
,
buf
)
if
err
!=
nil
{
if
err
!=
nil
{
s
.
Logger
.
Debugf
(
"file: read entry %s: %v"
,
addr
,
err
)
s
.
Logger
.
Debugf
(
"file
download
: read entry %s: %v"
,
addr
,
err
)
s
.
Logger
.
Error
(
"file: read entry"
)
s
.
Logger
.
Error
f
(
"file download: read entry %s"
,
addr
)
jsonhttp
.
InternalServerError
(
w
,
"error reading entry"
)
jsonhttp
.
InternalServerError
(
w
,
"error reading entry"
)
return
return
}
}
e
:=
&
entry
.
Entry
{}
e
:=
&
entry
.
Entry
{}
err
=
e
.
UnmarshalBinary
(
buf
.
Bytes
())
err
=
e
.
UnmarshalBinary
(
buf
.
Bytes
())
if
err
!=
nil
{
if
err
!=
nil
{
s
.
Logger
.
Debugf
(
"file
: unmarshal
l entry %s: %v"
,
addr
,
err
)
s
.
Logger
.
Debugf
(
"file
download: unmarsha
l entry %s: %v"
,
addr
,
err
)
s
.
Logger
.
Error
(
"file: unmarshall entry"
)
s
.
Logger
.
Error
f
(
"file download: unmarshal entry %s"
,
addr
)
jsonhttp
.
InternalServerError
(
w
,
"error unmarshal
l
ing entry"
)
jsonhttp
.
InternalServerError
(
w
,
"error unmarshaling entry"
)
return
return
}
}
...
@@ -222,17 +228,17 @@ func (s *server) bzzFileDownloadHandler(w http.ResponseWriter, r *http.Request)
...
@@ -222,17 +228,17 @@ func (s *server) bzzFileDownloadHandler(w http.ResponseWriter, r *http.Request)
buf
=
bytes
.
NewBuffer
(
nil
)
buf
=
bytes
.
NewBuffer
(
nil
)
_
,
err
=
file
.
JoinReadAll
(
j
,
e
.
Metadata
(),
buf
)
_
,
err
=
file
.
JoinReadAll
(
j
,
e
.
Metadata
(),
buf
)
if
err
!=
nil
{
if
err
!=
nil
{
s
.
Logger
.
Debugf
(
"file: read metadata %s: %v"
,
addr
,
err
)
s
.
Logger
.
Debugf
(
"file
download
: read metadata %s: %v"
,
addr
,
err
)
s
.
Logger
.
Error
(
"file: read netadata"
)
s
.
Logger
.
Error
f
(
"file download: read metadata %s"
,
addr
)
jsonhttp
.
InternalServerError
(
w
,
"error reading metadata"
)
jsonhttp
.
InternalServerError
(
w
,
"error reading metadata"
)
return
return
}
}
metaData
:=
&
entry
.
Metadata
{}
metaData
:=
&
entry
.
Metadata
{}
err
=
json
.
Unmarshal
(
buf
.
Bytes
(),
metaData
)
err
=
json
.
Unmarshal
(
buf
.
Bytes
(),
metaData
)
if
err
!=
nil
{
if
err
!=
nil
{
s
.
Logger
.
Debugf
(
"file
: unmarshal
l metadata %s: %v"
,
addr
,
err
)
s
.
Logger
.
Debugf
(
"file
download: unmarsha
l metadata %s: %v"
,
addr
,
err
)
s
.
Logger
.
Error
(
"file: unmarshall metadata"
)
s
.
Logger
.
Error
f
(
"file download: unmarshal metadata %s"
,
addr
)
jsonhttp
.
InternalServerError
(
w
,
"error unmarshal
l
ing metadata"
)
jsonhttp
.
InternalServerError
(
w
,
"error unmarshaling metadata"
)
return
return
}
}
...
@@ -240,13 +246,13 @@ func (s *server) bzzFileDownloadHandler(w http.ResponseWriter, r *http.Request)
...
@@ -240,13 +246,13 @@ func (s *server) bzzFileDownloadHandler(w http.ResponseWriter, r *http.Request)
dataSize
,
err
:=
j
.
Size
(
r
.
Context
(),
address
)
dataSize
,
err
:=
j
.
Size
(
r
.
Context
(),
address
)
if
err
!=
nil
{
if
err
!=
nil
{
if
errors
.
Is
(
err
,
storage
.
ErrNotFound
)
{
if
errors
.
Is
(
err
,
storage
.
ErrNotFound
)
{
s
.
Logger
.
Debugf
(
"file: not found %s: %v"
,
address
,
err
)
s
.
Logger
.
Debugf
(
"file
download
: not found %s: %v"
,
address
,
err
)
s
.
Logger
.
Error
(
"file: not found"
)
s
.
Logger
.
Error
f
(
"file download: not found %s"
,
addr
)
jsonhttp
.
NotFound
(
w
,
"not found"
)
jsonhttp
.
NotFound
(
w
,
"not found"
)
return
return
}
}
s
.
Logger
.
Debugf
(
"file: invalid root chunk %s: %v"
,
address
,
err
)
s
.
Logger
.
Debugf
(
"file
download
: invalid root chunk %s: %v"
,
address
,
err
)
s
.
Logger
.
Error
(
"file: invalid root chunk"
)
s
.
Logger
.
Error
f
(
"file download: invalid root chunk %s"
,
addr
)
jsonhttp
.
BadRequest
(
w
,
"invalid root chunk"
)
jsonhttp
.
BadRequest
(
w
,
"invalid root chunk"
)
return
return
}
}
...
@@ -254,8 +260,8 @@ func (s *server) bzzFileDownloadHandler(w http.ResponseWriter, r *http.Request)
...
@@ -254,8 +260,8 @@ func (s *server) bzzFileDownloadHandler(w http.ResponseWriter, r *http.Request)
outBuffer
:=
bytes
.
NewBuffer
(
nil
)
outBuffer
:=
bytes
.
NewBuffer
(
nil
)
c
,
err
:=
file
.
JoinReadAll
(
j
,
e
.
Reference
(),
outBuffer
)
c
,
err
:=
file
.
JoinReadAll
(
j
,
e
.
Reference
(),
outBuffer
)
if
err
!=
nil
&&
c
==
0
{
if
err
!=
nil
&&
c
==
0
{
s
.
Logger
.
Debugf
(
"file: data read %s: %v"
,
addr
,
err
)
s
.
Logger
.
Debugf
(
"file
download
: data read %s: %v"
,
addr
,
err
)
s
.
Logger
.
Error
(
"file: data read"
)
s
.
Logger
.
Error
f
(
"file download: data read %s"
,
addr
)
jsonhttp
.
InternalServerError
(
w
,
"error reading data"
)
jsonhttp
.
InternalServerError
(
w
,
"error reading data"
)
return
return
}
}
...
@@ -265,25 +271,3 @@ func (s *server) bzzFileDownloadHandler(w http.ResponseWriter, r *http.Request)
...
@@ -265,25 +271,3 @@ func (s *server) bzzFileDownloadHandler(w http.ResponseWriter, r *http.Request)
w
.
Header
()
.
Set
(
"Content-Length"
,
fmt
.
Sprintf
(
"%d"
,
dataSize
))
w
.
Header
()
.
Set
(
"Content-Length"
,
fmt
.
Sprintf
(
"%d"
,
dataSize
))
_
,
_
=
io
.
Copy
(
w
,
outBuffer
)
_
,
_
=
io
.
Copy
(
w
,
outBuffer
)
}
}
// storeMeta is used to store metadata information as a whole.
func
(
s
*
server
)
storeMeta
(
ctx
context
.
Context
,
dataBytes
[]
byte
)
(
swarm
.
Address
,
error
)
{
dataBuf
:=
bytes
.
NewBuffer
(
dataBytes
)
dataReadCloser
:=
ioutil
.
NopCloser
(
dataBuf
)
o
,
err
:=
s
.
splitUpload
(
ctx
,
dataReadCloser
,
int64
(
len
(
dataBytes
)))
if
err
!=
nil
{
return
swarm
.
ZeroAddress
,
err
}
bytesResp
:=
o
.
(
bytesPostResponse
)
return
bytesResp
.
Reference
,
nil
}
// storePartData stores file data belonging to one of the part of multipart.
func
(
s
*
server
)
storePartData
(
ctx
context
.
Context
,
r
io
.
ReadCloser
,
l
uint64
)
(
swarm
.
Address
,
error
)
{
o
,
err
:=
s
.
splitUpload
(
ctx
,
r
,
int64
(
l
))
if
err
!=
nil
{
return
swarm
.
ZeroAddress
,
err
}
bytesResp
:=
o
.
(
bytesPostResponse
)
return
bytesResp
.
Reference
,
nil
}
pkg/api/file_test.go
View file @
53885e61
...
@@ -10,6 +10,7 @@ import (
...
@@ -10,6 +10,7 @@ import (
"io/ioutil"
"io/ioutil"
"mime"
"mime"
"net/http"
"net/http"
"strings"
"testing"
"testing"
"github.com/ethersphere/bee/pkg/api"
"github.com/ethersphere/bee/pkg/api"
...
@@ -21,29 +22,29 @@ import (
...
@@ -21,29 +22,29 @@ import (
"github.com/ethersphere/bee/pkg/tags"
"github.com/ethersphere/bee/pkg/tags"
)
)
func
Test
Bzz
(
t
*
testing
.
T
)
{
func
Test
Files
(
t
*
testing
.
T
)
{
var
(
var
(
simpleResource
=
func
()
string
{
return
"/files"
}
fileUploadResource
=
"/files"
address
Resource
=
func
(
addr
string
)
string
{
return
"/files/"
+
addr
}
fileDownload
Resource
=
func
(
addr
string
)
string
{
return
"/files/"
+
addr
}
simpleData
=
[]
byte
(
"this is a simple text"
)
simpleData
=
[]
byte
(
"this is a simple text"
)
mockStorer
=
mock
.
NewStorer
()
client
=
newTestServer
(
t
,
testServerOptions
{
client
=
newTestServer
(
t
,
testServerOptions
{
Storer
:
mock
Storer
,
Storer
:
mock
.
NewStorer
()
,
Tags
:
tags
.
NewTags
(),
Tags
:
tags
.
NewTags
(),
Logger
:
logging
.
New
(
ioutil
.
Discard
,
5
),
Logger
:
logging
.
New
(
ioutil
.
Discard
,
5
),
})
})
)
)
t
.
Run
(
"simple-upload"
,
func
(
t
*
testing
.
T
)
{
jsonhttptest
.
ResponseDirectSendHeadersAndReceiveHeaders
(
t
,
client
,
http
.
MethodPost
,
simpleResource
(),
bytes
.
NewReader
(
simpleData
),
http
.
StatusBadRequest
,
jsonhttp
.
StatusResponse
{
t
.
Run
(
"invalid-content-type"
,
func
(
t
*
testing
.
T
)
{
Message
:
"not a mutlipart/form-data"
,
jsonhttptest
.
ResponseDirectSendHeadersAndReceiveHeaders
(
t
,
client
,
http
.
MethodPost
,
fileUploadResource
,
bytes
.
NewReader
(
simpleData
),
http
.
StatusBadRequest
,
jsonhttp
.
StatusResponse
{
Message
:
"invalid content-type header"
,
Code
:
http
.
StatusBadRequest
,
Code
:
http
.
StatusBadRequest
,
},
nil
)
},
nil
)
})
})
t
.
Run
(
"
simple
-upload"
,
func
(
t
*
testing
.
T
)
{
t
.
Run
(
"
multipart
-upload"
,
func
(
t
*
testing
.
T
)
{
fileName
:=
"simple_file.txt"
fileName
:=
"simple_file.txt"
rootHash
:=
"295673cf7aa55d119dd6f82528c91d45b53dd63dc2e4ca4abf4ed8b3a0788085"
rootHash
:=
"295673cf7aa55d119dd6f82528c91d45b53dd63dc2e4ca4abf4ed8b3a0788085"
_
=
jsonhttptest
.
ResponseDirectWithMultiPart
(
t
,
client
,
http
.
MethodPost
,
simpleResource
()
,
fileName
,
simpleData
,
http
.
StatusOK
,
""
,
api
.
FileUploadResponse
{
_
=
jsonhttptest
.
ResponseDirectWithMultiPart
(
t
,
client
,
http
.
MethodPost
,
fileUploadResource
,
fileName
,
simpleData
,
http
.
StatusOK
,
""
,
api
.
FileUploadResponse
{
Reference
:
swarm
.
MustParseHexAddress
(
rootHash
),
Reference
:
swarm
.
MustParseHexAddress
(
rootHash
),
})
})
})
})
...
@@ -51,11 +52,35 @@ func TestBzz(t *testing.T) {
...
@@ -51,11 +52,35 @@ func TestBzz(t *testing.T) {
t
.
Run
(
"check-content-type-detection"
,
func
(
t
*
testing
.
T
)
{
t
.
Run
(
"check-content-type-detection"
,
func
(
t
*
testing
.
T
)
{
fileName
:=
"my-pictures.jpeg"
fileName
:=
"my-pictures.jpeg"
rootHash
:=
"f2e761160deda91c1fbfab065a5abf530b0766b3e102b51fbd626ba37c3bc581"
rootHash
:=
"f2e761160deda91c1fbfab065a5abf530b0766b3e102b51fbd626ba37c3bc581"
_
=
jsonhttptest
.
ResponseDirectWithMultiPart
(
t
,
client
,
http
.
MethodPost
,
simpleResource
(),
fileName
,
simpleData
,
http
.
StatusOK
,
"image/jpeg; charset=utf-8"
,
api
.
FileUploadResponse
{
t
.
Run
(
"binary"
,
func
(
t
*
testing
.
T
)
{
headers
:=
make
(
http
.
Header
)
headers
.
Add
(
"Content-Type"
,
"image/jpeg; charset=utf-8"
)
_
=
jsonhttptest
.
ResponseDirectSendHeadersAndReceiveHeaders
(
t
,
client
,
http
.
MethodPost
,
fileUploadResource
+
"?name="
+
fileName
,
bytes
.
NewReader
(
simpleData
),
http
.
StatusOK
,
api
.
FileUploadResponse
{
Reference
:
swarm
.
MustParseHexAddress
(
rootHash
),
},
headers
)
rcvdHeader
:=
jsonhttptest
.
ResponseDirectCheckBinaryResponse
(
t
,
client
,
http
.
MethodGet
,
fileDownloadResource
(
rootHash
),
nil
,
http
.
StatusOK
,
simpleData
,
nil
)
cd
:=
rcvdHeader
.
Get
(
"Content-Disposition"
)
_
,
params
,
err
:=
mime
.
ParseMediaType
(
cd
)
if
err
!=
nil
{
t
.
Fatal
(
err
)
}
if
params
[
"filename"
]
!=
fileName
{
t
.
Fatal
(
"Invalid file name detected"
)
}
if
rcvdHeader
.
Get
(
"Content-Type"
)
!=
"image/jpeg; charset=utf-8"
{
t
.
Fatal
(
"Invalid content type detected"
)
}
})
t
.
Run
(
"multipart"
,
func
(
t
*
testing
.
T
)
{
_
=
jsonhttptest
.
ResponseDirectWithMultiPart
(
t
,
client
,
http
.
MethodPost
,
fileUploadResource
,
fileName
,
simpleData
,
http
.
StatusOK
,
"image/jpeg; charset=utf-8"
,
api
.
FileUploadResponse
{
Reference
:
swarm
.
MustParseHexAddress
(
rootHash
),
Reference
:
swarm
.
MustParseHexAddress
(
rootHash
),
})
})
rcvdHeader
:=
jsonhttptest
.
ResponseDirectCheckBinaryResponse
(
t
,
client
,
http
.
MethodGet
,
address
Resource
(
rootHash
),
nil
,
http
.
StatusOK
,
simpleData
,
nil
)
rcvdHeader
:=
jsonhttptest
.
ResponseDirectCheckBinaryResponse
(
t
,
client
,
http
.
MethodGet
,
fileDownload
Resource
(
rootHash
),
nil
,
http
.
StatusOK
,
simpleData
,
nil
)
cd
:=
rcvdHeader
.
Get
(
"Content-Disposition"
)
cd
:=
rcvdHeader
.
Get
(
"Content-Disposition"
)
_
,
params
,
err
:=
mime
.
ParseMediaType
(
cd
)
_
,
params
,
err
:=
mime
.
ParseMediaType
(
cd
)
if
err
!=
nil
{
if
err
!=
nil
{
...
@@ -68,6 +93,7 @@ func TestBzz(t *testing.T) {
...
@@ -68,6 +93,7 @@ func TestBzz(t *testing.T) {
t
.
Fatal
(
"Invalid content type detected"
)
t
.
Fatal
(
"Invalid content type detected"
)
}
}
})
})
})
t
.
Run
(
"upload-then-download-and-check-data"
,
func
(
t
*
testing
.
T
)
{
t
.
Run
(
"upload-then-download-and-check-data"
,
func
(
t
*
testing
.
T
)
{
fileName
:=
"sample.html"
fileName
:=
"sample.html"
...
@@ -83,7 +109,37 @@ func TestBzz(t *testing.T) {
...
@@ -83,7 +109,37 @@ func TestBzz(t *testing.T) {
</body>
</body>
</html>`
</html>`
rcvdHeader
:=
jsonhttptest
.
ResponseDirectWithMultiPart
(
t
,
client
,
http
.
MethodPost
,
simpleResource
(),
fileName
,
[]
byte
(
sampleHtml
),
http
.
StatusOK
,
""
,
api
.
FileUploadResponse
{
t
.
Run
(
"binary"
,
func
(
t
*
testing
.
T
)
{
headers
:=
make
(
http
.
Header
)
headers
.
Add
(
"Content-Type"
,
"text/html; charset=utf-8"
)
rcvdHeader
:=
jsonhttptest
.
ResponseDirectSendHeadersAndReceiveHeaders
(
t
,
client
,
http
.
MethodPost
,
fileUploadResource
+
"?name="
+
fileName
,
strings
.
NewReader
(
sampleHtml
),
http
.
StatusOK
,
api
.
FileUploadResponse
{
Reference
:
swarm
.
MustParseHexAddress
(
rootHash
),
},
headers
)
if
rcvdHeader
.
Get
(
"ETag"
)
!=
fmt
.
Sprintf
(
"%q"
,
rootHash
)
{
t
.
Fatal
(
"Invalid ETags header received"
)
}
// try to fetch the same file and check the data
rcvdHeader
=
jsonhttptest
.
ResponseDirectCheckBinaryResponse
(
t
,
client
,
http
.
MethodGet
,
fileDownloadResource
(
rootHash
),
nil
,
http
.
StatusOK
,
[]
byte
(
sampleHtml
),
nil
)
// check the headers
cd
:=
rcvdHeader
.
Get
(
"Content-Disposition"
)
_
,
params
,
err
:=
mime
.
ParseMediaType
(
cd
)
if
err
!=
nil
{
t
.
Fatal
(
err
)
}
if
params
[
"filename"
]
!=
fileName
{
t
.
Fatal
(
"Invalid filename detected"
)
}
if
rcvdHeader
.
Get
(
"Content-Type"
)
!=
"text/html; charset=utf-8"
{
t
.
Fatal
(
"Invalid content type detected"
)
}
})
t
.
Run
(
"multipart"
,
func
(
t
*
testing
.
T
)
{
rcvdHeader
:=
jsonhttptest
.
ResponseDirectWithMultiPart
(
t
,
client
,
http
.
MethodPost
,
fileUploadResource
,
fileName
,
[]
byte
(
sampleHtml
),
http
.
StatusOK
,
""
,
api
.
FileUploadResponse
{
Reference
:
swarm
.
MustParseHexAddress
(
rootHash
),
Reference
:
swarm
.
MustParseHexAddress
(
rootHash
),
})
})
...
@@ -92,7 +148,7 @@ func TestBzz(t *testing.T) {
...
@@ -92,7 +148,7 @@ func TestBzz(t *testing.T) {
}
}
// try to fetch the same file and check the data
// try to fetch the same file and check the data
rcvdHeader
=
jsonhttptest
.
ResponseDirectCheckBinaryResponse
(
t
,
client
,
http
.
MethodGet
,
address
Resource
(
rootHash
),
nil
,
http
.
StatusOK
,
[]
byte
(
sampleHtml
),
nil
)
rcvdHeader
=
jsonhttptest
.
ResponseDirectCheckBinaryResponse
(
t
,
client
,
http
.
MethodGet
,
fileDownload
Resource
(
rootHash
),
nil
,
http
.
StatusOK
,
[]
byte
(
sampleHtml
),
nil
)
// check the headers
// check the headers
cd
:=
rcvdHeader
.
Get
(
"Content-Disposition"
)
cd
:=
rcvdHeader
.
Get
(
"Content-Disposition"
)
...
@@ -107,4 +163,6 @@ func TestBzz(t *testing.T) {
...
@@ -107,4 +163,6 @@ func TestBzz(t *testing.T) {
t
.
Fatal
(
"Invalid content type detected"
)
t
.
Fatal
(
"Invalid content type detected"
)
}
}
})
})
})
}
}
pkg/api/router.go
View file @
53885e61
...
@@ -35,18 +35,16 @@ func (s *server) setupRouting() {
...
@@ -35,18 +35,16 @@ func (s *server) setupRouting() {
fmt
.
Fprintln
(
w
,
"User-agent: *
\n
Disallow: /"
)
fmt
.
Fprintln
(
w
,
"User-agent: *
\n
Disallow: /"
)
})
})
handle
(
router
,
"/bytes"
,
jsonhttp
.
MethodHandler
{
"POST"
:
http
.
HandlerFunc
(
s
.
bytesUploadHandler
),
})
handle
(
router
,
"/files"
,
jsonhttp
.
MethodHandler
{
handle
(
router
,
"/files"
,
jsonhttp
.
MethodHandler
{
"POST"
:
http
.
HandlerFunc
(
s
.
bzzF
ileUploadHandler
),
"POST"
:
http
.
HandlerFunc
(
s
.
f
ileUploadHandler
),
})
})
handle
(
router
,
"/files/{addr}"
,
jsonhttp
.
MethodHandler
{
handle
(
router
,
"/files/{addr}"
,
jsonhttp
.
MethodHandler
{
"GET"
:
http
.
HandlerFunc
(
s
.
bzzF
ileDownloadHandler
),
"GET"
:
http
.
HandlerFunc
(
s
.
f
ileDownloadHandler
),
})
})
handle
(
router
,
"/bytes"
,
jsonhttp
.
MethodHandler
{
"POST"
:
http
.
HandlerFunc
(
s
.
bytesUploadHandler
),
})
handle
(
router
,
"/bytes/{address}"
,
jsonhttp
.
MethodHandler
{
handle
(
router
,
"/bytes/{address}"
,
jsonhttp
.
MethodHandler
{
"GET"
:
http
.
HandlerFunc
(
s
.
bytesGetHandler
),
"GET"
:
http
.
HandlerFunc
(
s
.
bytesGetHandler
),
})
})
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment