Skip to content
Projects
Groups
Snippets
Help
Loading...
Help
Support
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
M
mybee
Project
Project
Details
Activity
Releases
Cycle Analytics
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Charts
Issues
0
Issues
0
List
Boards
Labels
Milestones
Merge Requests
0
Merge Requests
0
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Charts
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Charts
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
vicotor
mybee
Commits
53885e61
Unverified
Commit
53885e61
authored
Jun 16, 2020
by
Janoš Guljaš
Committed by
GitHub
Jun 16, 2020
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
support binary upload to api files endpoint (#306)
parent
77c1f634
Changes
5
Hide whitespace changes
Inline
Side-by-side
Showing
5 changed files
with
256 additions
and
220 deletions
+256
-220
bytes.go
pkg/api/bytes.go
+12
-17
export_test.go
pkg/api/export_test.go
+2
-1
file.go
pkg/api/file.go
+132
-148
file_test.go
pkg/api/file_test.go
+105
-47
router.go
pkg/api/router.go
+5
-7
No files found.
pkg/api/bytes.go
View file @
53885e61
...
@@ -27,45 +27,40 @@ type bytesPostResponse struct {
...
@@ -27,45 +27,40 @@ type bytesPostResponse struct {
// bytesUploadHandler handles upload of raw binary data of arbitrary length.
// bytesUploadHandler handles upload of raw binary data of arbitrary length.
func
(
s
*
server
)
bytesUploadHandler
(
w
http
.
ResponseWriter
,
r
*
http
.
Request
)
{
func
(
s
*
server
)
bytesUploadHandler
(
w
http
.
ResponseWriter
,
r
*
http
.
Request
)
{
ctx
:=
r
.
Context
()
ctx
:=
r
.
Context
()
responseObject
,
err
:=
s
.
splitUpload
(
ctx
,
r
.
Body
,
r
.
ContentLength
)
address
,
err
:=
s
.
splitUpload
(
ctx
,
r
.
Body
,
r
.
ContentLength
)
if
err
!=
nil
{
if
err
!=
nil
{
s
.
Logger
.
Debugf
(
"bytes upload: %v"
,
err
)
s
.
Logger
.
Debugf
(
"bytes upload: %v"
,
err
)
var
response
jsonhttp
.
StatusResponse
jsonhttp
.
InternalServerError
(
w
,
nil
)
response
.
Message
=
"upload error"
return
response
.
Code
=
http
.
StatusInternalServerError
jsonhttp
.
Respond
(
w
,
response
.
Code
,
response
)
}
else
{
jsonhttp
.
OK
(
w
,
responseObject
)
}
}
jsonhttp
.
OK
(
w
,
bytesPostResponse
{
Reference
:
address
,
})
}
}
func
(
s
*
server
)
splitUpload
(
ctx
context
.
Context
,
r
io
.
Read
Closer
,
l
int64
)
(
interface
{}
,
error
)
{
func
(
s
*
server
)
splitUpload
(
ctx
context
.
Context
,
r
io
.
Read
er
,
l
int64
)
(
swarm
.
Address
,
error
)
{
chunkPipe
:=
file
.
NewChunkPipe
()
chunkPipe
:=
file
.
NewChunkPipe
()
go
func
()
{
go
func
()
{
buf
:=
make
([]
byte
,
swarm
.
ChunkSize
)
buf
:=
make
([]
byte
,
swarm
.
ChunkSize
)
c
,
err
:=
io
.
CopyBuffer
(
chunkPipe
,
r
,
buf
)
c
,
err
:=
io
.
CopyBuffer
(
chunkPipe
,
r
,
buf
)
if
err
!=
nil
{
if
err
!=
nil
{
s
.
Logger
.
Debugf
(
"split upload: io error %d: %v"
,
c
,
err
)
s
.
Logger
.
Debugf
(
"split upload: io error %d: %v"
,
c
,
err
)
s
.
Logger
.
Error
(
"io error"
)
s
.
Logger
.
Error
(
"
split upload:
io error"
)
return
return
}
}
if
c
!=
l
{
if
c
!=
l
{
s
.
Logger
.
Debugf
(
"split upload: read count mismatch %d: %v"
,
c
,
err
)
s
.
Logger
.
Debugf
(
"split upload: read count mismatch %d: %v"
,
c
,
err
)
s
.
Logger
.
Error
(
"read count mismatch"
)
s
.
Logger
.
Error
(
"
split upload:
read count mismatch"
)
return
return
}
}
err
=
chunkPipe
.
Close
()
err
=
chunkPipe
.
Close
()
if
err
!=
nil
{
if
err
!=
nil
{
s
.
Logger
.
Error
f
(
"split upload: incomplete file write close %v"
,
err
)
s
.
Logger
.
Debug
f
(
"split upload: incomplete file write close %v"
,
err
)
s
.
Logger
.
Error
(
"incomplete file write close"
)
s
.
Logger
.
Error
(
"
split upload:
incomplete file write close"
)
}
}
}()
}()
sp
:=
splitter
.
NewSimpleSplitter
(
s
.
Storer
)
sp
:=
splitter
.
NewSimpleSplitter
(
s
.
Storer
)
address
,
err
:=
sp
.
Split
(
ctx
,
chunkPipe
,
l
)
return
sp
.
Split
(
ctx
,
chunkPipe
,
l
)
if
err
!=
nil
{
return
swarm
.
ZeroAddress
,
err
}
return
bytesPostResponse
{
Reference
:
address
},
nil
}
}
// bytesGetHandler handles retrieval of raw binary data of arbitrary length.
// bytesGetHandler handles retrieval of raw binary data of arbitrary length.
...
...
pkg/api/export_test.go
View file @
53885e61
...
@@ -5,5 +5,6 @@
...
@@ -5,5 +5,6 @@
package
api
package
api
type
(
type
(
BytesPostResponse
=
bytesPostResponse
BytesPostResponse
=
bytesPostResponse
FileUploadResponse
=
fileUploadResponse
)
)
pkg/api/file.go
View file @
53885e61
...
@@ -7,7 +7,6 @@ package api
...
@@ -7,7 +7,6 @@ package api
import
(
import
(
"bufio"
"bufio"
"bytes"
"bytes"
"context"
"encoding/json"
"encoding/json"
"errors"
"errors"
"fmt"
"fmt"
...
@@ -28,163 +27,170 @@ import (
...
@@ -28,163 +27,170 @@ import (
"github.com/gorilla/mux"
"github.com/gorilla/mux"
)
)
const
(
const
multipartFormDataMediaType
=
"multipart/form-data"
MultiPartFormData
=
"multipart/form-data"
)
type
F
ileUploadResponse
struct
{
type
f
ileUploadResponse
struct
{
Reference
swarm
.
Address
`json:"reference"`
Reference
swarm
.
Address
`json:"reference"`
}
}
// bzzFileUploadHandler uploads the file and its metadata supplied as a multipart http message.
// fileUploadHandler uploads the file and its metadata supplied as:
func
(
s
*
server
)
bzzFileUploadHandler
(
w
http
.
ResponseWriter
,
r
*
http
.
Request
)
{
// - multipart http message
contentType
,
params
,
err
:=
mime
.
ParseMediaType
(
r
.
Header
.
Get
(
"Content-Type"
))
// - other content types as complete file body
if
contentType
!=
MultiPartFormData
{
func
(
s
*
server
)
fileUploadHandler
(
w
http
.
ResponseWriter
,
r
*
http
.
Request
)
{
s
.
Logger
.
Debugf
(
"file: no mutlipart: %v"
,
err
)
contentType
:=
r
.
Header
.
Get
(
"Content-Type"
)
s
.
Logger
.
Error
(
"file: no mutlipart"
)
mediaType
,
params
,
err
:=
mime
.
ParseMediaType
(
contentType
)
jsonhttp
.
BadRequest
(
w
,
"not a mutlipart/form-data"
)
if
err
!=
nil
{
s
.
Logger
.
Debugf
(
"file upload: parse content type header %q: %v"
,
contentType
,
err
)
s
.
Logger
.
Errorf
(
"file upload: parse content type header %q"
,
contentType
)
jsonhttp
.
BadRequest
(
w
,
"invalid content-type header"
)
return
return
}
}
mr
:=
multipart
.
NewReader
(
r
.
Body
,
params
[
"boundary"
])
ctx
:=
r
.
Context
()
for
{
var
reader
io
.
Reader
var
fileName
,
contentLength
string
var
fileSize
uint64
if
mediaType
==
multipartFormDataMediaType
{
mr
:=
multipart
.
NewReader
(
r
.
Body
,
params
[
"boundary"
])
// read only the first part, as only one file upload is supported
part
,
err
:=
mr
.
NextPart
()
part
,
err
:=
mr
.
NextPart
()
if
err
==
io
.
EOF
{
if
err
!=
nil
{
break
s
.
Logger
.
Debugf
(
"file upload: read multipart: %v"
,
err
)
}
else
if
err
!=
nil
{
s
.
Logger
.
Error
(
"file upload: read multipart"
)
s
.
Logger
.
Debugf
(
"file: read mutlipart: %v"
,
err
)
jsonhttp
.
BadRequest
(
w
,
"invalid multipart/form-data"
)
s
.
Logger
.
Error
(
"file: read mutlipart"
)
jsonhttp
.
BadRequest
(
w
,
"error reading a mutlipart/form-data"
)
return
return
}
}
ctx
:=
r
.
Context
()
// try to find filename
// try to find filename
// 1) in part header params
// 1) in part header params
// 2) as formname
// 2) as formname
// 3) file reference hash (after uploading the file)
// 3) file reference hash (after uploading the file)
fileName
:=
part
.
FileName
()
if
fileName
=
part
.
FileName
();
fileName
==
""
{
if
fileName
==
""
{
fileName
=
part
.
FormName
()
fileName
=
part
.
FormName
()
}
}
var
reader
io
.
ReadCloser
// then find out content type
// then find out content type
contentType
:
=
part
.
Header
.
Get
(
"Content-Type"
)
contentType
=
part
.
Header
.
Get
(
"Content-Type"
)
if
contentType
==
""
{
if
contentType
==
""
{
br
:=
bufio
.
NewReader
(
part
)
br
:=
bufio
.
NewReader
(
part
)
buf
,
err
:=
br
.
Peek
(
512
)
buf
,
err
:=
br
.
Peek
(
512
)
if
err
!=
nil
&&
err
!=
io
.
EOF
{
if
err
!=
nil
&&
err
!=
io
.
EOF
{
s
.
Logger
.
Debugf
(
"file
: read content type: %v, file name %s"
,
err
,
fileName
)
s
.
Logger
.
Debugf
(
"file
upload: read content type, file %q: %v"
,
fileName
,
err
)
s
.
Logger
.
Error
(
"file: read content type"
)
s
.
Logger
.
Error
f
(
"file upload: read content type, file %q"
,
fileName
)
jsonhttp
.
BadRequest
(
w
,
"error reading content type"
)
jsonhttp
.
BadRequest
(
w
,
"error reading content type"
)
return
return
}
}
contentType
=
http
.
DetectContentType
(
buf
)
contentType
=
http
.
DetectContentType
(
buf
)
reader
=
ioutil
.
NopCloser
(
br
)
reader
=
br
}
else
{
}
else
{
reader
=
part
reader
=
part
}
}
contentLength
=
part
.
Header
.
Get
(
"Content-Length"
)
}
else
{
fileName
=
r
.
URL
.
Query
()
.
Get
(
"name"
)
contentLength
=
r
.
Header
.
Get
(
"Content-Length"
)
reader
=
r
.
Body
}
var
fileSize
uint64
if
contentLength
!=
""
{
if
contentLength
:=
part
.
Header
.
Get
(
"Content-Length"
);
contentLength
!=
""
{
fileSize
,
err
=
strconv
.
ParseUint
(
contentLength
,
10
,
64
)
fileSize
,
err
=
strconv
.
ParseUint
(
contentLength
,
10
,
64
)
if
err
!=
nil
{
s
.
Logger
.
Debugf
(
"file: content length: %v"
,
err
)
s
.
Logger
.
Error
(
"file: content length"
)
jsonhttp
.
BadRequest
(
w
,
"invalid content length header"
)
return
}
}
else
{
// copy the part to a tmp file to get its size
tmp
,
err
:=
ioutil
.
TempFile
(
""
,
"bee-multipart"
)
if
err
!=
nil
{
s
.
Logger
.
Debugf
(
"file: create temporary file: %v"
,
err
)
s
.
Logger
.
Error
(
"file: create temporary file"
)
jsonhttp
.
InternalServerError
(
w
,
nil
)
return
}
defer
os
.
Remove
(
tmp
.
Name
())
defer
tmp
.
Close
()
n
,
err
:=
io
.
Copy
(
tmp
,
part
)
if
err
!=
nil
{
s
.
Logger
.
Debugf
(
"file: write temporary file: %v"
,
err
)
s
.
Logger
.
Error
(
"file: write temporary file"
)
jsonhttp
.
InternalServerError
(
w
,
nil
)
return
}
if
_
,
err
:=
tmp
.
Seek
(
0
,
io
.
SeekStart
);
err
!=
nil
{
s
.
Logger
.
Debugf
(
"file: seek to beginning of temporary file: %v"
,
err
)
s
.
Logger
.
Error
(
"file: seek to beginning of temporary file"
)
jsonhttp
.
InternalServerError
(
w
,
nil
)
return
}
fileSize
=
uint64
(
n
)
reader
=
tmp
}
// first store the file and get its reference
fr
,
err
:=
s
.
storePartData
(
ctx
,
reader
,
fileSize
)
if
err
!=
nil
{
s
.
Logger
.
Debugf
(
"file: file store: %v,"
,
err
)
s
.
Logger
.
Error
(
"file: file store"
)
jsonhttp
.
InternalServerError
(
w
,
"could not store file data"
)
return
}
// If filename is still empty, use the file hash the filename
if
fileName
==
""
{
fileName
=
fr
.
String
()
}
// then store the metadata and get its reference
m
:=
entry
.
NewMetadata
(
fileName
)
m
.
MimeType
=
contentType
metadataBytes
,
err
:=
json
.
Marshal
(
m
)
if
err
!=
nil
{
if
err
!=
nil
{
s
.
Logger
.
Debugf
(
"file
: metadata marshall: %v, file name %s"
,
err
,
fileName
)
s
.
Logger
.
Debugf
(
"file
upload: content length, file %q: %v"
,
fileName
,
err
)
s
.
Logger
.
Error
(
"file: metadata marshall"
)
s
.
Logger
.
Error
f
(
"file upload: content length, file %q"
,
fileName
)
jsonhttp
.
InternalServerError
(
w
,
"metadata marshall erro
r"
)
jsonhttp
.
BadRequest
(
w
,
"invalid content length heade
r"
)
return
return
}
}
mr
,
err
:=
s
.
storeMeta
(
ctx
,
metadataBytes
)
}
else
{
// copy the part to a tmp file to get its size
tmp
,
err
:=
ioutil
.
TempFile
(
""
,
"bee-multipart"
)
if
err
!=
nil
{
if
err
!=
nil
{
s
.
Logger
.
Debugf
(
"file
: metadata store: %v, file name %s"
,
err
,
fileName
)
s
.
Logger
.
Debugf
(
"file
upload: create temporary file: %v"
,
err
)
s
.
Logger
.
Error
(
"file: metadata stor
e"
)
s
.
Logger
.
Error
f
(
"file upload: create temporary fil
e"
)
jsonhttp
.
InternalServerError
(
w
,
"could not store metadata"
)
jsonhttp
.
InternalServerError
(
w
,
nil
)
return
return
}
}
defer
os
.
Remove
(
tmp
.
Name
())
// now join both references (mr,fr) to create an entry and store it.
defer
tmp
.
Close
()
entrie
:=
entry
.
New
(
fr
,
mr
)
n
,
err
:=
io
.
Copy
(
tmp
,
reader
)
fileEntryBytes
,
err
:=
entrie
.
MarshalBinary
()
if
err
!=
nil
{
if
err
!=
nil
{
s
.
Logger
.
Debugf
(
"file
: entry marshall: %v, file name %s"
,
err
,
fileName
)
s
.
Logger
.
Debugf
(
"file
upload: write temporary file: %v"
,
err
)
s
.
Logger
.
Error
(
"file
: entry marshall
"
)
s
.
Logger
.
Error
(
"file
upload: write temporary file
"
)
jsonhttp
.
InternalServerError
(
w
,
"entry marshall error"
)
jsonhttp
.
InternalServerError
(
w
,
nil
)
return
return
}
}
er
,
err
:=
s
.
storeMeta
(
ctx
,
fileEntryBytes
)
if
_
,
err
:=
tmp
.
Seek
(
0
,
io
.
SeekStart
);
err
!=
nil
{
if
err
!=
nil
{
s
.
Logger
.
Debugf
(
"file upload: seek to beginning of temporary file: %v"
,
err
)
s
.
Logger
.
Debugf
(
"file: entry store: %v, file name %s"
,
err
,
fileName
)
s
.
Logger
.
Error
(
"file upload: seek to beginning of temporary file"
)
s
.
Logger
.
Error
(
"file: entry store"
)
jsonhttp
.
InternalServerError
(
w
,
nil
)
jsonhttp
.
InternalServerError
(
w
,
"could not store entry"
)
return
return
}
}
w
.
Header
()
.
Set
(
"ETag"
,
fmt
.
Sprintf
(
"%q"
,
er
.
String
())
)
fileSize
=
uint64
(
n
)
jsonhttp
.
OK
(
w
,
&
FileUploadResponse
{
Reference
:
er
})
reader
=
tmp
}
}
// first store the file and get its reference
fr
,
err
:=
s
.
splitUpload
(
ctx
,
reader
,
int64
(
fileSize
))
if
err
!=
nil
{
s
.
Logger
.
Debugf
(
"file upload: file store, file %q: %v"
,
fileName
,
err
)
s
.
Logger
.
Errorf
(
"file upload: file store, file %q"
,
fileName
)
jsonhttp
.
InternalServerError
(
w
,
"could not store file data"
)
return
}
// If filename is still empty, use the file hash as the filename
if
fileName
==
""
{
fileName
=
fr
.
String
()
}
// then store the metadata and get its reference
m
:=
entry
.
NewMetadata
(
fileName
)
m
.
MimeType
=
contentType
metadataBytes
,
err
:=
json
.
Marshal
(
m
)
if
err
!=
nil
{
s
.
Logger
.
Debugf
(
"file upload: metadata marshal, file %q: %v"
,
fileName
,
err
)
s
.
Logger
.
Errorf
(
"file upload: metadata marshal, file %q"
,
fileName
)
jsonhttp
.
InternalServerError
(
w
,
"metadata marshal error"
)
return
}
mr
,
err
:=
s
.
splitUpload
(
ctx
,
bytes
.
NewReader
(
metadataBytes
),
int64
(
len
(
metadataBytes
)))
if
err
!=
nil
{
s
.
Logger
.
Debugf
(
"file upload: metadata store, file %q: %v"
,
fileName
,
err
)
s
.
Logger
.
Errorf
(
"file upload: metadata store, file %q"
,
fileName
)
jsonhttp
.
InternalServerError
(
w
,
"could not store metadata"
)
return
}
// now join both references (mr,fr) to create an entry and store it.
entrie
:=
entry
.
New
(
fr
,
mr
)
fileEntryBytes
,
err
:=
entrie
.
MarshalBinary
()
if
err
!=
nil
{
s
.
Logger
.
Debugf
(
"file upload: entry marshal, file %q: %v"
,
fileName
,
err
)
s
.
Logger
.
Errorf
(
"file upload: entry marshal, file %q"
,
fileName
)
jsonhttp
.
InternalServerError
(
w
,
"entry marshal error"
)
return
}
reference
,
err
:=
s
.
splitUpload
(
ctx
,
bytes
.
NewReader
(
fileEntryBytes
),
int64
(
len
(
fileEntryBytes
)))
if
err
!=
nil
{
s
.
Logger
.
Debugf
(
"file upload: entry store, file %q: %v"
,
fileName
,
err
)
s
.
Logger
.
Errorf
(
"file upload: entry store, file %q"
,
fileName
)
jsonhttp
.
InternalServerError
(
w
,
"could not store entry"
)
return
}
w
.
Header
()
.
Set
(
"ETag"
,
fmt
.
Sprintf
(
"%q"
,
reference
.
String
()))
jsonhttp
.
OK
(
w
,
fileUploadResponse
{
Reference
:
reference
,
})
}
}
//
bzzF
ileDownloadHandler downloads the file given the entry's reference.
//
f
ileDownloadHandler downloads the file given the entry's reference.
func
(
s
*
server
)
bzzF
ileDownloadHandler
(
w
http
.
ResponseWriter
,
r
*
http
.
Request
)
{
func
(
s
*
server
)
f
ileDownloadHandler
(
w
http
.
ResponseWriter
,
r
*
http
.
Request
)
{
addr
:=
mux
.
Vars
(
r
)[
"addr"
]
addr
:=
mux
.
Vars
(
r
)[
"addr"
]
address
,
err
:=
swarm
.
ParseHexAddress
(
addr
)
address
,
err
:=
swarm
.
ParseHexAddress
(
addr
)
if
err
!=
nil
{
if
err
!=
nil
{
s
.
Logger
.
Debugf
(
"file: parse file address %s: %v"
,
addr
,
err
)
s
.
Logger
.
Debugf
(
"file
download
: parse file address %s: %v"
,
addr
,
err
)
s
.
Logger
.
Error
(
"file: parse file address"
)
s
.
Logger
.
Error
f
(
"file download: parse file address %s"
,
addr
)
jsonhttp
.
BadRequest
(
w
,
"invalid file address"
)
jsonhttp
.
BadRequest
(
w
,
"invalid file address"
)
return
return
}
}
...
@@ -194,17 +200,17 @@ func (s *server) bzzFileDownloadHandler(w http.ResponseWriter, r *http.Request)
...
@@ -194,17 +200,17 @@ func (s *server) bzzFileDownloadHandler(w http.ResponseWriter, r *http.Request)
buf
:=
bytes
.
NewBuffer
(
nil
)
buf
:=
bytes
.
NewBuffer
(
nil
)
_
,
err
=
file
.
JoinReadAll
(
j
,
address
,
buf
)
_
,
err
=
file
.
JoinReadAll
(
j
,
address
,
buf
)
if
err
!=
nil
{
if
err
!=
nil
{
s
.
Logger
.
Debugf
(
"file: read entry %s: %v"
,
addr
,
err
)
s
.
Logger
.
Debugf
(
"file
download
: read entry %s: %v"
,
addr
,
err
)
s
.
Logger
.
Error
(
"file: read entry"
)
s
.
Logger
.
Error
f
(
"file download: read entry %s"
,
addr
)
jsonhttp
.
InternalServerError
(
w
,
"error reading entry"
)
jsonhttp
.
InternalServerError
(
w
,
"error reading entry"
)
return
return
}
}
e
:=
&
entry
.
Entry
{}
e
:=
&
entry
.
Entry
{}
err
=
e
.
UnmarshalBinary
(
buf
.
Bytes
())
err
=
e
.
UnmarshalBinary
(
buf
.
Bytes
())
if
err
!=
nil
{
if
err
!=
nil
{
s
.
Logger
.
Debugf
(
"file
: unmarshal
l entry %s: %v"
,
addr
,
err
)
s
.
Logger
.
Debugf
(
"file
download: unmarsha
l entry %s: %v"
,
addr
,
err
)
s
.
Logger
.
Error
(
"file: unmarshall entry"
)
s
.
Logger
.
Error
f
(
"file download: unmarshal entry %s"
,
addr
)
jsonhttp
.
InternalServerError
(
w
,
"error unmarshal
l
ing entry"
)
jsonhttp
.
InternalServerError
(
w
,
"error unmarshaling entry"
)
return
return
}
}
...
@@ -222,17 +228,17 @@ func (s *server) bzzFileDownloadHandler(w http.ResponseWriter, r *http.Request)
...
@@ -222,17 +228,17 @@ func (s *server) bzzFileDownloadHandler(w http.ResponseWriter, r *http.Request)
buf
=
bytes
.
NewBuffer
(
nil
)
buf
=
bytes
.
NewBuffer
(
nil
)
_
,
err
=
file
.
JoinReadAll
(
j
,
e
.
Metadata
(),
buf
)
_
,
err
=
file
.
JoinReadAll
(
j
,
e
.
Metadata
(),
buf
)
if
err
!=
nil
{
if
err
!=
nil
{
s
.
Logger
.
Debugf
(
"file: read metadata %s: %v"
,
addr
,
err
)
s
.
Logger
.
Debugf
(
"file
download
: read metadata %s: %v"
,
addr
,
err
)
s
.
Logger
.
Error
(
"file: read netadata"
)
s
.
Logger
.
Error
f
(
"file download: read metadata %s"
,
addr
)
jsonhttp
.
InternalServerError
(
w
,
"error reading metadata"
)
jsonhttp
.
InternalServerError
(
w
,
"error reading metadata"
)
return
return
}
}
metaData
:=
&
entry
.
Metadata
{}
metaData
:=
&
entry
.
Metadata
{}
err
=
json
.
Unmarshal
(
buf
.
Bytes
(),
metaData
)
err
=
json
.
Unmarshal
(
buf
.
Bytes
(),
metaData
)
if
err
!=
nil
{
if
err
!=
nil
{
s
.
Logger
.
Debugf
(
"file
: unmarshal
l metadata %s: %v"
,
addr
,
err
)
s
.
Logger
.
Debugf
(
"file
download: unmarsha
l metadata %s: %v"
,
addr
,
err
)
s
.
Logger
.
Error
(
"file: unmarshall metadata"
)
s
.
Logger
.
Error
f
(
"file download: unmarshal metadata %s"
,
addr
)
jsonhttp
.
InternalServerError
(
w
,
"error unmarshal
l
ing metadata"
)
jsonhttp
.
InternalServerError
(
w
,
"error unmarshaling metadata"
)
return
return
}
}
...
@@ -240,13 +246,13 @@ func (s *server) bzzFileDownloadHandler(w http.ResponseWriter, r *http.Request)
...
@@ -240,13 +246,13 @@ func (s *server) bzzFileDownloadHandler(w http.ResponseWriter, r *http.Request)
dataSize
,
err
:=
j
.
Size
(
r
.
Context
(),
address
)
dataSize
,
err
:=
j
.
Size
(
r
.
Context
(),
address
)
if
err
!=
nil
{
if
err
!=
nil
{
if
errors
.
Is
(
err
,
storage
.
ErrNotFound
)
{
if
errors
.
Is
(
err
,
storage
.
ErrNotFound
)
{
s
.
Logger
.
Debugf
(
"file: not found %s: %v"
,
address
,
err
)
s
.
Logger
.
Debugf
(
"file
download
: not found %s: %v"
,
address
,
err
)
s
.
Logger
.
Error
(
"file: not found"
)
s
.
Logger
.
Error
f
(
"file download: not found %s"
,
addr
)
jsonhttp
.
NotFound
(
w
,
"not found"
)
jsonhttp
.
NotFound
(
w
,
"not found"
)
return
return
}
}
s
.
Logger
.
Debugf
(
"file: invalid root chunk %s: %v"
,
address
,
err
)
s
.
Logger
.
Debugf
(
"file
download
: invalid root chunk %s: %v"
,
address
,
err
)
s
.
Logger
.
Error
(
"file: invalid root chunk"
)
s
.
Logger
.
Error
f
(
"file download: invalid root chunk %s"
,
addr
)
jsonhttp
.
BadRequest
(
w
,
"invalid root chunk"
)
jsonhttp
.
BadRequest
(
w
,
"invalid root chunk"
)
return
return
}
}
...
@@ -254,8 +260,8 @@ func (s *server) bzzFileDownloadHandler(w http.ResponseWriter, r *http.Request)
...
@@ -254,8 +260,8 @@ func (s *server) bzzFileDownloadHandler(w http.ResponseWriter, r *http.Request)
outBuffer
:=
bytes
.
NewBuffer
(
nil
)
outBuffer
:=
bytes
.
NewBuffer
(
nil
)
c
,
err
:=
file
.
JoinReadAll
(
j
,
e
.
Reference
(),
outBuffer
)
c
,
err
:=
file
.
JoinReadAll
(
j
,
e
.
Reference
(),
outBuffer
)
if
err
!=
nil
&&
c
==
0
{
if
err
!=
nil
&&
c
==
0
{
s
.
Logger
.
Debugf
(
"file: data read %s: %v"
,
addr
,
err
)
s
.
Logger
.
Debugf
(
"file
download
: data read %s: %v"
,
addr
,
err
)
s
.
Logger
.
Error
(
"file: data read"
)
s
.
Logger
.
Error
f
(
"file download: data read %s"
,
addr
)
jsonhttp
.
InternalServerError
(
w
,
"error reading data"
)
jsonhttp
.
InternalServerError
(
w
,
"error reading data"
)
return
return
}
}
...
@@ -265,25 +271,3 @@ func (s *server) bzzFileDownloadHandler(w http.ResponseWriter, r *http.Request)
...
@@ -265,25 +271,3 @@ func (s *server) bzzFileDownloadHandler(w http.ResponseWriter, r *http.Request)
w
.
Header
()
.
Set
(
"Content-Length"
,
fmt
.
Sprintf
(
"%d"
,
dataSize
))
w
.
Header
()
.
Set
(
"Content-Length"
,
fmt
.
Sprintf
(
"%d"
,
dataSize
))
_
,
_
=
io
.
Copy
(
w
,
outBuffer
)
_
,
_
=
io
.
Copy
(
w
,
outBuffer
)
}
}
// storeMeta is used to store metadata information as a whole.
func
(
s
*
server
)
storeMeta
(
ctx
context
.
Context
,
dataBytes
[]
byte
)
(
swarm
.
Address
,
error
)
{
dataBuf
:=
bytes
.
NewBuffer
(
dataBytes
)
dataReadCloser
:=
ioutil
.
NopCloser
(
dataBuf
)
o
,
err
:=
s
.
splitUpload
(
ctx
,
dataReadCloser
,
int64
(
len
(
dataBytes
)))
if
err
!=
nil
{
return
swarm
.
ZeroAddress
,
err
}
bytesResp
:=
o
.
(
bytesPostResponse
)
return
bytesResp
.
Reference
,
nil
}
// storePartData stores file data belonging to one of the part of multipart.
func
(
s
*
server
)
storePartData
(
ctx
context
.
Context
,
r
io
.
ReadCloser
,
l
uint64
)
(
swarm
.
Address
,
error
)
{
o
,
err
:=
s
.
splitUpload
(
ctx
,
r
,
int64
(
l
))
if
err
!=
nil
{
return
swarm
.
ZeroAddress
,
err
}
bytesResp
:=
o
.
(
bytesPostResponse
)
return
bytesResp
.
Reference
,
nil
}
pkg/api/file_test.go
View file @
53885e61
...
@@ -10,6 +10,7 @@ import (
...
@@ -10,6 +10,7 @@ import (
"io/ioutil"
"io/ioutil"
"mime"
"mime"
"net/http"
"net/http"
"strings"
"testing"
"testing"
"github.com/ethersphere/bee/pkg/api"
"github.com/ethersphere/bee/pkg/api"
...
@@ -21,29 +22,29 @@ import (
...
@@ -21,29 +22,29 @@ import (
"github.com/ethersphere/bee/pkg/tags"
"github.com/ethersphere/bee/pkg/tags"
)
)
func
Test
Bzz
(
t
*
testing
.
T
)
{
func
Test
Files
(
t
*
testing
.
T
)
{
var
(
var
(
simpleResource
=
func
()
string
{
return
"/files"
}
fileUploadResource
=
"/files"
addressResource
=
func
(
addr
string
)
string
{
return
"/files/"
+
addr
}
fileDownloadResource
=
func
(
addr
string
)
string
{
return
"/files/"
+
addr
}
simpleData
=
[]
byte
(
"this is a simple text"
)
simpleData
=
[]
byte
(
"this is a simple text"
)
mockStorer
=
mock
.
NewStorer
()
client
=
newTestServer
(
t
,
testServerOptions
{
client
=
newTestServer
(
t
,
testServerOptions
{
Storer
:
mock
.
NewStorer
(),
Storer
:
mockStorer
,
Tags
:
tags
.
NewTags
(),
Tags
:
tags
.
NewTags
(),
Logger
:
logging
.
New
(
ioutil
.
Discard
,
5
),
Logger
:
logging
.
New
(
ioutil
.
Discard
,
5
),
})
})
)
)
t
.
Run
(
"simple-upload"
,
func
(
t
*
testing
.
T
)
{
jsonhttptest
.
ResponseDirectSendHeadersAndReceiveHeaders
(
t
,
client
,
http
.
MethodPost
,
simpleResource
(),
bytes
.
NewReader
(
simpleData
),
http
.
StatusBadRequest
,
jsonhttp
.
StatusResponse
{
t
.
Run
(
"invalid-content-type"
,
func
(
t
*
testing
.
T
)
{
Message
:
"not a mutlipart/form-data"
,
jsonhttptest
.
ResponseDirectSendHeadersAndReceiveHeaders
(
t
,
client
,
http
.
MethodPost
,
fileUploadResource
,
bytes
.
NewReader
(
simpleData
),
http
.
StatusBadRequest
,
jsonhttp
.
StatusResponse
{
Message
:
"invalid content-type header"
,
Code
:
http
.
StatusBadRequest
,
Code
:
http
.
StatusBadRequest
,
},
nil
)
},
nil
)
})
})
t
.
Run
(
"
simple
-upload"
,
func
(
t
*
testing
.
T
)
{
t
.
Run
(
"
multipart
-upload"
,
func
(
t
*
testing
.
T
)
{
fileName
:=
"simple_file.txt"
fileName
:=
"simple_file.txt"
rootHash
:=
"295673cf7aa55d119dd6f82528c91d45b53dd63dc2e4ca4abf4ed8b3a0788085"
rootHash
:=
"295673cf7aa55d119dd6f82528c91d45b53dd63dc2e4ca4abf4ed8b3a0788085"
_
=
jsonhttptest
.
ResponseDirectWithMultiPart
(
t
,
client
,
http
.
MethodPost
,
simpleResource
()
,
fileName
,
simpleData
,
http
.
StatusOK
,
""
,
api
.
FileUploadResponse
{
_
=
jsonhttptest
.
ResponseDirectWithMultiPart
(
t
,
client
,
http
.
MethodPost
,
fileUploadResource
,
fileName
,
simpleData
,
http
.
StatusOK
,
""
,
api
.
FileUploadResponse
{
Reference
:
swarm
.
MustParseHexAddress
(
rootHash
),
Reference
:
swarm
.
MustParseHexAddress
(
rootHash
),
})
})
})
})
...
@@ -51,22 +52,47 @@ func TestBzz(t *testing.T) {
...
@@ -51,22 +52,47 @@ func TestBzz(t *testing.T) {
t
.
Run
(
"check-content-type-detection"
,
func
(
t
*
testing
.
T
)
{
t
.
Run
(
"check-content-type-detection"
,
func
(
t
*
testing
.
T
)
{
fileName
:=
"my-pictures.jpeg"
fileName
:=
"my-pictures.jpeg"
rootHash
:=
"f2e761160deda91c1fbfab065a5abf530b0766b3e102b51fbd626ba37c3bc581"
rootHash
:=
"f2e761160deda91c1fbfab065a5abf530b0766b3e102b51fbd626ba37c3bc581"
_
=
jsonhttptest
.
ResponseDirectWithMultiPart
(
t
,
client
,
http
.
MethodPost
,
simpleResource
(),
fileName
,
simpleData
,
http
.
StatusOK
,
"image/jpeg; charset=utf-8"
,
api
.
FileUploadResponse
{
Reference
:
swarm
.
MustParseHexAddress
(
rootHash
),
t
.
Run
(
"binary"
,
func
(
t
*
testing
.
T
)
{
headers
:=
make
(
http
.
Header
)
headers
.
Add
(
"Content-Type"
,
"image/jpeg; charset=utf-8"
)
_
=
jsonhttptest
.
ResponseDirectSendHeadersAndReceiveHeaders
(
t
,
client
,
http
.
MethodPost
,
fileUploadResource
+
"?name="
+
fileName
,
bytes
.
NewReader
(
simpleData
),
http
.
StatusOK
,
api
.
FileUploadResponse
{
Reference
:
swarm
.
MustParseHexAddress
(
rootHash
),
},
headers
)
rcvdHeader
:=
jsonhttptest
.
ResponseDirectCheckBinaryResponse
(
t
,
client
,
http
.
MethodGet
,
fileDownloadResource
(
rootHash
),
nil
,
http
.
StatusOK
,
simpleData
,
nil
)
cd
:=
rcvdHeader
.
Get
(
"Content-Disposition"
)
_
,
params
,
err
:=
mime
.
ParseMediaType
(
cd
)
if
err
!=
nil
{
t
.
Fatal
(
err
)
}
if
params
[
"filename"
]
!=
fileName
{
t
.
Fatal
(
"Invalid file name detected"
)
}
if
rcvdHeader
.
Get
(
"Content-Type"
)
!=
"image/jpeg; charset=utf-8"
{
t
.
Fatal
(
"Invalid content type detected"
)
}
})
})
rcvdHeader
:=
jsonhttptest
.
ResponseDirectCheckBinaryResponse
(
t
,
client
,
http
.
MethodGet
,
addressResource
(
rootHash
),
nil
,
http
.
StatusOK
,
simpleData
,
nil
)
t
.
Run
(
"multipart"
,
func
(
t
*
testing
.
T
)
{
cd
:=
rcvdHeader
.
Get
(
"Content-Disposition"
)
_
=
jsonhttptest
.
ResponseDirectWithMultiPart
(
t
,
client
,
http
.
MethodPost
,
fileUploadResource
,
fileName
,
simpleData
,
http
.
StatusOK
,
"image/jpeg; charset=utf-8"
,
api
.
FileUploadResponse
{
_
,
params
,
err
:=
mime
.
ParseMediaType
(
cd
)
Reference
:
swarm
.
MustParseHexAddress
(
rootHash
),
if
err
!=
nil
{
})
t
.
Fatal
(
err
)
}
rcvdHeader
:=
jsonhttptest
.
ResponseDirectCheckBinaryResponse
(
t
,
client
,
http
.
MethodGet
,
fileDownloadResource
(
rootHash
),
nil
,
http
.
StatusOK
,
simpleData
,
nil
)
if
params
[
"filename"
]
!=
fileName
{
cd
:=
rcvdHeader
.
Get
(
"Content-Disposition"
)
t
.
Fatal
(
"Invalid file name detected"
)
_
,
params
,
err
:=
mime
.
ParseMediaType
(
cd
)
}
if
err
!=
nil
{
if
rcvdHeader
.
Get
(
"Content-Type"
)
!=
"image/jpeg; charset=utf-8"
{
t
.
Fatal
(
err
)
t
.
Fatal
(
"Invalid content type detected"
)
}
}
if
params
[
"filename"
]
!=
fileName
{
t
.
Fatal
(
"Invalid file name detected"
)
}
if
rcvdHeader
.
Get
(
"Content-Type"
)
!=
"image/jpeg; charset=utf-8"
{
t
.
Fatal
(
"Invalid content type detected"
)
}
})
})
})
t
.
Run
(
"upload-then-download-and-check-data"
,
func
(
t
*
testing
.
T
)
{
t
.
Run
(
"upload-then-download-and-check-data"
,
func
(
t
*
testing
.
T
)
{
...
@@ -83,28 +109,60 @@ func TestBzz(t *testing.T) {
...
@@ -83,28 +109,60 @@ func TestBzz(t *testing.T) {
</body>
</body>
</html>`
</html>`
rcvdHeader
:=
jsonhttptest
.
ResponseDirectWithMultiPart
(
t
,
client
,
http
.
MethodPost
,
simpleResource
(),
fileName
,
[]
byte
(
sampleHtml
),
http
.
StatusOK
,
""
,
api
.
FileUploadResponse
{
t
.
Run
(
"binary"
,
func
(
t
*
testing
.
T
)
{
Reference
:
swarm
.
MustParseHexAddress
(
rootHash
),
headers
:=
make
(
http
.
Header
)
headers
.
Add
(
"Content-Type"
,
"text/html; charset=utf-8"
)
rcvdHeader
:=
jsonhttptest
.
ResponseDirectSendHeadersAndReceiveHeaders
(
t
,
client
,
http
.
MethodPost
,
fileUploadResource
+
"?name="
+
fileName
,
strings
.
NewReader
(
sampleHtml
),
http
.
StatusOK
,
api
.
FileUploadResponse
{
Reference
:
swarm
.
MustParseHexAddress
(
rootHash
),
},
headers
)
if
rcvdHeader
.
Get
(
"ETag"
)
!=
fmt
.
Sprintf
(
"%q"
,
rootHash
)
{
t
.
Fatal
(
"Invalid ETags header received"
)
}
// try to fetch the same file and check the data
rcvdHeader
=
jsonhttptest
.
ResponseDirectCheckBinaryResponse
(
t
,
client
,
http
.
MethodGet
,
fileDownloadResource
(
rootHash
),
nil
,
http
.
StatusOK
,
[]
byte
(
sampleHtml
),
nil
)
// check the headers
cd
:=
rcvdHeader
.
Get
(
"Content-Disposition"
)
_
,
params
,
err
:=
mime
.
ParseMediaType
(
cd
)
if
err
!=
nil
{
t
.
Fatal
(
err
)
}
if
params
[
"filename"
]
!=
fileName
{
t
.
Fatal
(
"Invalid filename detected"
)
}
if
rcvdHeader
.
Get
(
"Content-Type"
)
!=
"text/html; charset=utf-8"
{
t
.
Fatal
(
"Invalid content type detected"
)
}
})
})
if
rcvdHeader
.
Get
(
"ETag"
)
!=
fmt
.
Sprintf
(
"%q"
,
rootHash
)
{
t
.
Run
(
"multipart"
,
func
(
t
*
testing
.
T
)
{
t
.
Fatal
(
"Invalid ETags header received"
)
rcvdHeader
:=
jsonhttptest
.
ResponseDirectWithMultiPart
(
t
,
client
,
http
.
MethodPost
,
fileUploadResource
,
fileName
,
[]
byte
(
sampleHtml
),
http
.
StatusOK
,
""
,
api
.
FileUploadResponse
{
}
Reference
:
swarm
.
MustParseHexAddress
(
rootHash
),
})
// try to fetch the same file and check the data
rcvdHeader
=
jsonhttptest
.
ResponseDirectCheckBinaryResponse
(
t
,
client
,
http
.
MethodGet
,
addressResource
(
rootHash
),
nil
,
http
.
StatusOK
,
[]
byte
(
sampleHtml
),
nil
)
if
rcvdHeader
.
Get
(
"ETag"
)
!=
fmt
.
Sprintf
(
"%q"
,
rootHash
)
{
t
.
Fatal
(
"Invalid ETags header received"
)
// check the headers
}
cd
:=
rcvdHeader
.
Get
(
"Content-Disposition"
)
_
,
params
,
err
:=
mime
.
ParseMediaType
(
cd
)
// try to fetch the same file and check the data
if
err
!=
nil
{
rcvdHeader
=
jsonhttptest
.
ResponseDirectCheckBinaryResponse
(
t
,
client
,
http
.
MethodGet
,
fileDownloadResource
(
rootHash
),
nil
,
http
.
StatusOK
,
[]
byte
(
sampleHtml
),
nil
)
t
.
Fatal
(
err
)
}
// check the headers
if
params
[
"filename"
]
!=
fileName
{
cd
:=
rcvdHeader
.
Get
(
"Content-Disposition"
)
t
.
Fatal
(
"Invalid filename detected"
)
_
,
params
,
err
:=
mime
.
ParseMediaType
(
cd
)
}
if
err
!=
nil
{
if
rcvdHeader
.
Get
(
"Content-Type"
)
!=
"text/html; charset=utf-8"
{
t
.
Fatal
(
err
)
t
.
Fatal
(
"Invalid content type detected"
)
}
}
if
params
[
"filename"
]
!=
fileName
{
t
.
Fatal
(
"Invalid filename detected"
)
}
if
rcvdHeader
.
Get
(
"Content-Type"
)
!=
"text/html; charset=utf-8"
{
t
.
Fatal
(
"Invalid content type detected"
)
}
})
})
})
}
}
pkg/api/router.go
View file @
53885e61
...
@@ -35,18 +35,16 @@ func (s *server) setupRouting() {
...
@@ -35,18 +35,16 @@ func (s *server) setupRouting() {
fmt
.
Fprintln
(
w
,
"User-agent: *
\n
Disallow: /"
)
fmt
.
Fprintln
(
w
,
"User-agent: *
\n
Disallow: /"
)
})
})
handle
(
router
,
"/bytes"
,
jsonhttp
.
MethodHandler
{
"POST"
:
http
.
HandlerFunc
(
s
.
bytesUploadHandler
),
})
handle
(
router
,
"/files"
,
jsonhttp
.
MethodHandler
{
handle
(
router
,
"/files"
,
jsonhttp
.
MethodHandler
{
"POST"
:
http
.
HandlerFunc
(
s
.
bzzF
ileUploadHandler
),
"POST"
:
http
.
HandlerFunc
(
s
.
f
ileUploadHandler
),
})
})
handle
(
router
,
"/files/{addr}"
,
jsonhttp
.
MethodHandler
{
handle
(
router
,
"/files/{addr}"
,
jsonhttp
.
MethodHandler
{
"GET"
:
http
.
HandlerFunc
(
s
.
bzzF
ileDownloadHandler
),
"GET"
:
http
.
HandlerFunc
(
s
.
f
ileDownloadHandler
),
})
})
handle
(
router
,
"/bytes"
,
jsonhttp
.
MethodHandler
{
"POST"
:
http
.
HandlerFunc
(
s
.
bytesUploadHandler
),
})
handle
(
router
,
"/bytes/{address}"
,
jsonhttp
.
MethodHandler
{
handle
(
router
,
"/bytes/{address}"
,
jsonhttp
.
MethodHandler
{
"GET"
:
http
.
HandlerFunc
(
s
.
bytesGetHandler
),
"GET"
:
http
.
HandlerFunc
(
s
.
bytesGetHandler
),
})
})
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment