Commit 3fa63f4c authored by mortelli's avatar mortelli Committed by GitHub

api: upload directory (#447)

add endpoint for uploading directories as tarballs
parent 62066269
// Copyright 2020 The Swarm Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package api
import (
"archive/tar"
"bytes"
"context"
"errors"
"fmt"
"io"
"mime"
"net/http"
"path/filepath"
"strings"
"github.com/ethersphere/bee/pkg/jsonhttp"
"github.com/ethersphere/bee/pkg/logging"
"github.com/ethersphere/bee/pkg/manifest/jsonmanifest"
"github.com/ethersphere/bee/pkg/storage"
"github.com/ethersphere/bee/pkg/swarm"
)
const (
contentTypeHeader = "Content-Type"
contentTypeTar = "application/x-tar"
)
type toEncryptContextKey struct{}
// dirUploadHandler uploads a directory supplied as a tar in an HTTP request
func (s *server) dirUploadHandler(w http.ResponseWriter, r *http.Request) {
ctx, err := validateRequest(r)
if err != nil {
s.Logger.Errorf("dir upload, validate request")
s.Logger.Debugf("dir upload, validate request err: %v", err)
jsonhttp.BadRequest(w, "could not validate request")
return
}
reference, err := storeDir(ctx, r.Body, s.Storer, s.Logger)
if err != nil {
s.Logger.Errorf("dir upload, store dir")
s.Logger.Debugf("dir upload, store dir err: %v", err)
jsonhttp.InternalServerError(w, "could not store dir")
return
}
jsonhttp.OK(w, fileUploadResponse{
Reference: reference,
})
}
// validateRequest validates an HTTP request for a directory to be uploaded
// it returns a context based on the given request
func validateRequest(r *http.Request) (context.Context, error) {
ctx := r.Context()
if r.Body == http.NoBody {
return nil, errors.New("request has no body")
}
contentType := r.Header.Get(contentTypeHeader)
mediaType, _, err := mime.ParseMediaType(contentType)
if err != nil {
return nil, err
}
if mediaType != contentTypeTar {
return nil, errors.New("content-type not set to tar")
}
toEncrypt := strings.ToLower(r.Header.Get(EncryptHeader)) == "true"
return context.WithValue(ctx, toEncryptContextKey{}, toEncrypt), nil
}
// storeDir stores all files recursively contained in the directory given as a tar
// it returns the hash for the uploaded manifest corresponding to the uploaded dir
func storeDir(ctx context.Context, reader io.ReadCloser, s storage.Storer, logger logging.Logger) (swarm.Address, error) {
dirManifest := jsonmanifest.NewManifest()
// set up HTTP body reader
tarReader := tar.NewReader(reader)
defer reader.Close()
// iterate through the files in the supplied tar
for {
fileHeader, err := tarReader.Next()
if err == io.EOF {
break
} else if err != nil {
return swarm.ZeroAddress, fmt.Errorf("read tar stream error: %w", err)
}
filePath := fileHeader.Name
// only store regular files
if !fileHeader.FileInfo().Mode().IsRegular() {
logger.Warningf("skipping file upload for %s as it is not a regular file", filePath)
continue
}
fileName := fileHeader.FileInfo().Name()
contentType := mime.TypeByExtension(filepath.Ext(fileHeader.Name))
// upload file
fileInfo := &fileUploadInfo{
name: fileName,
size: fileHeader.FileInfo().Size(),
contentType: contentType,
reader: tarReader,
}
fileReference, err := storeFile(ctx, fileInfo, s)
if err != nil {
return swarm.ZeroAddress, fmt.Errorf("store dir file error: %w", err)
}
logger.Tracef("uploaded dir file %v with reference %v", filePath, fileReference)
// create manifest entry for uploaded file
headers := http.Header{}
headers.Set("Content-Type", contentType)
fileEntry := &jsonmanifest.JSONEntry{
Reference: fileReference,
Name: fileName,
Headers: headers,
}
// add entry to dir manifest
dirManifest.Add(filePath, fileEntry)
}
// check if files were uploaded through the manifest
if len(dirManifest.Entries) == 0 {
return swarm.ZeroAddress, fmt.Errorf("no files in tar")
}
// upload manifest
// first, serialize into byte array
b, err := dirManifest.Serialize()
if err != nil {
return swarm.ZeroAddress, fmt.Errorf("manifest serialize error: %w", err)
}
// set up reader for manifest file upload
r := bytes.NewReader(b)
// then, upload manifest
manifestFileInfo := &fileUploadInfo{
size: r.Size(),
contentType: ManifestContentType,
reader: r,
}
manifestReference, err := storeFile(ctx, manifestFileInfo, s)
if err != nil {
return swarm.ZeroAddress, fmt.Errorf("store manifest error: %w", err)
}
return manifestReference, nil
}
// Copyright 2020 The Swarm Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package api_test
import (
"archive/tar"
"bytes"
"io/ioutil"
"net/http"
"path"
"testing"
"github.com/ethersphere/bee/pkg/api"
"github.com/ethersphere/bee/pkg/jsonhttp"
"github.com/ethersphere/bee/pkg/jsonhttp/jsonhttptest"
"github.com/ethersphere/bee/pkg/logging"
"github.com/ethersphere/bee/pkg/manifest/jsonmanifest"
"github.com/ethersphere/bee/pkg/storage/mock"
"github.com/ethersphere/bee/pkg/swarm"
"github.com/ethersphere/bee/pkg/tags"
)
func TestDirs(t *testing.T) {
var (
dirUploadResource = "/dirs"
fileDownloadResource = func(addr string) string { return "/files/" + addr }
client = newTestServer(t, testServerOptions{
Storer: mock.NewStorer(),
Tags: tags.NewTags(),
Logger: logging.New(ioutil.Discard, 5),
})
)
t.Run("empty request body", func(t *testing.T) {
jsonhttptest.ResponseDirectSendHeadersAndReceiveHeaders(t, client, http.MethodPost, dirUploadResource, bytes.NewReader(nil), http.StatusBadRequest, jsonhttp.StatusResponse{
Message: "could not validate request",
Code: http.StatusBadRequest,
}, http.Header{
"Content-Type": {api.ContentTypeTar},
})
})
t.Run("non tar file", func(t *testing.T) {
file := bytes.NewReader([]byte("some data"))
jsonhttptest.ResponseDirectSendHeadersAndReceiveHeaders(t, client, http.MethodPost, dirUploadResource, file, http.StatusInternalServerError, jsonhttp.StatusResponse{
Message: "could not store dir",
Code: http.StatusInternalServerError,
}, http.Header{
"Content-Type": {api.ContentTypeTar},
})
})
t.Run("wrong content type", func(t *testing.T) {
tarReader := tarFiles(t, []f{{
data: []byte("some data"),
name: "binary-file",
}})
// submit valid tar, but with wrong content-type
jsonhttptest.ResponseDirectSendHeadersAndReceiveHeaders(t, client, http.MethodPost, dirUploadResource, tarReader, http.StatusBadRequest, jsonhttp.StatusResponse{
Message: "could not validate request",
Code: http.StatusBadRequest,
}, http.Header{
"Content-Type": {"other"},
})
})
// valid tars
for _, tc := range []struct {
name string
expectedHash string
files []f // files in dir for test case
}{
{
name: "non-nested files without extension",
expectedHash: "2fa041bd35ebff676727eb3023272f43b1e0fa71c8735cc1a7487e9131f963c4",
files: []f{
{
data: []byte("first file data"),
name: "file1",
dir: "",
reference: swarm.MustParseHexAddress("3c07cd2cf5c46208d69d554b038f4dce203f53ac02cb8a313a0fe1e3fe6cc3cf"),
headers: http.Header{
"Content-Type": {""},
},
},
{
data: []byte("second file data"),
name: "file2",
dir: "",
reference: swarm.MustParseHexAddress("47e1a2a8f16e02da187fac791d57e6794f3e9b5d2400edd00235da749ad36683"),
headers: http.Header{
"Content-Type": {""},
},
},
},
},
{
name: "nested files with extension",
expectedHash: "c3cb9fbe2efa7bbc979245d9bac1400bd4894371776b7560309d49e687514dd6",
files: []f{
{
data: []byte("robots text"),
name: "robots.txt",
dir: "",
reference: swarm.MustParseHexAddress("17b96d0a800edca59aaf7e40c6053f7c4c0fb80dd2eb3f8663d51876bf350b12"),
headers: http.Header{
"Content-Type": {"text/plain; charset=utf-8"},
},
},
{
data: []byte("image 1"),
name: "1.png",
dir: "img",
reference: swarm.MustParseHexAddress("3c1b3fc640e67f0595d9c1db23f10c7a2b0bdc9843b0e27c53e2ac2a2d6c4674"),
headers: http.Header{
"Content-Type": {"image/png"},
},
},
{
data: []byte("image 2"),
name: "2.png",
dir: "img",
reference: swarm.MustParseHexAddress("b234ea7954cab7b2ccc5e07fe8487e932df11b2275db6b55afcbb7bad0be73fb"),
headers: http.Header{
"Content-Type": {"image/png"},
},
},
},
},
} {
t.Run(tc.name, func(t *testing.T) {
// tar all the test case files
tarReader := tarFiles(t, tc.files)
// verify directory tar upload response
jsonhttptest.ResponseDirectSendHeadersAndReceiveHeaders(t, client, http.MethodPost, dirUploadResource, tarReader, http.StatusOK, api.FileUploadResponse{
Reference: swarm.MustParseHexAddress(tc.expectedHash),
}, http.Header{
"Content-Type": {api.ContentTypeTar},
})
// create expected manifest
expectedManifest := jsonmanifest.NewManifest()
for _, file := range tc.files {
e := &jsonmanifest.JSONEntry{
Reference: file.reference,
Name: file.name,
Headers: file.headers,
}
expectedManifest.Add(path.Join(file.dir, file.name), e)
}
b, err := expectedManifest.Serialize()
if err != nil {
t.Fatal(err)
}
// verify directory upload manifest through files api
jsonhttptest.ResponseDirectCheckBinaryResponse(t, client, http.MethodGet, fileDownloadResource(tc.expectedHash), nil, http.StatusOK, b, nil)
})
}
}
// tarFiles receives an array of test case files and creates a new tar with those files as a collection
// it returns a bytes.Buffer which can be used to read the created tar
func tarFiles(t *testing.T, files []f) *bytes.Buffer {
var buf bytes.Buffer
tw := tar.NewWriter(&buf)
for _, file := range files {
// create tar header and write it
hdr := &tar.Header{
Name: path.Join(file.dir, file.name),
Mode: 0600,
Size: int64(len(file.data)),
}
if err := tw.WriteHeader(hdr); err != nil {
t.Fatal(err)
}
// write the file data to the tar
if _, err := tw.Write(file.data); err != nil {
t.Fatal(err)
}
}
// finally close the tar writer
if err := tw.Close(); err != nil {
t.Fatal(err)
}
return &buf
}
// struct for dir files for test cases
type f struct {
data []byte
name string
dir string
reference swarm.Address
headers http.Header
}
......@@ -8,3 +8,7 @@ type (
BytesPostResponse = bytesPostResponse
FileUploadResponse = fileUploadResponse
)
var (
ContentTypeTar = contentTypeTar
)
......@@ -42,6 +42,7 @@ const (
type targetsContextKey struct{}
// fileUploadResponse is returned when an HTTP request to upload a file is successful
type fileUploadResponse struct {
Reference swarm.Address `json:"reference"`
}
......@@ -201,6 +202,61 @@ func (s *server) fileUploadHandler(w http.ResponseWriter, r *http.Request) {
})
}
// fileUploadInfo contains the data for a file to be uploaded
type fileUploadInfo struct {
name string // file name
size int64 // file size
contentType string
reader io.Reader
}
// storeFile uploads the given file and returns its reference
// this function was extracted from `fileUploadHandler` and should eventually replace its current code
func storeFile(ctx context.Context, fileInfo *fileUploadInfo, s storage.Storer) (swarm.Address, error) {
v := ctx.Value(toEncryptContextKey{})
toEncrypt, _ := v.(bool) // default is false
// first store the file and get its reference
sp := splitter.NewSimpleSplitter(s)
fr, err := file.SplitWriteAll(ctx, sp, fileInfo.reader, fileInfo.size, toEncrypt)
if err != nil {
return swarm.ZeroAddress, fmt.Errorf("split file error: %w", err)
}
// if filename is still empty, use the file hash as the filename
if fileInfo.name == "" {
fileInfo.name = fr.String()
}
// then store the metadata and get its reference
m := entry.NewMetadata(fileInfo.name)
m.MimeType = fileInfo.contentType
metadataBytes, err := json.Marshal(m)
if err != nil {
return swarm.ZeroAddress, fmt.Errorf("metadata marshal error: %w", err)
}
sp = splitter.NewSimpleSplitter(s)
mr, err := file.SplitWriteAll(ctx, sp, bytes.NewReader(metadataBytes), int64(len(metadataBytes)), toEncrypt)
if err != nil {
return swarm.ZeroAddress, fmt.Errorf("split metadata error: %w", err)
}
// now join both references (mr, fr) to create an entry and store it
e := entry.New(fr, mr)
fileEntryBytes, err := e.MarshalBinary()
if err != nil {
return swarm.ZeroAddress, fmt.Errorf("entry marshal error: %w", err)
}
sp = splitter.NewSimpleSplitter(s)
reference, err := file.SplitWriteAll(ctx, sp, bytes.NewReader(fileEntryBytes), int64(len(fileEntryBytes)), toEncrypt)
if err != nil {
return swarm.ZeroAddress, fmt.Errorf("split entry error: %w", err)
}
return reference, nil
}
// fileDownloadHandler downloads the file given the entry's reference.
func (s *server) fileDownloadHandler(w http.ResponseWriter, r *http.Request) {
addr := mux.Vars(r)["addr"]
......
......@@ -42,6 +42,10 @@ func (s *server) setupRouting() {
"GET": http.HandlerFunc(s.fileDownloadHandler),
})
handle(router, "/dirs", jsonhttp.MethodHandler{
"POST": http.HandlerFunc(s.dirUploadHandler),
})
handle(router, "/bytes", jsonhttp.MethodHandler{
"POST": http.HandlerFunc(s.bytesUploadHandler),
})
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment