api: upload directory (#447)

add endpoint for uploading directories as tarballs

pkg/api/dirs.go

+// Copyright 2020 The Swarm Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package api
+
+import (
+	"archive/tar"
+	"bytes"
+	"context"
+	"errors"
+	"fmt"
+	"io"
+	"mime"
+	"net/http"
+	"path/filepath"
+	"strings"
+
+	"github.com/ethersphere/bee/pkg/jsonhttp"
+	"github.com/ethersphere/bee/pkg/logging"
+	"github.com/ethersphere/bee/pkg/manifest/jsonmanifest"
+	"github.com/ethersphere/bee/pkg/storage"
+	"github.com/ethersphere/bee/pkg/swarm"
+)
+
+const (
+	contentTypeHeader = "Content-Type"
+	contentTypeTar    = "application/x-tar"
+)
+
+type toEncryptContextKey struct{}
+
+// dirUploadHandler uploads a directory supplied as a tar in an HTTP request
+func (s *server) dirUploadHandler(w http.ResponseWriter, r *http.Request) {
+	ctx, err := validateRequest(r)
+	if err != nil {
+		s.Logger.Errorf("dir upload, validate request")
+		s.Logger.Debugf("dir upload, validate request err: %v", err)
+		jsonhttp.BadRequest(w, "could not validate request")
+		return
+	}
+
+	reference, err := storeDir(ctx, r.Body, s.Storer, s.Logger)
+	if err != nil {
+		s.Logger.Errorf("dir upload, store dir")
+		s.Logger.Debugf("dir upload, store dir err: %v", err)
+		jsonhttp.InternalServerError(w, "could not store dir")
+		return
+	}
+
+	jsonhttp.OK(w, fileUploadResponse{
+		Reference: reference,
+	})
+}
+
+// validateRequest validates an HTTP request for a directory to be uploaded
+// it returns a context based on the given request
+func validateRequest(r *http.Request) (context.Context, error) {
+	ctx := r.Context()
+	if r.Body == http.NoBody {
+		return nil, errors.New("request has no body")
+	}
+	contentType := r.Header.Get(contentTypeHeader)
+	mediaType, _, err := mime.ParseMediaType(contentType)
+	if err != nil {
+		return nil, err
+	}
+	if mediaType != contentTypeTar {
+		return nil, errors.New("content-type not set to tar")
+	}
+	toEncrypt := strings.ToLower(r.Header.Get(EncryptHeader)) == "true"
+	return context.WithValue(ctx, toEncryptContextKey{}, toEncrypt), nil
+}
+
+// storeDir stores all files recursively contained in the directory given as a tar
+// it returns the hash for the uploaded manifest corresponding to the uploaded dir
+func storeDir(ctx context.Context, reader io.ReadCloser, s storage.Storer, logger logging.Logger) (swarm.Address, error) {
+	dirManifest := jsonmanifest.NewManifest()
+
+	// set up HTTP body reader
+	tarReader := tar.NewReader(reader)
+	defer reader.Close()
+
+	// iterate through the files in the supplied tar
+	for {
+		fileHeader, err := tarReader.Next()
+		if err == io.EOF {
+			break
+		} else if err != nil {
+			return swarm.ZeroAddress, fmt.Errorf("read tar stream error: %w", err)
+		}
+
+		filePath := fileHeader.Name
+
+		// only store regular files
+		if !fileHeader.FileInfo().Mode().IsRegular() {
+			logger.Warningf("skipping file upload for %s as it is not a regular file", filePath)
+			continue
+		}
+
+		fileName := fileHeader.FileInfo().Name()
+		contentType := mime.TypeByExtension(filepath.Ext(fileHeader.Name))
+
+		// upload file
+		fileInfo := &fileUploadInfo{
+			name:        fileName,
+			size:        fileHeader.FileInfo().Size(),
+			contentType: contentType,
+			reader:      tarReader,
+		}
+		fileReference, err := storeFile(ctx, fileInfo, s)
+		if err != nil {
+			return swarm.ZeroAddress, fmt.Errorf("store dir file error: %w", err)
+		}
+		logger.Tracef("uploaded dir file %v with reference %v", filePath, fileReference)
+
+		// create manifest entry for uploaded file
+		headers := http.Header{}
+		headers.Set("Content-Type", contentType)
+		fileEntry := &jsonmanifest.JSONEntry{
+			Reference: fileReference,
+			Name:      fileName,
+			Headers:   headers,
+		}
+
+		// add entry to dir manifest
+		dirManifest.Add(filePath, fileEntry)
+	}
+
+	// check if files were uploaded through the manifest
+	if len(dirManifest.Entries) == 0 {
+		return swarm.ZeroAddress, fmt.Errorf("no files in tar")
+	}
+
+	// upload manifest
+	// first, serialize into byte array
+	b, err := dirManifest.Serialize()
+	if err != nil {
+		return swarm.ZeroAddress, fmt.Errorf("manifest serialize error: %w", err)
+	}
+
+	// set up reader for manifest file upload
+	r := bytes.NewReader(b)
+
+	// then, upload manifest
+	manifestFileInfo := &fileUploadInfo{
+		size:        r.Size(),
+		contentType: ManifestContentType,
+		reader:      r,
+	}
+	manifestReference, err := storeFile(ctx, manifestFileInfo, s)
+	if err != nil {
+		return swarm.ZeroAddress, fmt.Errorf("store manifest error: %w", err)
+	}
+
+	return manifestReference, nil
+}

pkg/api/dirs_test.go

+// Copyright 2020 The Swarm Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package api_test
+
+import (
+	"archive/tar"
+	"bytes"
+	"io/ioutil"
+	"net/http"
+	"path"
+	"testing"
+
+	"github.com/ethersphere/bee/pkg/api"
+	"github.com/ethersphere/bee/pkg/jsonhttp"
+	"github.com/ethersphere/bee/pkg/jsonhttp/jsonhttptest"
+	"github.com/ethersphere/bee/pkg/logging"
+	"github.com/ethersphere/bee/pkg/manifest/jsonmanifest"
+	"github.com/ethersphere/bee/pkg/storage/mock"
+	"github.com/ethersphere/bee/pkg/swarm"
+	"github.com/ethersphere/bee/pkg/tags"
+)
+
+func TestDirs(t *testing.T) {
+	var (
+		dirUploadResource    = "/dirs"
+		fileDownloadResource = func(addr string) string { return "/files/" + addr }
+		client               = newTestServer(t, testServerOptions{
+			Storer: mock.NewStorer(),
+			Tags:   tags.NewTags(),
+			Logger: logging.New(ioutil.Discard, 5),
+		})
+	)
+
+	t.Run("empty request body", func(t *testing.T) {
+		jsonhttptest.ResponseDirectSendHeadersAndReceiveHeaders(t, client, http.MethodPost, dirUploadResource, bytes.NewReader(nil), http.StatusBadRequest, jsonhttp.StatusResponse{
+			Message: "could not validate request",
+			Code:    http.StatusBadRequest,
+		}, http.Header{
+			"Content-Type": {api.ContentTypeTar},
+		})
+	})
+
+	t.Run("non tar file", func(t *testing.T) {
+		file := bytes.NewReader([]byte("some data"))
+
+		jsonhttptest.ResponseDirectSendHeadersAndReceiveHeaders(t, client, http.MethodPost, dirUploadResource, file, http.StatusInternalServerError, jsonhttp.StatusResponse{
+			Message: "could not store dir",
+			Code:    http.StatusInternalServerError,
+		}, http.Header{
+			"Content-Type": {api.ContentTypeTar},
+		})
+	})
+
+	t.Run("wrong content type", func(t *testing.T) {
+		tarReader := tarFiles(t, []f{{
+			data: []byte("some data"),
+			name: "binary-file",
+		}})
+
+		// submit valid tar, but with wrong content-type
+		jsonhttptest.ResponseDirectSendHeadersAndReceiveHeaders(t, client, http.MethodPost, dirUploadResource, tarReader, http.StatusBadRequest, jsonhttp.StatusResponse{
+			Message: "could not validate request",
+			Code:    http.StatusBadRequest,
+		}, http.Header{
+			"Content-Type": {"other"},
+		})
+	})
+
+	// valid tars
+	for _, tc := range []struct {
+		name         string
+		expectedHash string
+		files        []f // files in dir for test case
+	}{
+		{
+			name:         "non-nested files without extension",
+			expectedHash: "2fa041bd35ebff676727eb3023272f43b1e0fa71c8735cc1a7487e9131f963c4",
+			files: []f{
+				{
+					data:      []byte("first file data"),
+					name:      "file1",
+					dir:       "",
+					reference: swarm.MustParseHexAddress("3c07cd2cf5c46208d69d554b038f4dce203f53ac02cb8a313a0fe1e3fe6cc3cf"),
+					headers: http.Header{
+						"Content-Type": {""},
+					},
+				},
+				{
+					data:      []byte("second file data"),
+					name:      "file2",
+					dir:       "",
+					reference: swarm.MustParseHexAddress("47e1a2a8f16e02da187fac791d57e6794f3e9b5d2400edd00235da749ad36683"),
+					headers: http.Header{
+						"Content-Type": {""},
+					},
+				},
+			},
+		},
+		{
+			name:         "nested files with extension",
+			expectedHash: "c3cb9fbe2efa7bbc979245d9bac1400bd4894371776b7560309d49e687514dd6",
+			files: []f{
+				{
+					data:      []byte("robots text"),
+					name:      "robots.txt",
+					dir:       "",
+					reference: swarm.MustParseHexAddress("17b96d0a800edca59aaf7e40c6053f7c4c0fb80dd2eb3f8663d51876bf350b12"),
+					headers: http.Header{
+						"Content-Type": {"text/plain; charset=utf-8"},
+					},
+				},
+				{
+					data:      []byte("image 1"),
+					name:      "1.png",
+					dir:       "img",
+					reference: swarm.MustParseHexAddress("3c1b3fc640e67f0595d9c1db23f10c7a2b0bdc9843b0e27c53e2ac2a2d6c4674"),
+					headers: http.Header{
+						"Content-Type": {"image/png"},
+					},
+				},
+				{
+					data:      []byte("image 2"),
+					name:      "2.png",
+					dir:       "img",
+					reference: swarm.MustParseHexAddress("b234ea7954cab7b2ccc5e07fe8487e932df11b2275db6b55afcbb7bad0be73fb"),
+					headers: http.Header{
+						"Content-Type": {"image/png"},
+					},
+				},
+			},
+		},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			// tar all the test case files
+			tarReader := tarFiles(t, tc.files)
+
+			// verify directory tar upload response
+			jsonhttptest.ResponseDirectSendHeadersAndReceiveHeaders(t, client, http.MethodPost, dirUploadResource, tarReader, http.StatusOK, api.FileUploadResponse{
+				Reference: swarm.MustParseHexAddress(tc.expectedHash),
+			}, http.Header{
+				"Content-Type": {api.ContentTypeTar},
+			})
+
+			// create expected manifest
+			expectedManifest := jsonmanifest.NewManifest()
+			for _, file := range tc.files {
+				e := &jsonmanifest.JSONEntry{
+					Reference: file.reference,
+					Name:      file.name,
+					Headers:   file.headers,
+				}
+				expectedManifest.Add(path.Join(file.dir, file.name), e)
+			}
+
+			b, err := expectedManifest.Serialize()
+			if err != nil {
+				t.Fatal(err)
+			}
+
+			// verify directory upload manifest through files api
+			jsonhttptest.ResponseDirectCheckBinaryResponse(t, client, http.MethodGet, fileDownloadResource(tc.expectedHash), nil, http.StatusOK, b, nil)
+		})
+	}
+}
+
+// tarFiles receives an array of test case files and creates a new tar with those files as a collection
+// it returns a bytes.Buffer which can be used to read the created tar
+func tarFiles(t *testing.T, files []f) *bytes.Buffer {
+	var buf bytes.Buffer
+	tw := tar.NewWriter(&buf)
+
+	for _, file := range files {
+		// create tar header and write it
+		hdr := &tar.Header{
+			Name: path.Join(file.dir, file.name),
+			Mode: 0600,
+			Size: int64(len(file.data)),
+		}
+		if err := tw.WriteHeader(hdr); err != nil {
+			t.Fatal(err)
+		}
+
+		// write the file data to the tar
+		if _, err := tw.Write(file.data); err != nil {
+			t.Fatal(err)
+		}
+	}
+
+	// finally close the tar writer
+	if err := tw.Close(); err != nil {
+		t.Fatal(err)
+	}
+
+	return &buf
+}
+
+// struct for dir files for test cases
+type f struct {
+	data      []byte
+	name      string
+	dir       string
+	reference swarm.Address
+	headers   http.Header
+}

pkg/api/export_test.go

@@ -8,3 +8,7 @@ type (
 	BytesPostResponse  = bytesPostResponse
 	FileUploadResponse = fileUploadResponse
 )
+
+var (
+	ContentTypeTar = contentTypeTar
+)

pkg/api/file.go

@@ -42,6 +42,7 @@ const (
 
 type targetsContextKey struct{}
 
+// fileUploadResponse is returned when an HTTP request to upload a file is successful
 type fileUploadResponse struct {
 	Reference swarm.Address `json:"reference"`
 }
@@ -201,6 +202,61 @@ func (s *server) fileUploadHandler(w http.ResponseWriter, r *http.Request) {
 	})
 }
 
+// fileUploadInfo contains the data for a file to be uploaded
+type fileUploadInfo struct {
+	name        string // file name
+	size        int64  // file size
+	contentType string
+	reader      io.Reader
+}
+
+// storeFile uploads the given file and returns its reference
+// this function was extracted from `fileUploadHandler` and should eventually replace its current code
+func storeFile(ctx context.Context, fileInfo *fileUploadInfo, s storage.Storer) (swarm.Address, error) {
+	v := ctx.Value(toEncryptContextKey{})
+	toEncrypt, _ := v.(bool) // default is false
+
+	// first store the file and get its reference
+	sp := splitter.NewSimpleSplitter(s)
+	fr, err := file.SplitWriteAll(ctx, sp, fileInfo.reader, fileInfo.size, toEncrypt)
+	if err != nil {
+		return swarm.ZeroAddress, fmt.Errorf("split file error: %w", err)
+	}
+
+	// if filename is still empty, use the file hash as the filename
+	if fileInfo.name == "" {
+		fileInfo.name = fr.String()
+	}
+
+	// then store the metadata and get its reference
+	m := entry.NewMetadata(fileInfo.name)
+	m.MimeType = fileInfo.contentType
+	metadataBytes, err := json.Marshal(m)
+	if err != nil {
+		return swarm.ZeroAddress, fmt.Errorf("metadata marshal error: %w", err)
+	}
+
+	sp = splitter.NewSimpleSplitter(s)
+	mr, err := file.SplitWriteAll(ctx, sp, bytes.NewReader(metadataBytes), int64(len(metadataBytes)), toEncrypt)
+	if err != nil {
+		return swarm.ZeroAddress, fmt.Errorf("split metadata error: %w", err)
+	}
+
+	// now join both references (mr, fr) to create an entry and store it
+	e := entry.New(fr, mr)
+	fileEntryBytes, err := e.MarshalBinary()
+	if err != nil {
+		return swarm.ZeroAddress, fmt.Errorf("entry marshal error: %w", err)
+	}
+	sp = splitter.NewSimpleSplitter(s)
+	reference, err := file.SplitWriteAll(ctx, sp, bytes.NewReader(fileEntryBytes), int64(len(fileEntryBytes)), toEncrypt)
+	if err != nil {
+		return swarm.ZeroAddress, fmt.Errorf("split entry error: %w", err)
+	}
+
+	return reference, nil
+}
+
 // fileDownloadHandler downloads the file given the entry's reference.
 func (s *server) fileDownloadHandler(w http.ResponseWriter, r *http.Request) {
 	addr := mux.Vars(r)["addr"]

pkg/api/router.go

@@ -42,6 +42,10 @@ func (s *server) setupRouting() {
 		"GET": http.HandlerFunc(s.fileDownloadHandler),
 	})
 
+	handle(router, "/dirs", jsonhttp.MethodHandler{
+		"POST": http.HandlerFunc(s.dirUploadHandler),
+	})
+
 	handle(router, "/bytes", jsonhttp.MethodHandler{
 		"POST": http.HandlerFunc(s.bytesUploadHandler),
 	})