Removing collections package and API cleanup (#1501)

cmd/bee-file/main.go

@@ -5,31 +5,13 @@
 package main
 
 import (
-	"bytes"
-	"context"
-	"encoding/json"
+	"errors"
 	"fmt"
-	"io"
-	"io/ioutil"
 	"os"
-	"path/filepath"
 
-	cmdfile "github.com/ethersphere/bee/cmd/internal/file"
-	"github.com/ethersphere/bee/pkg/collection/entry"
-	"github.com/ethersphere/bee/pkg/file"
-	"github.com/ethersphere/bee/pkg/file/joiner"
-	"github.com/ethersphere/bee/pkg/file/splitter"
-	"github.com/ethersphere/bee/pkg/logging"
-	"github.com/ethersphere/bee/pkg/storage"
-	"github.com/ethersphere/bee/pkg/swarm"
 	"github.com/spf13/cobra"
 )
 
-const (
-	defaultMimeType     = "application/octet-stream"
-	limitMetadataLength = swarm.ChunkSize
-)
-
 var (
 	filename     string // flag variable, filename to use in metadata
 	mimeType     string // flag variable, mime type to use in metadata
@@ -41,186 +23,11 @@ var (
 	ssl          bool   // flag variable, uses https for api if set
 	retrieve     bool   // flag variable, if set will resolve and retrieve referenced file
 	verbosity    string // flag variable, debug level
-	logger       logging.Logger
 )
 
-// getEntry handles retrieving and writing a file from the file entry
-// referenced by the given address.
-func getEntry(cmd *cobra.Command, args []string) (err error) {
-	// process the reference to retrieve
-	addr, err := swarm.ParseHexAddress(args[0])
-	if err != nil {
-		return err
-	}
-
-	// initialize interface with HTTP API
-	store := cmdfile.NewApiStore(host, port, ssl)
-
-	buf := bytes.NewBuffer(nil)
-	writeCloser := cmdfile.NopWriteCloser(buf)
-	limitBuf := cmdfile.NewLimitWriteCloser(writeCloser, limitMetadataLength)
-	j, _, err := joiner.New(cmd.Context(), store, addr)
-	if err != nil {
-		return err
-	}
-
-	_, err = file.JoinReadAll(cmd.Context(), j, limitBuf)
-	if err != nil {
-		return err
-	}
-	e := &entry.Entry{}
-	err = e.UnmarshalBinary(buf.Bytes())
-	if err != nil {
-		return err
-	}
-
-	j, _, err = joiner.New(cmd.Context(), store, e.Metadata())
-	if err != nil {
-		return err
-	}
-
-	buf = bytes.NewBuffer(nil)
-
-	_, err = file.JoinReadAll(cmd.Context(), j, buf)
-	if err != nil {
-		return err
-	}
-
-	// retrieve metadata
-	metaData := &entry.Metadata{}
-	err = json.Unmarshal(buf.Bytes(), metaData)
-	if err != nil {
-		return err
-	}
-	logger.Debugf("Filename: %s", metaData.Filename)
-	logger.Debugf("MIME-type: %s", metaData.MimeType)
-
-	if outDir == "" {
-		outDir = "."
-	} else {
-		err := os.MkdirAll(outDir, 0o777) // skipcq: GSC-G301
-		if err != nil {
-			return err
-		}
-	}
-	outFilePath := filepath.Join(outDir, metaData.Filename)
-
-	// create output dir if not exist
-	if outDir != "." {
-		err := os.MkdirAll(outDir, 0o777) // skipcq: GSC-G301
-		if err != nil {
-			return err
-		}
-	}
-
-	// protect any existing file unless explicitly told not to
-	outFileFlags := os.O_CREATE | os.O_WRONLY
-	if outFileForce {
-		outFileFlags |= os.O_TRUNC
-	} else {
-		outFileFlags |= os.O_EXCL
-	}
-
-	// open the file
-	outFile, err := os.OpenFile(outFilePath, outFileFlags, 0o666) // skipcq: GSC-G302
-	if err != nil {
-		return err
-	}
-	defer outFile.Close()
-
-	j, _, err = joiner.New(cmd.Context(), store, e.Reference())
-	if err != nil {
-		return err
-	}
-
-	_, err = file.JoinReadAll(cmd.Context(), j, outFile)
-	return err
-}
-
-// putEntry creates a new file entry with the given reference.
-func putEntry(cmd *cobra.Command, args []string) (err error) {
-	// process the reference to retrieve
-	addr, err := swarm.ParseHexAddress(args[0])
-	if err != nil {
-		return err
-	}
-	// add the fsStore and/or apiStore, depending on flags
-	stores := cmdfile.NewTeeStore()
-	if outDir != "" {
-		err := os.MkdirAll(outDir, 0o777) // skipcq: GSC-G301
-		if err != nil {
-			return err
-		}
-		store := cmdfile.NewFsStore(outDir)
-		stores.Add(store)
-	}
-	if useHttp {
-		store := cmdfile.NewApiStore(host, port, ssl)
-		stores.Add(store)
-	}
-
-	// create metadata object, with defaults for missing values
-	if filename == "" {
-		filename = args[0]
-	}
-	if mimeType == "" {
-		mimeType = defaultMimeType
-	}
-	metadata := entry.NewMetadata(filename)
-	metadata.MimeType = mimeType
-
-	// serialize metadata and send it to splitter
-	metadataBytes, err := json.Marshal(metadata)
-	if err != nil {
-		return err
-	}
-	logger.Debugf("metadata contents: %s", metadataBytes)
-
-	// set up splitter to process the metadata
-	s := splitter.NewSimpleSplitter(stores, storage.ModePutUpload)
-	ctx := context.Background()
-
-	// first add metadata
-	metadataBuf := bytes.NewBuffer(metadataBytes)
-	metadataReader := io.LimitReader(metadataBuf, int64(len(metadataBytes)))
-	metadataReadCloser := ioutil.NopCloser(metadataReader)
-	metadataAddr, err := s.Split(ctx, metadataReadCloser, int64(len(metadataBytes)), false)
-	if err != nil {
-		return err
-	}
-
-	// create entry from given reference and metadata,
-	// serialize and send to splitter
-	fileEntry := entry.New(addr, metadataAddr)
-	fileEntryBytes, err := fileEntry.MarshalBinary()
-	if err != nil {
-		return err
-	}
-	fileEntryBuf := bytes.NewBuffer(fileEntryBytes)
-	fileEntryReader := io.LimitReader(fileEntryBuf, int64(len(fileEntryBytes)))
-	fileEntryReadCloser := ioutil.NopCloser(fileEntryReader)
-	fileEntryAddr, err := s.Split(ctx, fileEntryReadCloser, int64(len(fileEntryBytes)), false)
-	if err != nil {
-		return err
-	}
-
-	// output reference to file entry
-	cmd.Println(fileEntryAddr)
-	return nil
-}
-
 // Entry is the underlying procedure for the CLI command
 func Entry(cmd *cobra.Command, args []string) (err error) {
-	logger, err = cmdfile.SetLogger(cmd, verbosity)
-	if err != nil {
-		fmt.Fprintln(os.Stderr, err.Error())
-		os.Exit(1)
-	}
-
-	if retrieve {
-		return getEntry(cmd, args)
-	}
-	return putEntry(cmd, args)
+	return errors.New("command is deprecated")
 }
 
 func main() {

openapi/Swarm.yaml

@@ -154,22 +154,31 @@ paths:
         default:
           description: Default response
 
-  "/files":
+  "/bzz":
     post:
-      summary: "Upload file"
+      summary: "Upload file or a collection of files"
+      description: "In order to upload a collection, user can send a multipart request with all the files populated in the form data with appropriate headers.\n\n
+        User can also upload a tar file along with the swarm-collection header. This will upload the tar file after extracting the entire directory structure.\n\n
+        If the swarm-collection header is absent, all requests (including tar files) are considered as single file uploads.\n\n
+        A multipart request is treated as a collection regardless of whether the swarm-collection header is present. This means in order to serve single files
+        uploaded as a multipart request, the swarm-index-document header should be used with the name of the file."
       tags:
         - File
+        - Collection
       parameters:
         - in: query
           name: name
           schema:
             $ref: "SwarmCommon.yaml#/components/schemas/FileName"
           required: false
-          description: Filename
+          description: Filename when uploading single file
         - $ref: "SwarmCommon.yaml#/components/parameters/SwarmTagParameter"
         - $ref: "SwarmCommon.yaml#/components/parameters/SwarmPinParameter"
         - $ref: "SwarmCommon.yaml#/components/parameters/SwarmEncryptParameter"
         - $ref: "SwarmCommon.yaml#/components/parameters/ContentTypePreserved"
+        - $ref: "SwarmCommon.yaml#/components/parameters/SwarmCollection"
+        - $ref: "SwarmCommon.yaml#/components/parameters/SwarmIndexDocumentParameter"
+        - $ref: "SwarmCommon.yaml#/components/parameters/SwarmErrorDocumentParameter"
       requestBody:
         content:
           multipart/form-data:
@@ -184,76 +193,6 @@ paths:
             schema:
               type: string
               format: binary
-      responses:
-        "200":
-          description: Ok
-          headers:
-            "swarm-tag":
-              $ref: "SwarmCommon.yaml#/components/headers/SwarmTag"
-            "etag":
-              $ref: "SwarmCommon.yaml#/components/headers/ETag" 
-          content:
-            application/json:
-              schema:
-                $ref: "SwarmCommon.yaml#/components/schemas/ReferenceResponse"
-        "400":
-          $ref: "SwarmCommon.yaml#/components/responses/400"
-        "403":
-          $ref: "SwarmCommon.yaml#/components/responses/403"
-        "500":
-          $ref: "SwarmCommon.yaml#/components/responses/500"
-        default:
-          description: Default response
-
-  "/files/{reference}":
-    get:
-      summary: "Get referenced file"
-      tags:
-        - File
-      parameters:
-        - in: path
-          name: reference
-          schema:
-            $ref: "SwarmCommon.yaml#/components/schemas/SwarmReference"
-          required: true
-          description: Swarm address of content
-        - $ref: "SwarmCommon.yaml#/components/parameters/SwarmRecoveryTargetsParameter"
-      responses:
-        "200":
-          description: Ok
-          headers:
-            "swarm-recovery-targets":
-              $ref: "SwarmCommon.yaml#/components/headers/SwarmRecoveryTargets"
-            "ETag":
-              $ref: "SwarmCommon.yaml#/components/headers/ETag"
-          content:
-            application/octet-stream:
-              schema:
-                type: string
-                format: binary
-        "400":
-          $ref: "SwarmCommon.yaml#/components/responses/400"
-        "404":
-          $ref: "SwarmCommon.yaml#/components/responses/404"
-        "500":
-          $ref: "SwarmCommon.yaml#/components/responses/500"
-        default:
-          description: Default response
-
-  "/dirs":
-    post:
-      summary: "Upload a collection"
-      tags:
-        - Collection
-      parameters:
-        - $ref: "SwarmCommon.yaml#/components/parameters/SwarmTagParameter"
-        - $ref: "SwarmCommon.yaml#/components/parameters/SwarmPinParameter"
-        - $ref: "SwarmCommon.yaml#/components/parameters/SwarmEncryptParameter"
-        - $ref: "SwarmCommon.yaml#/components/parameters/SwarmIndexDocumentParameter"
-        - $ref: "SwarmCommon.yaml#/components/parameters/SwarmErrorDocumentParameter"
-        - $ref: "SwarmCommon.yaml#/components/parameters/ContentTypePreserved"
-      requestBody:
-        content:
           application/x-tar:
             schema:
               type: string
@@ -264,6 +203,8 @@ paths:
           headers:
             "swarm-tag":
               $ref: "SwarmCommon.yaml#/components/headers/SwarmTag"
+            "etag":
+              $ref: "SwarmCommon.yaml#/components/headers/ETag" 
           content:
             application/json:
               schema:
@@ -279,7 +220,7 @@ paths:
 
   "/bzz/{reference}":
     get:
-      summary: "Get index document from a collection of files"
+      summary: "Get file or index document from a collection of files"
       tags:
         - Collection
       parameters:

openapi/SwarmCommon.yaml

@@ -493,6 +493,14 @@ components:
       required: false
       description: Configure custom error document to be returned when a specified path can not be found in collection
 
+    SwarmCollection:
+      in: header
+      name: swarm-collection
+      schema:
+        type: boolean
+      required: false
+      description: Upload file/files as a collection
+
   responses:
     "204":
       description: The resource was deleted successfully.

pkg/api/api.go

@@ -40,6 +40,7 @@ const (
 	SwarmErrorDocumentHeader = "Swarm-Error-Document"
 	SwarmFeedIndexHeader     = "Swarm-Feed-Index"
 	SwarmFeedIndexNextHeader = "Swarm-Feed-Index-Next"
+	SwarmCollectionHeader    = "Swarm-Collection"
 )
 
 // The size of buffer used for prefetching content with Langos.
@@ -54,9 +55,20 @@ const (
 	largeBufferFilesizeThreshold = 10 * 1000000 // ten megs
 )
 
+const (
+	contentTypeHeader = "Content-Type"
+	multiPartFormData = "multipart/form-data"
+	contentTypeTar    = "application/x-tar"
+)
+
 var (
 	errInvalidNameOrAddress = errors.New("invalid name or bzz address")
 	errNoResolver           = errors.New("no resolver connected")
+	invalidRequest          = errors.New("could not validate request")
+	invalidContentType      = errors.New("invalid content-type")
+	invalidContentLength    = errors.New("invalid content-length")
+	directoryStoreError     = errors.New("could not store directory")
+	fileStoreError          = errors.New("could not store file")
 )
 
 // Service is the API service interface.

pkg/api/bytes_test.go

@@ -74,11 +74,20 @@ func TestBytes(t *testing.T) {
 	})
 
 	t.Run("not found", func(t *testing.T) {
-		jsonhttptest.Request(t, client, http.MethodGet, resource+"/abcd", http.StatusNotFound,
+		jsonhttptest.Request(t, client, http.MethodGet, resource+"/0xabcd", http.StatusNotFound,
 			jsonhttptest.WithExpectedJSONResponse(jsonhttp.StatusResponse{
 				Message: "Not Found",
 				Code:    http.StatusNotFound,
 			}),
 		)
 	})
+
+	t.Run("internal error", func(t *testing.T) {
+		jsonhttptest.Request(t, client, http.MethodGet, resource+"/abcd", http.StatusInternalServerError,
+			jsonhttptest.WithExpectedJSONResponse(jsonhttp.StatusResponse{
+				Message: "Internal Server Error",
+				Code:    http.StatusInternalServerError,
+			}),
+		)
+	})
 }

pkg/api/export_test.go

@@ -13,7 +13,7 @@ type (
 	ChunkAddressResponse     = chunkAddressResponse
 	SocPostResponse          = socPostResponse
 	FeedReferenceResponse    = feedReferenceResponse
-	FileUploadResponse       = fileUploadResponse
+	BzzUploadResponse        = bzzUploadResponse
 	TagResponse              = tagResponse
 	TagRequest               = tagRequest
 	ListTagsResponse         = listTagsResponse
@@ -23,13 +23,13 @@ type (
 )
 
 var (
-	ContentTypeTar = contentTypeTar
+	InvalidContentType  = invalidContentType
+	InvalidRequest      = invalidRequest
+	DirectoryStoreError = directoryStoreError
 )
 
 var (
-	ManifestRootPath                      = manifestRootPath
-	ManifestWebsiteIndexDocumentSuffixKey = manifestWebsiteIndexDocumentSuffixKey
-	ManifestWebsiteErrorDocumentPathKey   = manifestWebsiteErrorDocumentPathKey
+	ContentTypeTar = contentTypeTar
 )
 
 var (

pkg/api/pin_bytes_test.go

@@ -47,7 +47,7 @@ func TestPinBytesHandler(t *testing.T) {
 
 		jsonhttptest.Request(t, client, http.MethodPost, bytesUploadResource, http.StatusOK,
 			jsonhttptest.WithRequestBody(bytes.NewReader(simpleData)),
-			jsonhttptest.WithExpectedJSONResponse(api.FileUploadResponse{
+			jsonhttptest.WithExpectedJSONResponse(api.BzzUploadResponse{
 				Reference: swarm.MustParseHexAddress(rootHash),
 			}),
 		)
@@ -110,7 +110,7 @@ func TestPinBytesHandler(t *testing.T) {
 
 		jsonhttptest.Request(t, client, http.MethodPost, bytesUploadResource, http.StatusOK,
 			jsonhttptest.WithRequestBody(bytes.NewReader(b)),
-			jsonhttptest.WithExpectedJSONResponse(api.FileUploadResponse{
+			jsonhttptest.WithExpectedJSONResponse(api.BzzUploadResponse{
 				Reference: swarm.MustParseHexAddress(rootHash),
 			}),
 		)

pkg/api/pin_bzz_test.go

@@ -22,7 +22,7 @@ import (
 
 func TestPinBzzHandler(t *testing.T) {
 	var (
-		dirUploadResource     = "/dirs"
+		dirUploadResource     = "/bzz"
 		pinBzzResource        = "/pin/bzz"
 		pinBzzAddressResource = func(addr string) string { return pinBzzResource + "/" + addr }
 		pinChunksResource     = "/pin/chunks"
@@ -35,6 +35,7 @@ func TestPinBzzHandler(t *testing.T) {
 			Storer:    mockStorer,
 			Traversal: traversalService,
 			Tags:      tags.NewTags(mockStatestore, logger),
+			Logger:    logger,
 		})
 	)
 
@@ -49,13 +50,14 @@ func TestPinBzzHandler(t *testing.T) {
 
 		tarReader := tarFiles(t, files)
 
-		rootHash := "a85aaea6a34a5c7127a3546196f2111f866fe369c6d6562ed5d3313a99388c03"
+		rootHash := "9e178dbd1ed4b748379e25144e28dfb29c07a4b5114896ef454480115a56b237"
 
 		// verify directory tar upload response
 		jsonhttptest.Request(t, client, http.MethodPost, dirUploadResource, http.StatusOK,
 			jsonhttptest.WithRequestBody(tarReader),
 			jsonhttptest.WithRequestHeader("Content-Type", api.ContentTypeTar),
-			jsonhttptest.WithExpectedJSONResponse(api.FileUploadResponse{
+			jsonhttptest.WithRequestHeader(api.SwarmCollectionHeader, "True"),
+			jsonhttptest.WithExpectedJSONResponse(api.BzzUploadResponse{
 				Reference: swarm.MustParseHexAddress(rootHash),
 			}),
 		)
@@ -67,7 +69,7 @@ func TestPinBzzHandler(t *testing.T) {
 			}),
 		)
 
-		expectedChunkCount := 7
+		expectedChunkCount := 3
 
 		// get the reference as everytime it will change because of random encryption key
 		var resp api.ListPinnedChunksResponse
@@ -82,7 +84,7 @@ func TestPinBzzHandler(t *testing.T) {
 	})
 
 	t.Run("unpin-bzz-1", func(t *testing.T) {
-		rootHash := "a85aaea6a34a5c7127a3546196f2111f866fe369c6d6562ed5d3313a99388c03"
+		rootHash := "9e178dbd1ed4b748379e25144e28dfb29c07a4b5114896ef454480115a56b237"
 
 		jsonhttptest.Request(t, client, http.MethodDelete, pinBzzAddressResource(rootHash), http.StatusOK,
 			jsonhttptest.WithExpectedJSONResponse(jsonhttp.StatusResponse{

pkg/api/pin_files.go

@@ -48,7 +48,7 @@ func (s *server) pinFile(w http.ResponseWriter, r *http.Request) {
 
 	chunkAddressFn := s.pinChunkAddressFn(ctx, addr)
 
-	err = s.traversal.TraverseFileAddresses(ctx, addr, chunkAddressFn)
+	err = s.traversal.TraverseAddresses(ctx, addr, chunkAddressFn)
 	if err != nil {
 		s.logger.Debugf("pin files: traverse chunks: %v, addr %s", err, addr)
 
@@ -93,7 +93,7 @@ func (s *server) unpinFile(w http.ResponseWriter, r *http.Request) {
 
 	chunkAddressFn := s.unpinChunkAddressFn(ctx, addr)
 
-	err = s.traversal.TraverseFileAddresses(ctx, addr, chunkAddressFn)
+	err = s.traversal.TraverseAddresses(ctx, addr, chunkAddressFn)
 	if err != nil {
 		s.logger.Debugf("pin files: traverse chunks: %v, addr %s", err, addr)
 

pkg/api/pin_files_test.go

@@ -8,7 +8,6 @@ import (
 	"bytes"
 	"io/ioutil"
 	"net/http"
-	"sort"
 	"testing"
 
 	"github.com/ethersphere/bee/pkg/api"
@@ -24,7 +23,7 @@ import (
 
 func TestPinFilesHandler(t *testing.T) {
 	var (
-		fileUploadResource      = "/files"
+		fileUploadResource      = "/bzz"
 		pinFilesResource        = "/pin/files"
 		pinFilesAddressResource = func(addr string) string { return pinFilesResource + "/" + addr }
 		pinChunksResource       = "/pin/chunks"
@@ -39,17 +38,20 @@ func TestPinFilesHandler(t *testing.T) {
 			Storer:    mockStorer,
 			Traversal: traversalService,
 			Tags:      tags.NewTags(mockStatestore, logger),
+			Logger:    logger,
 		})
 	)
 
 	t.Run("pin-file-1", func(t *testing.T) {
-		rootHash := "dc82503e0ed041a57327ad558d7aa69a867024c8221306c461ae359dc34d1c6a"
-		metadataHash := "d936d7180f230b3424842ea10848aa205f2f0e830cb9cc7588a39c9381544bf9"
+		rootHash := "dd13a5a6cc9db3ef514d645e6719178dbfb1a90b49b9262cafce35b0d27cf245"
+		metadataHash := "0cc878d32c96126d47f63fbe391114ee1438cd521146fc975dea1546d302b6c0"
+		metadataHash2 := "a14d1ef845307c634e9ec74539bd668d0d1b37f37de4128939d57098135850da"
 		contentHash := "838d0a193ecd1152d1bb1432d5ecc02398533b2494889e23b8bd5ace30ac2aeb"
 
-		jsonhttptest.Request(t, client, http.MethodPost, fileUploadResource, http.StatusOK,
+		jsonhttptest.Request(t, client, http.MethodPost,
+			fileUploadResource+"?name=somefile.txt", http.StatusOK,
 			jsonhttptest.WithRequestBody(bytes.NewReader(simpleData)),
-			jsonhttptest.WithExpectedJSONResponse(api.FileUploadResponse{
+			jsonhttptest.WithExpectedJSONResponse(api.BzzUploadResponse{
 				Reference: swarm.MustParseHexAddress(rootHash),
 			}),
 			jsonhttptest.WithRequestHeader("Content-Type", "text/plain"),
@@ -62,27 +64,36 @@ func TestPinFilesHandler(t *testing.T) {
 			}),
 		)
 
-		hashes := []string{rootHash, metadataHash, contentHash}
-		sort.Strings(hashes)
-
-		expectedResponse := api.ListPinnedChunksResponse{
-			Chunks: []api.PinnedChunk{},
+		hashes := map[string]int{
+			rootHash:      1,
+			metadataHash:  1,
+			metadataHash2: 1,
+			contentHash:   1,
 		}
 
-		for _, h := range hashes {
-			expectedResponse.Chunks = append(expectedResponse.Chunks, api.PinnedChunk{
-				Address:    swarm.MustParseHexAddress(h),
-				PinCounter: 1,
-			})
+		actualResponse := api.ListPinnedChunksResponse{
+			Chunks: []api.PinnedChunk{},
 		}
 
 		jsonhttptest.Request(t, client, http.MethodGet, pinChunksResource, http.StatusOK,
-			jsonhttptest.WithExpectedJSONResponse(expectedResponse),
+			jsonhttptest.WithUnmarshalJSONResponse(&actualResponse),
 		)
+		if len(actualResponse.Chunks) != len(hashes) {
+			t.Fatalf("Response chunk count mismatch Expected: %d Found: %d",
+				len(hashes), len(actualResponse.Chunks))
+		}
+		for _, v := range actualResponse.Chunks {
+			if counter, ok := hashes[v.Address.String()]; !ok {
+				t.Fatalf("found unexpected hash %s", v.Address.String())
+			} else if uint64(counter) != v.PinCounter {
+				t.Fatalf("found unexpected pin counter: Expected: %d, Found: %d",
+					counter, v.PinCounter)
+			}
+		}
 	})
 
 	t.Run("unpin-file-1", func(t *testing.T) {
-		rootHash := "dc82503e0ed041a57327ad558d7aa69a867024c8221306c461ae359dc34d1c6a"
+		rootHash := "dd13a5a6cc9db3ef514d645e6719178dbfb1a90b49b9262cafce35b0d27cf245"
 
 		jsonhttptest.Request(t, client, http.MethodDelete, pinFilesAddressResource(rootHash), http.StatusOK,
 			jsonhttptest.WithExpectedJSONResponse(jsonhttp.StatusResponse{

pkg/api/router.go

@@ -38,26 +38,6 @@ func (s *server) setupRouting() {
 		fmt.Fprintln(w, "User-agent: *\nDisallow: /")
 	})
 
-	handle(router, "/files", jsonhttp.MethodHandler{
-		"POST": web.ChainHandlers(
-			s.newTracingHandler("files-upload"),
-			web.FinalHandlerFunc(s.fileUploadHandler),
-		),
-	})
-	handle(router, "/files/{addr}", jsonhttp.MethodHandler{
-		"GET": web.ChainHandlers(
-			s.newTracingHandler("files-download"),
-			web.FinalHandlerFunc(s.fileDownloadHandler),
-		),
-	})
-
-	handle(router, "/dirs", jsonhttp.MethodHandler{
-		"POST": web.ChainHandlers(
-			s.newTracingHandler("dirs-upload"),
-			web.FinalHandlerFunc(s.dirUploadHandler),
-		),
-	})
-
 	handle(router, "/bytes", jsonhttp.MethodHandler{
 		"POST": web.ChainHandlers(
 			s.newTracingHandler("bytes-upload"),
@@ -97,6 +77,12 @@ func (s *server) setupRouting() {
 		),
 	})
 
+	handle(router, "/bzz", jsonhttp.MethodHandler{
+		"POST": web.ChainHandlers(
+			s.newTracingHandler("bzz-upload"),
+			web.FinalHandlerFunc(s.bzzUploadHandler),
+		),
+	})
 	handle(router, "/bzz/{address}", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		u := r.URL
 		u.Path += "/"

pkg/api/tag_test.go

@@ -35,8 +35,7 @@ func tagsWithIdResource(id uint32) string { return fmt.Sprintf("/tags/%d", id) }
 
 func TestTags(t *testing.T) {
 	var (
-		filesResource  = "/files"
-		dirResource    = "/dirs"
+		bzzResource    = "/bzz"
 		bytesResource  = "/bytes"
 		chunksResource = "/chunks"
 		tagsResource   = "/tags"
@@ -47,6 +46,7 @@ func TestTags(t *testing.T) {
 		client, _, _   = newTestServer(t, testServerOptions{
 			Storer: mock.NewStorer(),
 			Tags:   tag,
+			Logger: logger,
 		})
 	)
 
@@ -263,10 +263,11 @@ func TestTags(t *testing.T) {
 
 	t.Run("file tags", func(t *testing.T) {
 		// upload a file without supplying tag
-		expectedHash := swarm.MustParseHexAddress("8e27bb803ff049e8c2f4650357026723220170c15ebf9b635a7026539879a1a8")
-		expectedResponse := api.FileUploadResponse{Reference: expectedHash}
+		expectedHash := swarm.MustParseHexAddress("40e739ebdfd18292925bba4138cd097db9aa18c1b57e74042f48469b48da33a8")
+		expectedResponse := api.BzzUploadResponse{Reference: expectedHash}
 
-		respHeaders := jsonhttptest.Request(t, client, http.MethodPost, filesResource, http.StatusOK,
+		respHeaders := jsonhttptest.Request(t, client, http.MethodPost,
+			bzzResource+"?name=somefile", http.StatusOK,
 			jsonhttptest.WithRequestBody(bytes.NewReader([]byte("some data"))),
 			jsonhttptest.WithExpectedJSONResponse(expectedResponse),
 			jsonhttptest.WithRequestHeader("Content-Type", "application/octet-stream"),
@@ -276,7 +277,7 @@ func TestTags(t *testing.T) {
 		if err != nil {
 			t.Fatal(err)
 		}
-		tagValueTest(t, uint32(tagId), 3, 3, 0, 0, 0, 3, expectedHash, client)
+		tagValueTest(t, uint32(tagId), 4, 4, 0, 0, 0, 4, expectedHash, client)
 	})
 
 	t.Run("dir tags", func(t *testing.T) {
@@ -285,11 +286,12 @@ func TestTags(t *testing.T) {
 			data: []byte("some dir data"),
 			name: "binary-file",
 		}})
-		expectedHash := swarm.MustParseHexAddress("3dc643abeb3db60a4dfb72008b577dd9a573abaa74c6afe37a75c63ceea829f6")
-		expectedResponse := api.FileUploadResponse{Reference: expectedHash}
+		expectedHash := swarm.MustParseHexAddress("42bc27c9137c93705ffbc2945fa1aab0e8e1826f1500b7f06f6e3f86f617213b")
+		expectedResponse := api.BzzUploadResponse{Reference: expectedHash}
 
-		respHeaders := jsonhttptest.Request(t, client, http.MethodPost, dirResource, http.StatusOK,
+		respHeaders := jsonhttptest.Request(t, client, http.MethodPost, bzzResource, http.StatusOK,
 			jsonhttptest.WithRequestBody(tarReader),
+			jsonhttptest.WithRequestHeader(api.SwarmCollectionHeader, "True"),
 			jsonhttptest.WithExpectedJSONResponse(expectedResponse),
 			jsonhttptest.WithRequestHeader("Content-Type", api.ContentTypeTar),
 		)
@@ -298,7 +300,7 @@ func TestTags(t *testing.T) {
 		if err != nil {
 			t.Fatal(err)
 		}
-		tagValueTest(t, uint32(tagId), 7, 7, 0, 0, 0, 7, expectedHash, client)
+		tagValueTest(t, uint32(tagId), 3, 3, 0, 0, 0, 3, expectedHash, client)
 	})
 
 	t.Run("bytes tags", func(t *testing.T) {

pkg/collection/collection.go

-// Copyright 2020 The Swarm Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-// Package collection provides high-level abstractions for collections of files
-package collection
-
-import (
-	"github.com/ethersphere/bee/pkg/swarm"
-)
-
-// Collection provides a specific ordering of a collection of binary data vectors
-// stored in bee.
-type Collection interface {
-	Addresses() []swarm.Address
-}
-
-// Entry encapsulates data defining a single file entry.
-// It may contain any number of data blobs providing context to the
-// given data vector concealed by Reference.
-type Entry interface {
-	Reference() swarm.Address
-	Metadata() swarm.Address
-}

pkg/collection/entry/entry.go

-// Copyright 2020 The Swarm Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-package entry
-
-import (
-	"errors"
-	"math"
-
-	"github.com/ethersphere/bee/pkg/collection"
-	"github.com/ethersphere/bee/pkg/encryption"
-	"github.com/ethersphere/bee/pkg/swarm"
-)
-
-var (
-	_                           = collection.Entry(&Entry{})
-	serializedDataSize          = swarm.SectionSize * 2
-	encryptedSerializedDataSize = encryption.ReferenceSize * 2
-)
-
-// Entry provides addition of metadata to a data reference.
-// Implements collection.Entry.
-type Entry struct {
-	reference swarm.Address
-	metadata  swarm.Address
-}
-
-// New creates a new Entry.
-func New(reference, metadata swarm.Address) *Entry {
-	return &Entry{
-		reference: reference,
-		metadata:  metadata,
-	}
-}
-
-// CanUnmarshal returns whether the entry may be might be unmarshaled based on
-// the size.
-func CanUnmarshal(size int64) bool {
-	if size < math.MaxInt32 {
-		switch int(size) {
-		case serializedDataSize, encryptedSerializedDataSize:
-			return true
-		}
-	}
-	return false
-}
-
-// Reference implements collection.Entry
-func (e *Entry) Reference() swarm.Address {
-	return e.reference
-}
-
-// Metadata implements collection.Entry
-func (e *Entry) Metadata() swarm.Address {
-	return e.metadata
-}
-
-// MarshalBinary implements encoding.BinaryMarshaler
-func (e *Entry) MarshalBinary() ([]byte, error) {
-	br := e.reference.Bytes()
-	bm := e.metadata.Bytes()
-	b := append(br, bm...)
-	return b, nil
-}
-
-// UnmarshalBinary implements encoding.BinaryUnmarshaler
-func (e *Entry) UnmarshalBinary(b []byte) error {
-	var size int
-	if len(b) == serializedDataSize {
-		size = serializedDataSize
-	} else if len(b) == encryptedSerializedDataSize {
-		size = encryptedSerializedDataSize
-	} else {
-		return errors.New("invalid data length")
-	}
-	e.reference = swarm.NewAddress(b[:size/2])
-	e.metadata = swarm.NewAddress(b[size/2:])
-	return nil
-}

pkg/collection/entry/entry_test.go

-// Copyright 2020 The Swarm Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-package entry_test
-
-import (
-	"testing"
-
-	"github.com/ethersphere/bee/pkg/collection/entry"
-	"github.com/ethersphere/bee/pkg/swarm/test"
-)
-
-// TestEntrySerialize verifies integrity of serialization.
-func TestEntrySerialize(t *testing.T) {
-	referenceAddress := test.RandomAddress()
-	metadataAddress := test.RandomAddress()
-	e := entry.New(referenceAddress, metadataAddress)
-	entrySerialized, err := e.MarshalBinary()
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	entryRecovered := &entry.Entry{}
-	err = entryRecovered.UnmarshalBinary(entrySerialized)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	if !referenceAddress.Equal(entryRecovered.Reference()) {
-		t.Fatalf("expected reference %s, got %s", referenceAddress, entryRecovered.Reference())
-	}
-
-	metadataAddressRecovered := entryRecovered.Metadata()
-	if !metadataAddress.Equal(metadataAddressRecovered) {
-		t.Fatalf("expected metadata %s, got %s", metadataAddress, metadataAddressRecovered)
-	}
-}

pkg/collection/entry/metadata.go

-// Copyright 2020 The Swarm Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-package entry
-
-import (
-	"encoding/json"
-)
-
-// Metadata provides mime type and filename to file entry.
-type Metadata struct {
-	MimeType string `json:"mimetype"`
-	Filename string `json:"filename"`
-}
-
-// NewMetadata creates a new Metadata.
-func NewMetadata(fileName string) *Metadata {
-	return &Metadata{
-		Filename: fileName,
-	}
-}
-
-func (m *Metadata) String() string {
-	j, _ := json.Marshal(m)
-	return string(j)
-}

pkg/file/loadsave/loadsave.go

@@ -11,17 +11,22 @@ import (
 	"github.com/ethersphere/bee/pkg/swarm"
 )
 
+type PutGetter interface {
+	storage.Putter
+	storage.Getter
+}
+
 // loadSave is needed for manifest operations and provides
 // simple wrapping over load and save operations using file
 // package abstractions. use with caution since Loader will
 // load all of the subtrie of a given hash in memory.
 type loadSave struct {
-	storer    storage.Storer
+	storer    PutGetter
 	mode      storage.ModePut
 	encrypted bool
 }
 
-func New(storer storage.Storer, mode storage.ModePut, enc bool) file.LoadSaver {
+func New(storer PutGetter, mode storage.ModePut, enc bool) file.LoadSaver {
 	return &loadSave{
 		storer:    storer,
 		mode:      mode,

pkg/manifest/manifest.go

@@ -16,6 +16,14 @@ import (
 
 const DefaultManifestType = ManifestMantarayContentType
 
+const (
+	RootPath                      = "/"
+	WebsiteIndexDocumentSuffixKey = "website-index-document"
+	WebsiteErrorDocumentPathKey   = "website-error-document"
+	EntryMetadataContentTypeKey   = "Content-Type"
+	EntryMetadataFilenameKey      = "Filename"
+)
+
 var (
 	// ErrNotFound is returned when an Entry is not found in the manifest.
 	ErrNotFound = errors.New("manifest: not found")
@@ -68,6 +76,14 @@ func NewDefaultManifest(
 	return NewManifest(DefaultManifestType, ls, encrypted)
 }
 
+// NewDefaultManifest creates a new manifest with default type.
+func NewDefaultManifestReference(
+	reference swarm.Address,
+	ls file.LoadSaver,
+) (Interface, error) {
+	return NewManifestReference(DefaultManifestType, reference, ls)
+}
+
 // NewManifest creates a new manifest.
 func NewManifest(
 	manifestType string,

pkg/manifest/mantaray.go

@@ -149,7 +149,7 @@ func (m *mantarayManifest) IterateAddresses(ctx context.Context, fn swarm.Addres
 				}
 			}
 
-			if node.IsValueType() && node.Entry() != nil {
+			if node.IsValueType() && len(node.Entry()) > 0 {
 				entry := swarm.NewAddress(node.Entry())
 				err = fn(entry)
 				if err != nil {

pkg/manifest/mantaray/marshal.go

@@ -270,6 +270,13 @@ func (n *Node) UnmarshalBinary(data []byte) error {
 
 		n.entry = append([]byte{}, data[nodeHeaderSize:nodeHeaderSize+refBytesSize]...)
 		offset := nodeHeaderSize + refBytesSize // skip entry
+		// Currently we don't persist the root nodeType when we marshal the manifest, as a result
+		// the root nodeType information is lost on Unmarshal. This causes issues when we want to
+		// perform a path 'Walk' on the root. If there is more than 1 fork, the root node type
+		// is an edge, so we will deduce this information from index byte array
+		if !bytes.Equal(data[offset:offset+32], make([]byte, 32)) {
+			n.nodeType = nodeTypeEdge
+		}
 		n.forks = make(map[byte]*fork)
 		bb := &bitsForBytes{}
 		bb.fromBytes(data[offset:])

pkg/traversal/traversal.go

@@ -9,14 +9,10 @@
 package traversal
 
 import (
-	"bytes"
 	"context"
-	"encoding/json"
 	"errors"
 	"fmt"
 
-	"github.com/ethersphere/bee/pkg/collection/entry"
-	"github.com/ethersphere/bee/pkg/file"
 	"github.com/ethersphere/bee/pkg/file/joiner"
 	"github.com/ethersphere/bee/pkg/file/loadsave"
 	"github.com/ethersphere/bee/pkg/manifest"
@@ -37,8 +33,6 @@ type Service interface {
 
 	// TraverseBytesAddresses iterates through each address of a bytes.
 	TraverseBytesAddresses(context.Context, swarm.Address, swarm.AddressIterFunc) error
-	// TraverseFileAddresses iterates through each address of a file.
-	TraverseFileAddresses(context.Context, swarm.Address, swarm.AddressIterFunc) error
 	// TraverseManifestAddresses iterates through each address of a manifest,
 	// as well as each entry found in it.
 	TraverseManifestAddresses(context.Context, swarm.Address, swarm.AddressIterFunc) error
@@ -60,57 +54,17 @@ func (s *traversalService) TraverseAddresses(
 	chunkAddressFunc swarm.AddressIterFunc,
 ) error {
 
-	isFile, e, metadata, err := s.checkIsFile(ctx, reference)
+	isManifest, m, err := s.checkIsManifest(ctx, reference)
 	if err != nil {
 		return err
 	}
 
-	// reference address could be missrepresented as file when:
-	// - content size is 64 bytes (or 128 for encrypted reference)
-	// - second reference exists and is JSON (and not actually file metadata)
-
-	if isFile {
-
-		isManifest, m, err := s.checkIsManifest(ctx, reference, e, metadata)
-		if err != nil {
-			return err
-		}
-
-		// reference address could be missrepresented as manifest when:
-		// - file content type is actually on of manifest type (manually set)
-		// - content was unmarshalled
-		//
-		// even though content could be unmarshaled in some case, iteration
-		// through addresses will not be possible
-
-		if isManifest {
-			// process as manifest
-
-			err = m.IterateAddresses(ctx, func(manifestNodeAddr swarm.Address) error {
-				return s.traverseChunkAddressesFromManifest(ctx, manifestNodeAddr, chunkAddressFunc)
-			})
-			if err != nil {
-				return fmt.Errorf("traversal: iterate chunks: %s: %w", reference, err)
-			}
-
-			metadataReference := e.Metadata()
-
-			err = s.processBytes(ctx, metadataReference, chunkAddressFunc)
-			if err != nil {
-				return err
-			}
-
-			_ = chunkAddressFunc(reference)
-
-		} else {
-			return s.traverseChunkAddressesAsFile(ctx, reference, chunkAddressFunc, e)
-		}
-
-	} else {
-		return s.processBytes(ctx, reference, chunkAddressFunc)
+	if isManifest {
+		return m.IterateAddresses(ctx, func(manifestNodeAddr swarm.Address) error {
+			return s.processBytes(ctx, manifestNodeAddr, chunkAddressFunc)
+		})
 	}
-
-	return nil
+	return s.processBytes(ctx, reference, chunkAddressFunc)
 }
 
 func (s *traversalService) TraverseBytesAddresses(
@@ -121,213 +75,39 @@ func (s *traversalService) TraverseBytesAddresses(
 	return s.processBytes(ctx, reference, chunkAddressFunc)
 }
 
-func (s *traversalService) TraverseFileAddresses(
-	ctx context.Context,
-	reference swarm.Address,
-	chunkAddressFunc swarm.AddressIterFunc,
-) error {
-
-	isFile, e, _, err := s.checkIsFile(ctx, reference)
-	if err != nil {
-		return err
-	}
-
-	// reference address could be missrepresented as file when:
-	// - content size is 64 bytes (or 128 for encrypted reference)
-	// - second reference exists and is JSON (and not actually file metadata)
-
-	if !isFile {
-		return ErrInvalidType
-	}
-
-	return s.traverseChunkAddressesAsFile(ctx, reference, chunkAddressFunc, e)
-}
-
 func (s *traversalService) TraverseManifestAddresses(
 	ctx context.Context,
 	reference swarm.Address,
 	chunkAddressFunc swarm.AddressIterFunc,
 ) error {
 
-	isFile, e, metadata, err := s.checkIsFile(ctx, reference)
+	isManifest, m, err := s.checkIsManifest(ctx, reference)
 	if err != nil {
 		return err
 	}
-
-	if !isFile {
-		return ErrInvalidType
-	}
-
-	isManifest, m, err := s.checkIsManifest(ctx, reference, e, metadata)
-	if err != nil {
-		return err
-	}
-
-	// reference address could be missrepresented as manifest when:
-	// - file content type is actually on of manifest type (manually set)
-	// - content was unmarshalled
-	//
-	// even though content could be unmarshaled in some case, iteration
-	// through addresses will not be possible
-
 	if !isManifest {
 		return ErrInvalidType
 	}
 
 	err = m.IterateAddresses(ctx, func(manifestNodeAddr swarm.Address) error {
-		return s.traverseChunkAddressesFromManifest(ctx, manifestNodeAddr, chunkAddressFunc)
+		return s.processBytes(ctx, manifestNodeAddr, chunkAddressFunc)
 	})
 	if err != nil {
 		return fmt.Errorf("traversal: iterate chunks: %s: %w", reference, err)
 	}
 
-	metadataReference := e.Metadata()
-
-	err = s.processBytes(ctx, metadataReference, chunkAddressFunc)
-	if err != nil {
-		return err
-	}
-
-	_ = chunkAddressFunc(reference)
-
 	return nil
 }
 
-func (s *traversalService) traverseChunkAddressesFromManifest(
-	ctx context.Context,
-	reference swarm.Address,
-	chunkAddressFunc swarm.AddressIterFunc,
-) error {
-
-	isFile, e, _, err := s.checkIsFile(ctx, reference)
-	if err != nil {
-		return err
-	}
-
-	if isFile {
-		return s.traverseChunkAddressesAsFile(ctx, reference, chunkAddressFunc, e)
-	}
-
-	return s.processBytes(ctx, reference, chunkAddressFunc)
-}
-
-func (s *traversalService) traverseChunkAddressesAsFile(
-	ctx context.Context,
-	reference swarm.Address,
-	chunkAddressFunc swarm.AddressIterFunc,
-	e *entry.Entry,
-) (err error) {
-
-	bytesReference := e.Reference()
-
-	err = s.processBytes(ctx, bytesReference, chunkAddressFunc)
-	if err != nil {
-		// possible it was custom JSON bytes, which matches entry JSON
-		// but in fact is not file, and does not contain reference to
-		// existing address, which is why it was not found in storage
-		if !errors.Is(err, storage.ErrNotFound) {
-			return nil
-		}
-		// ignore
-	}
-
-	metadataReference := e.Metadata()
-
-	err = s.processBytes(ctx, metadataReference, chunkAddressFunc)
-	if err != nil {
-		return
-	}
-
-	_ = chunkAddressFunc(reference)
-
-	return nil
-}
-
-// checkIsFile checks if the content is file.
-func (s *traversalService) checkIsFile(
-	ctx context.Context,
-	reference swarm.Address,
-) (isFile bool, e *entry.Entry, metadata *entry.Metadata, err error) {
-
-	var (
-		j    file.Joiner
-		span int64
-	)
-
-	j, span, err = joiner.New(ctx, s.storer, reference)
-	if err != nil {
-		err = fmt.Errorf("traversal: joiner: %s: %w", reference, err)
-		return
-	}
-
-	maybeIsFile := entry.CanUnmarshal(span)
-
-	if maybeIsFile {
-		buf := bytes.NewBuffer(nil)
-		_, err = file.JoinReadAll(ctx, j, buf)
-		if err != nil {
-			err = fmt.Errorf("traversal: read entry: %s: %w", reference, err)
-			return
-		}
-
-		e = &entry.Entry{}
-		err = e.UnmarshalBinary(buf.Bytes())
-		if err != nil {
-			err = fmt.Errorf("traversal: unmarshal entry: %s: %w", reference, err)
-			return
-		}
-
-		// address sizes must match
-		if len(reference.Bytes()) != len(e.Reference().Bytes()) {
-			return
-		}
-
-		// NOTE: any bytes will unmarshall to addresses; we need to check metadata
-
-		// read metadata
-		j, _, err = joiner.New(ctx, s.storer, e.Metadata())
-		if err != nil {
-			// ignore
-			err = nil
-			return
-		}
-
-		buf = bytes.NewBuffer(nil)
-		_, err = file.JoinReadAll(ctx, j, buf)
-		if err != nil {
-			err = fmt.Errorf("traversal: read metadata: %s: %w", reference, err)
-			return
-		}
-
-		metadata = &entry.Metadata{}
-
-		dec := json.NewDecoder(buf)
-		dec.DisallowUnknownFields()
-		err = dec.Decode(metadata)
-		if err != nil {
-			// may not be metadata JSON
-			err = nil
-			return
-		}
-
-		isFile = true
-	}
-
-	return
-}
-
 // checkIsManifest checks if the content is manifest.
 func (s *traversalService) checkIsManifest(
 	ctx context.Context,
 	reference swarm.Address,
-	e *entry.Entry,
-	metadata *entry.Metadata,
 ) (isManifest bool, m manifest.Interface, err error) {
 
 	// NOTE: 'encrypted' parameter only used for saving manifest
-	m, err = manifest.NewManifestReference(
-		metadata.MimeType,
-		e.Reference(),
+	m, err = manifest.NewDefaultManifestReference(
+		reference,
 		loadsave.New(s.storer, storage.ModePutRequest, false),
 	)
 	if err != nil {
@@ -339,9 +119,7 @@ func (s *traversalService) checkIsManifest(
 		err = fmt.Errorf("traversal: read manifest: %s: %w", reference, err)
 		return
 	}
-
 	isManifest = true
-
 	return
 }