Commit 8206b490 authored by Alok Nerurkar's avatar Alok Nerurkar Committed by GitHub

Removing collections package and API cleanup (#1501)

parent 91a4a249
......@@ -5,31 +5,13 @@
package main
import (
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"io"
"io/ioutil"
"os"
"path/filepath"
cmdfile "github.com/ethersphere/bee/cmd/internal/file"
"github.com/ethersphere/bee/pkg/collection/entry"
"github.com/ethersphere/bee/pkg/file"
"github.com/ethersphere/bee/pkg/file/joiner"
"github.com/ethersphere/bee/pkg/file/splitter"
"github.com/ethersphere/bee/pkg/logging"
"github.com/ethersphere/bee/pkg/storage"
"github.com/ethersphere/bee/pkg/swarm"
"github.com/spf13/cobra"
)
const (
defaultMimeType = "application/octet-stream"
limitMetadataLength = swarm.ChunkSize
)
var (
filename string // flag variable, filename to use in metadata
mimeType string // flag variable, mime type to use in metadata
......@@ -41,186 +23,11 @@ var (
ssl bool // flag variable, uses https for api if set
retrieve bool // flag variable, if set will resolve and retrieve referenced file
verbosity string // flag variable, debug level
logger logging.Logger
)
// getEntry handles retrieving and writing a file from the file entry
// referenced by the given address.
func getEntry(cmd *cobra.Command, args []string) (err error) {
// process the reference to retrieve
addr, err := swarm.ParseHexAddress(args[0])
if err != nil {
return err
}
// initialize interface with HTTP API
store := cmdfile.NewApiStore(host, port, ssl)
buf := bytes.NewBuffer(nil)
writeCloser := cmdfile.NopWriteCloser(buf)
limitBuf := cmdfile.NewLimitWriteCloser(writeCloser, limitMetadataLength)
j, _, err := joiner.New(cmd.Context(), store, addr)
if err != nil {
return err
}
_, err = file.JoinReadAll(cmd.Context(), j, limitBuf)
if err != nil {
return err
}
e := &entry.Entry{}
err = e.UnmarshalBinary(buf.Bytes())
if err != nil {
return err
}
j, _, err = joiner.New(cmd.Context(), store, e.Metadata())
if err != nil {
return err
}
buf = bytes.NewBuffer(nil)
_, err = file.JoinReadAll(cmd.Context(), j, buf)
if err != nil {
return err
}
// retrieve metadata
metaData := &entry.Metadata{}
err = json.Unmarshal(buf.Bytes(), metaData)
if err != nil {
return err
}
logger.Debugf("Filename: %s", metaData.Filename)
logger.Debugf("MIME-type: %s", metaData.MimeType)
if outDir == "" {
outDir = "."
} else {
err := os.MkdirAll(outDir, 0o777) // skipcq: GSC-G301
if err != nil {
return err
}
}
outFilePath := filepath.Join(outDir, metaData.Filename)
// create output dir if not exist
if outDir != "." {
err := os.MkdirAll(outDir, 0o777) // skipcq: GSC-G301
if err != nil {
return err
}
}
// protect any existing file unless explicitly told not to
outFileFlags := os.O_CREATE | os.O_WRONLY
if outFileForce {
outFileFlags |= os.O_TRUNC
} else {
outFileFlags |= os.O_EXCL
}
// open the file
outFile, err := os.OpenFile(outFilePath, outFileFlags, 0o666) // skipcq: GSC-G302
if err != nil {
return err
}
defer outFile.Close()
j, _, err = joiner.New(cmd.Context(), store, e.Reference())
if err != nil {
return err
}
_, err = file.JoinReadAll(cmd.Context(), j, outFile)
return err
}
// putEntry creates a new file entry with the given reference.
func putEntry(cmd *cobra.Command, args []string) (err error) {
// process the reference to retrieve
addr, err := swarm.ParseHexAddress(args[0])
if err != nil {
return err
}
// add the fsStore and/or apiStore, depending on flags
stores := cmdfile.NewTeeStore()
if outDir != "" {
err := os.MkdirAll(outDir, 0o777) // skipcq: GSC-G301
if err != nil {
return err
}
store := cmdfile.NewFsStore(outDir)
stores.Add(store)
}
if useHttp {
store := cmdfile.NewApiStore(host, port, ssl)
stores.Add(store)
}
// create metadata object, with defaults for missing values
if filename == "" {
filename = args[0]
}
if mimeType == "" {
mimeType = defaultMimeType
}
metadata := entry.NewMetadata(filename)
metadata.MimeType = mimeType
// serialize metadata and send it to splitter
metadataBytes, err := json.Marshal(metadata)
if err != nil {
return err
}
logger.Debugf("metadata contents: %s", metadataBytes)
// set up splitter to process the metadata
s := splitter.NewSimpleSplitter(stores, storage.ModePutUpload)
ctx := context.Background()
// first add metadata
metadataBuf := bytes.NewBuffer(metadataBytes)
metadataReader := io.LimitReader(metadataBuf, int64(len(metadataBytes)))
metadataReadCloser := ioutil.NopCloser(metadataReader)
metadataAddr, err := s.Split(ctx, metadataReadCloser, int64(len(metadataBytes)), false)
if err != nil {
return err
}
// create entry from given reference and metadata,
// serialize and send to splitter
fileEntry := entry.New(addr, metadataAddr)
fileEntryBytes, err := fileEntry.MarshalBinary()
if err != nil {
return err
}
fileEntryBuf := bytes.NewBuffer(fileEntryBytes)
fileEntryReader := io.LimitReader(fileEntryBuf, int64(len(fileEntryBytes)))
fileEntryReadCloser := ioutil.NopCloser(fileEntryReader)
fileEntryAddr, err := s.Split(ctx, fileEntryReadCloser, int64(len(fileEntryBytes)), false)
if err != nil {
return err
}
// output reference to file entry
cmd.Println(fileEntryAddr)
return nil
}
// Entry is the underlying procedure for the CLI command
func Entry(cmd *cobra.Command, args []string) (err error) {
logger, err = cmdfile.SetLogger(cmd, verbosity)
if err != nil {
fmt.Fprintln(os.Stderr, err.Error())
os.Exit(1)
}
if retrieve {
return getEntry(cmd, args)
}
return putEntry(cmd, args)
return errors.New("command is deprecated")
}
func main() {
......
......@@ -154,22 +154,31 @@ paths:
default:
description: Default response
"/files":
"/bzz":
post:
summary: "Upload file"
summary: "Upload file or a collection of files"
description: "In order to upload a collection, user can send a multipart request with all the files populated in the form data with appropriate headers.\n\n
User can also upload a tar file along with the swarm-collection header. This will upload the tar file after extracting the entire directory structure.\n\n
If the swarm-collection header is absent, all requests (including tar files) are considered as single file uploads.\n\n
A multipart request is treated as a collection regardless of whether the swarm-collection header is present. This means in order to serve single files
uploaded as a multipart request, the swarm-index-document header should be used with the name of the file."
tags:
- File
- Collection
parameters:
- in: query
name: name
schema:
$ref: "SwarmCommon.yaml#/components/schemas/FileName"
required: false
description: Filename
description: Filename when uploading single file
- $ref: "SwarmCommon.yaml#/components/parameters/SwarmTagParameter"
- $ref: "SwarmCommon.yaml#/components/parameters/SwarmPinParameter"
- $ref: "SwarmCommon.yaml#/components/parameters/SwarmEncryptParameter"
- $ref: "SwarmCommon.yaml#/components/parameters/ContentTypePreserved"
- $ref: "SwarmCommon.yaml#/components/parameters/SwarmCollection"
- $ref: "SwarmCommon.yaml#/components/parameters/SwarmIndexDocumentParameter"
- $ref: "SwarmCommon.yaml#/components/parameters/SwarmErrorDocumentParameter"
requestBody:
content:
multipart/form-data:
......@@ -184,76 +193,6 @@ paths:
schema:
type: string
format: binary
responses:
"200":
description: Ok
headers:
"swarm-tag":
$ref: "SwarmCommon.yaml#/components/headers/SwarmTag"
"etag":
$ref: "SwarmCommon.yaml#/components/headers/ETag"
content:
application/json:
schema:
$ref: "SwarmCommon.yaml#/components/schemas/ReferenceResponse"
"400":
$ref: "SwarmCommon.yaml#/components/responses/400"
"403":
$ref: "SwarmCommon.yaml#/components/responses/403"
"500":
$ref: "SwarmCommon.yaml#/components/responses/500"
default:
description: Default response
"/files/{reference}":
get:
summary: "Get referenced file"
tags:
- File
parameters:
- in: path
name: reference
schema:
$ref: "SwarmCommon.yaml#/components/schemas/SwarmReference"
required: true
description: Swarm address of content
- $ref: "SwarmCommon.yaml#/components/parameters/SwarmRecoveryTargetsParameter"
responses:
"200":
description: Ok
headers:
"swarm-recovery-targets":
$ref: "SwarmCommon.yaml#/components/headers/SwarmRecoveryTargets"
"ETag":
$ref: "SwarmCommon.yaml#/components/headers/ETag"
content:
application/octet-stream:
schema:
type: string
format: binary
"400":
$ref: "SwarmCommon.yaml#/components/responses/400"
"404":
$ref: "SwarmCommon.yaml#/components/responses/404"
"500":
$ref: "SwarmCommon.yaml#/components/responses/500"
default:
description: Default response
"/dirs":
post:
summary: "Upload a collection"
tags:
- Collection
parameters:
- $ref: "SwarmCommon.yaml#/components/parameters/SwarmTagParameter"
- $ref: "SwarmCommon.yaml#/components/parameters/SwarmPinParameter"
- $ref: "SwarmCommon.yaml#/components/parameters/SwarmEncryptParameter"
- $ref: "SwarmCommon.yaml#/components/parameters/SwarmIndexDocumentParameter"
- $ref: "SwarmCommon.yaml#/components/parameters/SwarmErrorDocumentParameter"
- $ref: "SwarmCommon.yaml#/components/parameters/ContentTypePreserved"
requestBody:
content:
application/x-tar:
schema:
type: string
......@@ -264,6 +203,8 @@ paths:
headers:
"swarm-tag":
$ref: "SwarmCommon.yaml#/components/headers/SwarmTag"
"etag":
$ref: "SwarmCommon.yaml#/components/headers/ETag"
content:
application/json:
schema:
......@@ -279,7 +220,7 @@ paths:
"/bzz/{reference}":
get:
summary: "Get index document from a collection of files"
summary: "Get file or index document from a collection of files"
tags:
- Collection
parameters:
......
......@@ -493,6 +493,14 @@ components:
required: false
description: Configure custom error document to be returned when a specified path can not be found in collection
SwarmCollection:
in: header
name: swarm-collection
schema:
type: boolean
required: false
description: Upload file/files as a collection
responses:
"204":
description: The resource was deleted successfully.
......
......@@ -40,6 +40,7 @@ const (
SwarmErrorDocumentHeader = "Swarm-Error-Document"
SwarmFeedIndexHeader = "Swarm-Feed-Index"
SwarmFeedIndexNextHeader = "Swarm-Feed-Index-Next"
SwarmCollectionHeader = "Swarm-Collection"
)
// The size of buffer used for prefetching content with Langos.
......@@ -54,9 +55,20 @@ const (
largeBufferFilesizeThreshold = 10 * 1000000 // ten megs
)
const (
contentTypeHeader = "Content-Type"
multiPartFormData = "multipart/form-data"
contentTypeTar = "application/x-tar"
)
var (
errInvalidNameOrAddress = errors.New("invalid name or bzz address")
errNoResolver = errors.New("no resolver connected")
invalidRequest = errors.New("could not validate request")
invalidContentType = errors.New("invalid content-type")
invalidContentLength = errors.New("invalid content-length")
directoryStoreError = errors.New("could not store directory")
fileStoreError = errors.New("could not store file")
)
// Service is the API service interface.
......
......@@ -74,11 +74,20 @@ func TestBytes(t *testing.T) {
})
t.Run("not found", func(t *testing.T) {
jsonhttptest.Request(t, client, http.MethodGet, resource+"/abcd", http.StatusNotFound,
jsonhttptest.Request(t, client, http.MethodGet, resource+"/0xabcd", http.StatusNotFound,
jsonhttptest.WithExpectedJSONResponse(jsonhttp.StatusResponse{
Message: "Not Found",
Code: http.StatusNotFound,
}),
)
})
t.Run("internal error", func(t *testing.T) {
jsonhttptest.Request(t, client, http.MethodGet, resource+"/abcd", http.StatusInternalServerError,
jsonhttptest.WithExpectedJSONResponse(jsonhttp.StatusResponse{
Message: "Internal Server Error",
Code: http.StatusInternalServerError,
}),
)
})
}
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
......@@ -13,7 +13,7 @@ type (
ChunkAddressResponse = chunkAddressResponse
SocPostResponse = socPostResponse
FeedReferenceResponse = feedReferenceResponse
FileUploadResponse = fileUploadResponse
BzzUploadResponse = bzzUploadResponse
TagResponse = tagResponse
TagRequest = tagRequest
ListTagsResponse = listTagsResponse
......@@ -23,13 +23,13 @@ type (
)
var (
ContentTypeTar = contentTypeTar
InvalidContentType = invalidContentType
InvalidRequest = invalidRequest
DirectoryStoreError = directoryStoreError
)
var (
ManifestRootPath = manifestRootPath
ManifestWebsiteIndexDocumentSuffixKey = manifestWebsiteIndexDocumentSuffixKey
ManifestWebsiteErrorDocumentPathKey = manifestWebsiteErrorDocumentPathKey
ContentTypeTar = contentTypeTar
)
var (
......
This diff is collapsed.
This diff is collapsed.
......@@ -47,7 +47,7 @@ func TestPinBytesHandler(t *testing.T) {
jsonhttptest.Request(t, client, http.MethodPost, bytesUploadResource, http.StatusOK,
jsonhttptest.WithRequestBody(bytes.NewReader(simpleData)),
jsonhttptest.WithExpectedJSONResponse(api.FileUploadResponse{
jsonhttptest.WithExpectedJSONResponse(api.BzzUploadResponse{
Reference: swarm.MustParseHexAddress(rootHash),
}),
)
......@@ -110,7 +110,7 @@ func TestPinBytesHandler(t *testing.T) {
jsonhttptest.Request(t, client, http.MethodPost, bytesUploadResource, http.StatusOK,
jsonhttptest.WithRequestBody(bytes.NewReader(b)),
jsonhttptest.WithExpectedJSONResponse(api.FileUploadResponse{
jsonhttptest.WithExpectedJSONResponse(api.BzzUploadResponse{
Reference: swarm.MustParseHexAddress(rootHash),
}),
)
......
......@@ -22,7 +22,7 @@ import (
func TestPinBzzHandler(t *testing.T) {
var (
dirUploadResource = "/dirs"
dirUploadResource = "/bzz"
pinBzzResource = "/pin/bzz"
pinBzzAddressResource = func(addr string) string { return pinBzzResource + "/" + addr }
pinChunksResource = "/pin/chunks"
......@@ -35,6 +35,7 @@ func TestPinBzzHandler(t *testing.T) {
Storer: mockStorer,
Traversal: traversalService,
Tags: tags.NewTags(mockStatestore, logger),
Logger: logger,
})
)
......@@ -49,13 +50,14 @@ func TestPinBzzHandler(t *testing.T) {
tarReader := tarFiles(t, files)
rootHash := "a85aaea6a34a5c7127a3546196f2111f866fe369c6d6562ed5d3313a99388c03"
rootHash := "9e178dbd1ed4b748379e25144e28dfb29c07a4b5114896ef454480115a56b237"
// verify directory tar upload response
jsonhttptest.Request(t, client, http.MethodPost, dirUploadResource, http.StatusOK,
jsonhttptest.WithRequestBody(tarReader),
jsonhttptest.WithRequestHeader("Content-Type", api.ContentTypeTar),
jsonhttptest.WithExpectedJSONResponse(api.FileUploadResponse{
jsonhttptest.WithRequestHeader(api.SwarmCollectionHeader, "True"),
jsonhttptest.WithExpectedJSONResponse(api.BzzUploadResponse{
Reference: swarm.MustParseHexAddress(rootHash),
}),
)
......@@ -67,7 +69,7 @@ func TestPinBzzHandler(t *testing.T) {
}),
)
expectedChunkCount := 7
expectedChunkCount := 3
// get the reference as everytime it will change because of random encryption key
var resp api.ListPinnedChunksResponse
......@@ -82,7 +84,7 @@ func TestPinBzzHandler(t *testing.T) {
})
t.Run("unpin-bzz-1", func(t *testing.T) {
rootHash := "a85aaea6a34a5c7127a3546196f2111f866fe369c6d6562ed5d3313a99388c03"
rootHash := "9e178dbd1ed4b748379e25144e28dfb29c07a4b5114896ef454480115a56b237"
jsonhttptest.Request(t, client, http.MethodDelete, pinBzzAddressResource(rootHash), http.StatusOK,
jsonhttptest.WithExpectedJSONResponse(jsonhttp.StatusResponse{
......
......@@ -48,7 +48,7 @@ func (s *server) pinFile(w http.ResponseWriter, r *http.Request) {
chunkAddressFn := s.pinChunkAddressFn(ctx, addr)
err = s.traversal.TraverseFileAddresses(ctx, addr, chunkAddressFn)
err = s.traversal.TraverseAddresses(ctx, addr, chunkAddressFn)
if err != nil {
s.logger.Debugf("pin files: traverse chunks: %v, addr %s", err, addr)
......@@ -93,7 +93,7 @@ func (s *server) unpinFile(w http.ResponseWriter, r *http.Request) {
chunkAddressFn := s.unpinChunkAddressFn(ctx, addr)
err = s.traversal.TraverseFileAddresses(ctx, addr, chunkAddressFn)
err = s.traversal.TraverseAddresses(ctx, addr, chunkAddressFn)
if err != nil {
s.logger.Debugf("pin files: traverse chunks: %v, addr %s", err, addr)
......
......@@ -8,7 +8,6 @@ import (
"bytes"
"io/ioutil"
"net/http"
"sort"
"testing"
"github.com/ethersphere/bee/pkg/api"
......@@ -24,7 +23,7 @@ import (
func TestPinFilesHandler(t *testing.T) {
var (
fileUploadResource = "/files"
fileUploadResource = "/bzz"
pinFilesResource = "/pin/files"
pinFilesAddressResource = func(addr string) string { return pinFilesResource + "/" + addr }
pinChunksResource = "/pin/chunks"
......@@ -39,17 +38,20 @@ func TestPinFilesHandler(t *testing.T) {
Storer: mockStorer,
Traversal: traversalService,
Tags: tags.NewTags(mockStatestore, logger),
Logger: logger,
})
)
t.Run("pin-file-1", func(t *testing.T) {
rootHash := "dc82503e0ed041a57327ad558d7aa69a867024c8221306c461ae359dc34d1c6a"
metadataHash := "d936d7180f230b3424842ea10848aa205f2f0e830cb9cc7588a39c9381544bf9"
rootHash := "dd13a5a6cc9db3ef514d645e6719178dbfb1a90b49b9262cafce35b0d27cf245"
metadataHash := "0cc878d32c96126d47f63fbe391114ee1438cd521146fc975dea1546d302b6c0"
metadataHash2 := "a14d1ef845307c634e9ec74539bd668d0d1b37f37de4128939d57098135850da"
contentHash := "838d0a193ecd1152d1bb1432d5ecc02398533b2494889e23b8bd5ace30ac2aeb"
jsonhttptest.Request(t, client, http.MethodPost, fileUploadResource, http.StatusOK,
jsonhttptest.Request(t, client, http.MethodPost,
fileUploadResource+"?name=somefile.txt", http.StatusOK,
jsonhttptest.WithRequestBody(bytes.NewReader(simpleData)),
jsonhttptest.WithExpectedJSONResponse(api.FileUploadResponse{
jsonhttptest.WithExpectedJSONResponse(api.BzzUploadResponse{
Reference: swarm.MustParseHexAddress(rootHash),
}),
jsonhttptest.WithRequestHeader("Content-Type", "text/plain"),
......@@ -62,27 +64,36 @@ func TestPinFilesHandler(t *testing.T) {
}),
)
hashes := []string{rootHash, metadataHash, contentHash}
sort.Strings(hashes)
expectedResponse := api.ListPinnedChunksResponse{
Chunks: []api.PinnedChunk{},
hashes := map[string]int{
rootHash: 1,
metadataHash: 1,
metadataHash2: 1,
contentHash: 1,
}
for _, h := range hashes {
expectedResponse.Chunks = append(expectedResponse.Chunks, api.PinnedChunk{
Address: swarm.MustParseHexAddress(h),
PinCounter: 1,
})
actualResponse := api.ListPinnedChunksResponse{
Chunks: []api.PinnedChunk{},
}
jsonhttptest.Request(t, client, http.MethodGet, pinChunksResource, http.StatusOK,
jsonhttptest.WithExpectedJSONResponse(expectedResponse),
jsonhttptest.WithUnmarshalJSONResponse(&actualResponse),
)
if len(actualResponse.Chunks) != len(hashes) {
t.Fatalf("Response chunk count mismatch Expected: %d Found: %d",
len(hashes), len(actualResponse.Chunks))
}
for _, v := range actualResponse.Chunks {
if counter, ok := hashes[v.Address.String()]; !ok {
t.Fatalf("found unexpected hash %s", v.Address.String())
} else if uint64(counter) != v.PinCounter {
t.Fatalf("found unexpected pin counter: Expected: %d, Found: %d",
counter, v.PinCounter)
}
}
})
t.Run("unpin-file-1", func(t *testing.T) {
rootHash := "dc82503e0ed041a57327ad558d7aa69a867024c8221306c461ae359dc34d1c6a"
rootHash := "dd13a5a6cc9db3ef514d645e6719178dbfb1a90b49b9262cafce35b0d27cf245"
jsonhttptest.Request(t, client, http.MethodDelete, pinFilesAddressResource(rootHash), http.StatusOK,
jsonhttptest.WithExpectedJSONResponse(jsonhttp.StatusResponse{
......
......@@ -38,26 +38,6 @@ func (s *server) setupRouting() {
fmt.Fprintln(w, "User-agent: *\nDisallow: /")
})
handle(router, "/files", jsonhttp.MethodHandler{
"POST": web.ChainHandlers(
s.newTracingHandler("files-upload"),
web.FinalHandlerFunc(s.fileUploadHandler),
),
})
handle(router, "/files/{addr}", jsonhttp.MethodHandler{
"GET": web.ChainHandlers(
s.newTracingHandler("files-download"),
web.FinalHandlerFunc(s.fileDownloadHandler),
),
})
handle(router, "/dirs", jsonhttp.MethodHandler{
"POST": web.ChainHandlers(
s.newTracingHandler("dirs-upload"),
web.FinalHandlerFunc(s.dirUploadHandler),
),
})
handle(router, "/bytes", jsonhttp.MethodHandler{
"POST": web.ChainHandlers(
s.newTracingHandler("bytes-upload"),
......@@ -97,6 +77,12 @@ func (s *server) setupRouting() {
),
})
handle(router, "/bzz", jsonhttp.MethodHandler{
"POST": web.ChainHandlers(
s.newTracingHandler("bzz-upload"),
web.FinalHandlerFunc(s.bzzUploadHandler),
),
})
handle(router, "/bzz/{address}", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
u := r.URL
u.Path += "/"
......
......@@ -35,8 +35,7 @@ func tagsWithIdResource(id uint32) string { return fmt.Sprintf("/tags/%d", id) }
func TestTags(t *testing.T) {
var (
filesResource = "/files"
dirResource = "/dirs"
bzzResource = "/bzz"
bytesResource = "/bytes"
chunksResource = "/chunks"
tagsResource = "/tags"
......@@ -47,6 +46,7 @@ func TestTags(t *testing.T) {
client, _, _ = newTestServer(t, testServerOptions{
Storer: mock.NewStorer(),
Tags: tag,
Logger: logger,
})
)
......@@ -263,10 +263,11 @@ func TestTags(t *testing.T) {
t.Run("file tags", func(t *testing.T) {
// upload a file without supplying tag
expectedHash := swarm.MustParseHexAddress("8e27bb803ff049e8c2f4650357026723220170c15ebf9b635a7026539879a1a8")
expectedResponse := api.FileUploadResponse{Reference: expectedHash}
expectedHash := swarm.MustParseHexAddress("40e739ebdfd18292925bba4138cd097db9aa18c1b57e74042f48469b48da33a8")
expectedResponse := api.BzzUploadResponse{Reference: expectedHash}
respHeaders := jsonhttptest.Request(t, client, http.MethodPost, filesResource, http.StatusOK,
respHeaders := jsonhttptest.Request(t, client, http.MethodPost,
bzzResource+"?name=somefile", http.StatusOK,
jsonhttptest.WithRequestBody(bytes.NewReader([]byte("some data"))),
jsonhttptest.WithExpectedJSONResponse(expectedResponse),
jsonhttptest.WithRequestHeader("Content-Type", "application/octet-stream"),
......@@ -276,7 +277,7 @@ func TestTags(t *testing.T) {
if err != nil {
t.Fatal(err)
}
tagValueTest(t, uint32(tagId), 3, 3, 0, 0, 0, 3, expectedHash, client)
tagValueTest(t, uint32(tagId), 4, 4, 0, 0, 0, 4, expectedHash, client)
})
t.Run("dir tags", func(t *testing.T) {
......@@ -285,11 +286,12 @@ func TestTags(t *testing.T) {
data: []byte("some dir data"),
name: "binary-file",
}})
expectedHash := swarm.MustParseHexAddress("3dc643abeb3db60a4dfb72008b577dd9a573abaa74c6afe37a75c63ceea829f6")
expectedResponse := api.FileUploadResponse{Reference: expectedHash}
expectedHash := swarm.MustParseHexAddress("42bc27c9137c93705ffbc2945fa1aab0e8e1826f1500b7f06f6e3f86f617213b")
expectedResponse := api.BzzUploadResponse{Reference: expectedHash}
respHeaders := jsonhttptest.Request(t, client, http.MethodPost, dirResource, http.StatusOK,
respHeaders := jsonhttptest.Request(t, client, http.MethodPost, bzzResource, http.StatusOK,
jsonhttptest.WithRequestBody(tarReader),
jsonhttptest.WithRequestHeader(api.SwarmCollectionHeader, "True"),
jsonhttptest.WithExpectedJSONResponse(expectedResponse),
jsonhttptest.WithRequestHeader("Content-Type", api.ContentTypeTar),
)
......@@ -298,7 +300,7 @@ func TestTags(t *testing.T) {
if err != nil {
t.Fatal(err)
}
tagValueTest(t, uint32(tagId), 7, 7, 0, 0, 0, 7, expectedHash, client)
tagValueTest(t, uint32(tagId), 3, 3, 0, 0, 0, 3, expectedHash, client)
})
t.Run("bytes tags", func(t *testing.T) {
......
// Copyright 2020 The Swarm Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// Package collection provides high-level abstractions for collections of files
package collection
import (
"github.com/ethersphere/bee/pkg/swarm"
)
// Collection provides a specific ordering of a collection of binary data vectors
// stored in bee.
type Collection interface {
Addresses() []swarm.Address
}
// Entry encapsulates data defining a single file entry.
// It may contain any number of data blobs providing context to the
// given data vector concealed by Reference.
type Entry interface {
Reference() swarm.Address
Metadata() swarm.Address
}
// Copyright 2020 The Swarm Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package entry
import (
"errors"
"math"
"github.com/ethersphere/bee/pkg/collection"
"github.com/ethersphere/bee/pkg/encryption"
"github.com/ethersphere/bee/pkg/swarm"
)
var (
_ = collection.Entry(&Entry{})
serializedDataSize = swarm.SectionSize * 2
encryptedSerializedDataSize = encryption.ReferenceSize * 2
)
// Entry provides addition of metadata to a data reference.
// Implements collection.Entry.
type Entry struct {
reference swarm.Address
metadata swarm.Address
}
// New creates a new Entry.
func New(reference, metadata swarm.Address) *Entry {
return &Entry{
reference: reference,
metadata: metadata,
}
}
// CanUnmarshal returns whether the entry may be might be unmarshaled based on
// the size.
func CanUnmarshal(size int64) bool {
if size < math.MaxInt32 {
switch int(size) {
case serializedDataSize, encryptedSerializedDataSize:
return true
}
}
return false
}
// Reference implements collection.Entry
func (e *Entry) Reference() swarm.Address {
return e.reference
}
// Metadata implements collection.Entry
func (e *Entry) Metadata() swarm.Address {
return e.metadata
}
// MarshalBinary implements encoding.BinaryMarshaler
func (e *Entry) MarshalBinary() ([]byte, error) {
br := e.reference.Bytes()
bm := e.metadata.Bytes()
b := append(br, bm...)
return b, nil
}
// UnmarshalBinary implements encoding.BinaryUnmarshaler
func (e *Entry) UnmarshalBinary(b []byte) error {
var size int
if len(b) == serializedDataSize {
size = serializedDataSize
} else if len(b) == encryptedSerializedDataSize {
size = encryptedSerializedDataSize
} else {
return errors.New("invalid data length")
}
e.reference = swarm.NewAddress(b[:size/2])
e.metadata = swarm.NewAddress(b[size/2:])
return nil
}
// Copyright 2020 The Swarm Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package entry_test
import (
"testing"
"github.com/ethersphere/bee/pkg/collection/entry"
"github.com/ethersphere/bee/pkg/swarm/test"
)
// TestEntrySerialize verifies integrity of serialization.
func TestEntrySerialize(t *testing.T) {
referenceAddress := test.RandomAddress()
metadataAddress := test.RandomAddress()
e := entry.New(referenceAddress, metadataAddress)
entrySerialized, err := e.MarshalBinary()
if err != nil {
t.Fatal(err)
}
entryRecovered := &entry.Entry{}
err = entryRecovered.UnmarshalBinary(entrySerialized)
if err != nil {
t.Fatal(err)
}
if !referenceAddress.Equal(entryRecovered.Reference()) {
t.Fatalf("expected reference %s, got %s", referenceAddress, entryRecovered.Reference())
}
metadataAddressRecovered := entryRecovered.Metadata()
if !metadataAddress.Equal(metadataAddressRecovered) {
t.Fatalf("expected metadata %s, got %s", metadataAddress, metadataAddressRecovered)
}
}
// Copyright 2020 The Swarm Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package entry
import (
"encoding/json"
)
// Metadata provides mime type and filename to file entry.
type Metadata struct {
MimeType string `json:"mimetype"`
Filename string `json:"filename"`
}
// NewMetadata creates a new Metadata.
func NewMetadata(fileName string) *Metadata {
return &Metadata{
Filename: fileName,
}
}
func (m *Metadata) String() string {
j, _ := json.Marshal(m)
return string(j)
}
......@@ -11,17 +11,22 @@ import (
"github.com/ethersphere/bee/pkg/swarm"
)
type PutGetter interface {
storage.Putter
storage.Getter
}
// loadSave is needed for manifest operations and provides
// simple wrapping over load and save operations using file
// package abstractions. use with caution since Loader will
// load all of the subtrie of a given hash in memory.
type loadSave struct {
storer storage.Storer
storer PutGetter
mode storage.ModePut
encrypted bool
}
func New(storer storage.Storer, mode storage.ModePut, enc bool) file.LoadSaver {
func New(storer PutGetter, mode storage.ModePut, enc bool) file.LoadSaver {
return &loadSave{
storer: storer,
mode: mode,
......
......@@ -16,6 +16,14 @@ import (
const DefaultManifestType = ManifestMantarayContentType
const (
RootPath = "/"
WebsiteIndexDocumentSuffixKey = "website-index-document"
WebsiteErrorDocumentPathKey = "website-error-document"
EntryMetadataContentTypeKey = "Content-Type"
EntryMetadataFilenameKey = "Filename"
)
var (
// ErrNotFound is returned when an Entry is not found in the manifest.
ErrNotFound = errors.New("manifest: not found")
......@@ -68,6 +76,14 @@ func NewDefaultManifest(
return NewManifest(DefaultManifestType, ls, encrypted)
}
// NewDefaultManifest creates a new manifest with default type.
func NewDefaultManifestReference(
reference swarm.Address,
ls file.LoadSaver,
) (Interface, error) {
return NewManifestReference(DefaultManifestType, reference, ls)
}
// NewManifest creates a new manifest.
func NewManifest(
manifestType string,
......
......@@ -149,7 +149,7 @@ func (m *mantarayManifest) IterateAddresses(ctx context.Context, fn swarm.Addres
}
}
if node.IsValueType() && node.Entry() != nil {
if node.IsValueType() && len(node.Entry()) > 0 {
entry := swarm.NewAddress(node.Entry())
err = fn(entry)
if err != nil {
......
......@@ -270,6 +270,13 @@ func (n *Node) UnmarshalBinary(data []byte) error {
n.entry = append([]byte{}, data[nodeHeaderSize:nodeHeaderSize+refBytesSize]...)
offset := nodeHeaderSize + refBytesSize // skip entry
// Currently we don't persist the root nodeType when we marshal the manifest, as a result
// the root nodeType information is lost on Unmarshal. This causes issues when we want to
// perform a path 'Walk' on the root. If there is more than 1 fork, the root node type
// is an edge, so we will deduce this information from index byte array
if !bytes.Equal(data[offset:offset+32], make([]byte, 32)) {
n.nodeType = nodeTypeEdge
}
n.forks = make(map[byte]*fork)
bb := &bitsForBytes{}
bb.fromBytes(data[offset:])
......
......@@ -9,14 +9,10 @@
package traversal
import (
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"github.com/ethersphere/bee/pkg/collection/entry"
"github.com/ethersphere/bee/pkg/file"
"github.com/ethersphere/bee/pkg/file/joiner"
"github.com/ethersphere/bee/pkg/file/loadsave"
"github.com/ethersphere/bee/pkg/manifest"
......@@ -37,8 +33,6 @@ type Service interface {
// TraverseBytesAddresses iterates through each address of a bytes.
TraverseBytesAddresses(context.Context, swarm.Address, swarm.AddressIterFunc) error
// TraverseFileAddresses iterates through each address of a file.
TraverseFileAddresses(context.Context, swarm.Address, swarm.AddressIterFunc) error
// TraverseManifestAddresses iterates through each address of a manifest,
// as well as each entry found in it.
TraverseManifestAddresses(context.Context, swarm.Address, swarm.AddressIterFunc) error
......@@ -60,57 +54,17 @@ func (s *traversalService) TraverseAddresses(
chunkAddressFunc swarm.AddressIterFunc,
) error {
isFile, e, metadata, err := s.checkIsFile(ctx, reference)
isManifest, m, err := s.checkIsManifest(ctx, reference)
if err != nil {
return err
}
// reference address could be missrepresented as file when:
// - content size is 64 bytes (or 128 for encrypted reference)
// - second reference exists and is JSON (and not actually file metadata)
if isFile {
isManifest, m, err := s.checkIsManifest(ctx, reference, e, metadata)
if err != nil {
return err
}
// reference address could be missrepresented as manifest when:
// - file content type is actually on of manifest type (manually set)
// - content was unmarshalled
//
// even though content could be unmarshaled in some case, iteration
// through addresses will not be possible
if isManifest {
// process as manifest
err = m.IterateAddresses(ctx, func(manifestNodeAddr swarm.Address) error {
return s.traverseChunkAddressesFromManifest(ctx, manifestNodeAddr, chunkAddressFunc)
})
if err != nil {
return fmt.Errorf("traversal: iterate chunks: %s: %w", reference, err)
}
metadataReference := e.Metadata()
err = s.processBytes(ctx, metadataReference, chunkAddressFunc)
if err != nil {
return err
}
_ = chunkAddressFunc(reference)
} else {
return s.traverseChunkAddressesAsFile(ctx, reference, chunkAddressFunc, e)
}
} else {
return s.processBytes(ctx, reference, chunkAddressFunc)
if isManifest {
return m.IterateAddresses(ctx, func(manifestNodeAddr swarm.Address) error {
return s.processBytes(ctx, manifestNodeAddr, chunkAddressFunc)
})
}
return nil
return s.processBytes(ctx, reference, chunkAddressFunc)
}
func (s *traversalService) TraverseBytesAddresses(
......@@ -121,213 +75,39 @@ func (s *traversalService) TraverseBytesAddresses(
return s.processBytes(ctx, reference, chunkAddressFunc)
}
func (s *traversalService) TraverseFileAddresses(
ctx context.Context,
reference swarm.Address,
chunkAddressFunc swarm.AddressIterFunc,
) error {
isFile, e, _, err := s.checkIsFile(ctx, reference)
if err != nil {
return err
}
// reference address could be missrepresented as file when:
// - content size is 64 bytes (or 128 for encrypted reference)
// - second reference exists and is JSON (and not actually file metadata)
if !isFile {
return ErrInvalidType
}
return s.traverseChunkAddressesAsFile(ctx, reference, chunkAddressFunc, e)
}
func (s *traversalService) TraverseManifestAddresses(
ctx context.Context,
reference swarm.Address,
chunkAddressFunc swarm.AddressIterFunc,
) error {
isFile, e, metadata, err := s.checkIsFile(ctx, reference)
isManifest, m, err := s.checkIsManifest(ctx, reference)
if err != nil {
return err
}
if !isFile {
return ErrInvalidType
}
isManifest, m, err := s.checkIsManifest(ctx, reference, e, metadata)
if err != nil {
return err
}
// reference address could be missrepresented as manifest when:
// - file content type is actually on of manifest type (manually set)
// - content was unmarshalled
//
// even though content could be unmarshaled in some case, iteration
// through addresses will not be possible
if !isManifest {
return ErrInvalidType
}
err = m.IterateAddresses(ctx, func(manifestNodeAddr swarm.Address) error {
return s.traverseChunkAddressesFromManifest(ctx, manifestNodeAddr, chunkAddressFunc)
return s.processBytes(ctx, manifestNodeAddr, chunkAddressFunc)
})
if err != nil {
return fmt.Errorf("traversal: iterate chunks: %s: %w", reference, err)
}
metadataReference := e.Metadata()
err = s.processBytes(ctx, metadataReference, chunkAddressFunc)
if err != nil {
return err
}
_ = chunkAddressFunc(reference)
return nil
}
func (s *traversalService) traverseChunkAddressesFromManifest(
ctx context.Context,
reference swarm.Address,
chunkAddressFunc swarm.AddressIterFunc,
) error {
isFile, e, _, err := s.checkIsFile(ctx, reference)
if err != nil {
return err
}
if isFile {
return s.traverseChunkAddressesAsFile(ctx, reference, chunkAddressFunc, e)
}
return s.processBytes(ctx, reference, chunkAddressFunc)
}
func (s *traversalService) traverseChunkAddressesAsFile(
ctx context.Context,
reference swarm.Address,
chunkAddressFunc swarm.AddressIterFunc,
e *entry.Entry,
) (err error) {
bytesReference := e.Reference()
err = s.processBytes(ctx, bytesReference, chunkAddressFunc)
if err != nil {
// possible it was custom JSON bytes, which matches entry JSON
// but in fact is not file, and does not contain reference to
// existing address, which is why it was not found in storage
if !errors.Is(err, storage.ErrNotFound) {
return nil
}
// ignore
}
metadataReference := e.Metadata()
err = s.processBytes(ctx, metadataReference, chunkAddressFunc)
if err != nil {
return
}
_ = chunkAddressFunc(reference)
return nil
}
// checkIsFile checks if the content is file.
func (s *traversalService) checkIsFile(
ctx context.Context,
reference swarm.Address,
) (isFile bool, e *entry.Entry, metadata *entry.Metadata, err error) {
var (
j file.Joiner
span int64
)
j, span, err = joiner.New(ctx, s.storer, reference)
if err != nil {
err = fmt.Errorf("traversal: joiner: %s: %w", reference, err)
return
}
maybeIsFile := entry.CanUnmarshal(span)
if maybeIsFile {
buf := bytes.NewBuffer(nil)
_, err = file.JoinReadAll(ctx, j, buf)
if err != nil {
err = fmt.Errorf("traversal: read entry: %s: %w", reference, err)
return
}
e = &entry.Entry{}
err = e.UnmarshalBinary(buf.Bytes())
if err != nil {
err = fmt.Errorf("traversal: unmarshal entry: %s: %w", reference, err)
return
}
// address sizes must match
if len(reference.Bytes()) != len(e.Reference().Bytes()) {
return
}
// NOTE: any bytes will unmarshall to addresses; we need to check metadata
// read metadata
j, _, err = joiner.New(ctx, s.storer, e.Metadata())
if err != nil {
// ignore
err = nil
return
}
buf = bytes.NewBuffer(nil)
_, err = file.JoinReadAll(ctx, j, buf)
if err != nil {
err = fmt.Errorf("traversal: read metadata: %s: %w", reference, err)
return
}
metadata = &entry.Metadata{}
dec := json.NewDecoder(buf)
dec.DisallowUnknownFields()
err = dec.Decode(metadata)
if err != nil {
// may not be metadata JSON
err = nil
return
}
isFile = true
}
return
}
// checkIsManifest checks if the content is manifest.
func (s *traversalService) checkIsManifest(
ctx context.Context,
reference swarm.Address,
e *entry.Entry,
metadata *entry.Metadata,
) (isManifest bool, m manifest.Interface, err error) {
// NOTE: 'encrypted' parameter only used for saving manifest
m, err = manifest.NewManifestReference(
metadata.MimeType,
e.Reference(),
m, err = manifest.NewDefaultManifestReference(
reference,
loadsave.New(s.storer, storage.ModePutRequest, false),
)
if err != nil {
......@@ -339,9 +119,7 @@ func (s *traversalService) checkIsManifest(
err = fmt.Errorf("traversal: read manifest: %s: %w", reference, err)
return
}
isManifest = true
return
}
......
This diff is collapsed.
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment