Commit a96c4c23 authored by Nemanja Zbiljić's avatar Nemanja Zbiljić Committed by GitHub

Random manifest obfuscation key only when encrypting (#1065)

parent 96734961
......@@ -10,7 +10,7 @@ require (
github.com/ethereum/go-ethereum v1.9.20
github.com/ethersphere/bmt v0.1.4
github.com/ethersphere/langos v1.0.0
github.com/ethersphere/manifest v0.3.5
github.com/ethersphere/manifest v0.3.6
github.com/ethersphere/sw3-bindings/v2 v2.1.0
github.com/gogo/protobuf v1.3.1
github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e // indirect
......
......@@ -167,8 +167,8 @@ github.com/ethersphere/bmt v0.1.4 h1:+rkWYNtMgDx6bkNqGdWu+U9DgGI1rRZplpSW3YhBr1Q
github.com/ethersphere/bmt v0.1.4/go.mod h1:Yd8ft1U69WDuHevZc/rwPxUv1rzPSMpMnS6xbU53aY8=
github.com/ethersphere/langos v1.0.0 h1:NBtNKzXTTRSue95uOlzPN4py7Aofs0xWPzyj4AI1Vcc=
github.com/ethersphere/langos v1.0.0/go.mod h1:dlcN2j4O8sQ+BlCaxeBu43bgr4RQ+inJ+pHwLeZg5Tw=
github.com/ethersphere/manifest v0.3.5 h1:/UMN4X4eKyTCARS9dv2HqqdFCJI2Emu09tivYsp5FZM=
github.com/ethersphere/manifest v0.3.5/go.mod h1:frSxQFT67hQvmTN5CBtgVuqHzGQpg0V0oIIm/B3Am+U=
github.com/ethersphere/manifest v0.3.6 h1:38WgYoXAQyC2lrSTArj+HM62AecX8JfUn1oVr1q+CVg=
github.com/ethersphere/manifest v0.3.6/go.mod h1:frSxQFT67hQvmTN5CBtgVuqHzGQpg0V0oIIm/B3Am+U=
github.com/ethersphere/sw3-bindings/v2 v2.1.0 h1:QefDtzU94UelICMPXWr7m52E2oj6r018Yc0XLoCWOxw=
github.com/ethersphere/sw3-bindings/v2 v2.1.0/go.mod h1:ozMVBZZlAirS/FcUpFwzV60v8gC0nVbA/5ZXtCX3xCc=
github.com/fatih/color v1.3.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
......
......@@ -99,7 +99,7 @@ func TestBzz(t *testing.T) {
}
// save manifest
m, err := manifest.NewDefaultManifest(loadsave.New(storer, storage.ModePutRequest, false))
m, err := manifest.NewDefaultManifest(loadsave.New(storer, storage.ModePutRequest, false), false)
if err != nil {
t.Fatal(err)
}
......
......@@ -62,8 +62,9 @@ func (s *server) dirUploadHandler(w http.ResponseWriter, r *http.Request) {
// Add the tag to the context
ctx := sctx.SetTag(r.Context(), tag)
p := requestPipelineFn(s.Storer, r)
l := loadsave.New(s.Storer, requestModePut(r), requestEncrypt(r))
reference, err := storeDir(ctx, r.Body, s.Logger, p, l, r.Header.Get(SwarmIndexDocumentHeader), r.Header.Get(SwarmErrorDocumentHeader))
encrypt := requestEncrypt(r)
l := loadsave.New(s.Storer, requestModePut(r), encrypt)
reference, err := storeDir(ctx, encrypt, r.Body, s.Logger, p, l, r.Header.Get(SwarmIndexDocumentHeader), r.Header.Get(SwarmErrorDocumentHeader))
if err != nil {
logger.Debugf("dir upload: store dir err: %v", err)
logger.Errorf("dir upload: store dir")
......@@ -103,10 +104,10 @@ func validateRequest(r *http.Request) error {
// storeDir stores all files recursively contained in the directory given as a tar
// it returns the hash for the uploaded manifest corresponding to the uploaded dir
func storeDir(ctx context.Context, reader io.ReadCloser, log logging.Logger, p pipelineFunc, ls file.LoadSaver, indexFilename string, errorFilename string) (swarm.Address, error) {
func storeDir(ctx context.Context, encrypt bool, reader io.ReadCloser, log logging.Logger, p pipelineFunc, ls file.LoadSaver, indexFilename string, errorFilename string) (swarm.Address, error) {
logger := tracing.NewLoggerWithTraceID(ctx, log)
dirManifest, err := manifest.NewDefaultManifest(ls)
dirManifest, err := manifest.NewDefaultManifest(ls, encrypt)
if err != nil {
return swarm.ZeroAddress, err
}
......
......@@ -92,6 +92,8 @@ func TestDirs(t *testing.T) {
// valid tars
for _, tc := range []struct {
name string
expectedReference swarm.Address
encrypt bool
wantIndexFilename string
wantErrorFilename string
indexFilenameOption jsonhttptest.Option
......@@ -99,7 +101,8 @@ func TestDirs(t *testing.T) {
files []f // files in dir for test case
}{
{
name: "non-nested files without extension",
name: "non-nested files without extension",
expectedReference: swarm.MustParseHexAddress("126140bb0a33d62c4efb0523db2c26be849fcf458504618de785e2a219bad374"),
files: []f{
{
data: []byte("first file data"),
......@@ -122,7 +125,8 @@ func TestDirs(t *testing.T) {
},
},
{
name: "nested files with extension",
name: "nested files with extension",
expectedReference: swarm.MustParseHexAddress("cad4b3847bd59532d9e73623d67c52e0c8d4e017d308bbaecb54f2866a91769d"),
files: []f{
{
data: []byte("robots text"),
......@@ -154,7 +158,8 @@ func TestDirs(t *testing.T) {
},
},
{
name: "no index filename",
name: "no index filename",
expectedReference: swarm.MustParseHexAddress("a85aaea6a34a5c7127a3546196f2111f866fe369c6d6562ed5d3313a99388c03"),
files: []f{
{
data: []byte("<h1>Swarm"),
......@@ -169,6 +174,7 @@ func TestDirs(t *testing.T) {
},
{
name: "explicit index filename",
expectedReference: swarm.MustParseHexAddress("7d41402220f8e397ddf74d0cf4ac2055e753102bde0d622c45b03cea2b28b023"),
wantIndexFilename: "index.html",
indexFilenameOption: jsonhttptest.WithRequestHeader(api.SwarmIndexDocumentHeader, "index.html"),
files: []f{
......@@ -185,6 +191,7 @@ func TestDirs(t *testing.T) {
},
{
name: "nested index filename",
expectedReference: swarm.MustParseHexAddress("45249cf9caad842b31b29b831a1ff12aa2b711e7c282fa7a5f8c0fb544143421"),
wantIndexFilename: "index.html",
indexFilenameOption: jsonhttptest.WithRequestHeader(api.SwarmIndexDocumentHeader, "index.html"),
files: []f{
......@@ -201,6 +208,7 @@ func TestDirs(t *testing.T) {
},
{
name: "explicit index and error filename",
expectedReference: swarm.MustParseHexAddress("2046a4f758e2c0579ab923206a13fb041cec0925a6396f4f772c7ce859b8ca42"),
wantIndexFilename: "index.html",
wantErrorFilename: "error.html",
indexFilenameOption: jsonhttptest.WithRequestHeader(api.SwarmIndexDocumentHeader, "index.html"),
......@@ -227,7 +235,8 @@ func TestDirs(t *testing.T) {
},
},
{
name: "invalid archive paths",
name: "invalid archive paths",
expectedReference: swarm.MustParseHexAddress("6e6adb1ce936990cf1b7ecf8f01a8e3e8f939375b9bddb3d666151e0bdc08d4e"),
files: []f{
{
data: []byte("<h1>Swarm"),
......@@ -253,6 +262,18 @@ Disallow: /`),
},
},
},
{
name: "encrypted",
encrypt: true,
files: []f{
{
data: []byte("<h1>Swarm"),
name: "index.html",
dir: "",
filePath: "./index.html",
},
},
},
} {
t.Run(tc.name, func(t *testing.T) {
// tar all the test case files
......@@ -271,25 +292,33 @@ Disallow: /`),
if tc.errorFilenameOption != nil {
options = append(options, tc.errorFilenameOption)
}
if tc.encrypt {
options = append(options, jsonhttptest.WithRequestHeader(api.SwarmEncryptHeader, "true"))
}
// verify directory tar upload response
jsonhttptest.Request(t, client, http.MethodPost, dirUploadResource, http.StatusOK, options...)
read := bytes.NewReader(respBytes)
// get the reference as everytime it will change because of random encryption key
// get the reference
var resp api.FileUploadResponse
err := json.NewDecoder(read).Decode(&resp)
if err != nil {
t.Fatal(err)
}
// NOTE: reference will be different each time, due to manifest randomness
if resp.Reference.String() == "" {
t.Fatalf("expected file reference, did not got any")
}
// NOTE: reference will be different each time when encryption is enabled
if !tc.encrypt {
if !resp.Reference.Equal(tc.expectedReference) {
t.Fatalf("expected root reference to match %s, got %s", tc.expectedReference, resp.Reference)
}
}
// read manifest metadata
j, _, err := joiner.New(context.Background(), storer, resp.Reference)
if err != nil {
......@@ -327,8 +356,10 @@ Disallow: /`),
fileReference := entry.Reference()
if !bytes.Equal(file.reference.Bytes(), fileReference.Bytes()) {
t.Fatalf("expected file reference to match %s, got %s", file.reference, fileReference)
if !tc.encrypt {
if !bytes.Equal(file.reference.Bytes(), fileReference.Bytes()) {
t.Fatalf("expected file reference to match %s, got %s", file.reference, fileReference)
}
}
jsonhttptest.Request(t, client, http.MethodGet, fileDownloadResource(fileReference.String()), http.StatusOK,
......
......@@ -6,7 +6,6 @@ package api_test
import (
"bytes"
"encoding/json"
"fmt"
"io"
"io/ioutil"
......@@ -281,7 +280,7 @@ func TestRangeRequests(t *testing.T) {
uploadEndpoint: "/dirs",
downloadEndpoint: "/bzz",
filepath: "/ipsum/lorem.txt",
reference: "",
reference: "96c68b99304b0868189e5c1d6c10be1984d93e88aab0384907f6b8814f60150b",
reader: tarFiles(t, []f{
{
data: data,
......@@ -355,25 +354,6 @@ func TestRangeRequests(t *testing.T) {
jsonhttptest.WithPutResponseBody(&respBytes),
)
if uploadReference == "" {
// NOTE: reference will be different each time, due to manifest randomness
read := bytes.NewReader(respBytes)
// get the reference as everytime it will change because of random encryption key
var resp api.FileUploadResponse
err := json.NewDecoder(read).Decode(&resp)
if err != nil {
t.Fatal(err)
}
if resp.Reference.String() == "" {
t.Fatalf("expected file reference, did not got any")
}
uploadReference = resp.Reference.String()
}
for _, tc := range ranges {
t.Run(tc.name, func(t *testing.T) {
rangeHeader, want := createRangeHeader(data, tc.ranges)
......
......@@ -20,7 +20,6 @@ import (
"github.com/ethersphere/bee/pkg/swarm"
"github.com/ethersphere/bee/pkg/tags"
"github.com/ethersphere/bee/pkg/traversal"
"github.com/ethersphere/manifest/mantaray"
)
func TestPinBzzHandler(t *testing.T) {
......@@ -41,15 +40,6 @@ func TestPinBzzHandler(t *testing.T) {
})
)
var (
obfuscationKey = make([]byte, 32)
obfuscationKeyFn = func(p []byte) (n int, err error) {
n = copy(p, obfuscationKey)
return
}
)
mantaray.SetObfuscationKeyFn(obfuscationKeyFn)
t.Run("pin-bzz-1", func(t *testing.T) {
files := []f{
{
......@@ -89,7 +79,6 @@ func TestPinBzzHandler(t *testing.T) {
read := bytes.NewReader(respBytes)
// get the reference as everytime it will change because of random encryption key
var resp api.ListPinnedChunksResponse
err := json.NewDecoder(read).Decode(&resp)
if err != nil {
......
......@@ -6,7 +6,6 @@ package api_test
import (
"bytes"
"encoding/json"
"fmt"
"io/ioutil"
"net/http"
......@@ -306,32 +305,23 @@ func TestTags(t *testing.T) {
t.Run("dir tags", func(t *testing.T) {
// upload a dir without supplying tag
tarReader := tarFiles(t, []f{{
data: []byte("some data"),
data: []byte("some dir data"),
name: "binary-file",
}})
var respBytes []byte
expectedHash := swarm.MustParseHexAddress("3dc643abeb3db60a4dfb72008b577dd9a573abaa74c6afe37a75c63ceea829f6")
expectedResponse := api.FileUploadResponse{Reference: expectedHash}
respHeaders := jsonhttptest.Request(t, client, http.MethodPost, dirResource, http.StatusOK,
jsonhttptest.WithRequestBody(tarReader),
jsonhttptest.WithExpectedJSONResponse(expectedResponse),
jsonhttptest.WithRequestHeader("Content-Type", api.ContentTypeTar),
jsonhttptest.WithPutResponseBody(&respBytes),
)
read := bytes.NewReader(respBytes)
// get the reference as everytime it will change because of random encryption key
var resp api.FileUploadResponse
err := json.NewDecoder(read).Decode(&resp)
if err != nil {
t.Fatal(err)
}
tagId, err := strconv.Atoi(respHeaders.Get(api.SwarmTagUidHeader))
if err != nil {
t.Fatal(err)
}
tagValueTest(t, uint32(tagId), 7, 7, 1, 0, 0, 7, resp.Reference, client)
tagValueTest(t, uint32(tagId), 7, 7, 0, 0, 0, 7, expectedHash, client)
})
t.Run("bytes tags", func(t *testing.T) {
......
......@@ -59,20 +59,24 @@ type Entry interface {
}
// NewDefaultManifest creates a new manifest with default type.
func NewDefaultManifest(ls file.LoadSaver) (Interface, error) {
return NewManifest(DefaultManifestType, ls)
func NewDefaultManifest(
ls file.LoadSaver,
encrypted bool,
) (Interface, error) {
return NewManifest(DefaultManifestType, ls, encrypted)
}
// NewManifest creates a new manifest.
func NewManifest(
manifestType string,
ls file.LoadSaver,
encrypted bool,
) (Interface, error) {
switch manifestType {
case ManifestSimpleContentType:
return NewSimpleManifest(ls)
case ManifestMantarayContentType:
return NewMantarayManifest(ls)
return NewMantarayManifest(ls, encrypted)
default:
return nil, ErrInvalidManifestType
}
......
......@@ -27,26 +27,19 @@ type mantarayManifest struct {
}
// NewMantarayManifest creates a new mantaray-based manifest.
func NewMantarayManifest(ls file.LoadSaver) (Interface, error) {
return &mantarayManifest{
trie: mantaray.New(),
ls: ls,
}, nil
}
// NewMantarayManifestWithObfuscationKeyFn creates a new mantaray-based manifest
// with configured obfuscation key
//
// NOTE: This should only be used in tests.
func NewMantarayManifestWithObfuscationKeyFn(
func NewMantarayManifest(
ls file.LoadSaver,
obfuscationKeyFn func([]byte) (int, error),
encrypted bool,
) (Interface, error) {
mm := &mantarayManifest{
trie: mantaray.New(),
ls: ls,
}
mantaray.SetObfuscationKeyFn(obfuscationKeyFn)
// use empty obfuscation key if not encrypting
if !encrypted {
// NOTE: it will be copied to all trie nodes
mm.trie.SetObfuscationKey(mantaray.ZeroObfuscationKey)
}
return mm, nil
}
......
......@@ -265,14 +265,6 @@ func TestTraversalManifest(t *testing.T) {
return traversalService.TraverseManifestAddresses
}
var (
obfuscationKey = make([]byte, 32)
obfuscationKeyFn = func(p []byte) (n int, err error) {
n = copy(p, obfuscationKey)
return
}
)
testCases := []struct {
manifestType string
files []file
......@@ -470,7 +462,7 @@ func TestTraversalManifest(t *testing.T) {
t.Fatal(err)
}
case manifest.ManifestMantarayContentType:
dirManifest, err = manifest.NewMantarayManifestWithObfuscationKeyFn(ls, obfuscationKeyFn)
dirManifest, err = manifest.NewMantarayManifest(ls, false)
if err != nil {
t.Fatal(err)
}
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment