Skip to content

M
mybee
Commit 34cb57c9 authored by acud's avatar acud Committed by GitHub
Browse files
Options

manifest: import manifest repo into bee (#1519)

parent 4254a3e3
Expand all Hide whitespace changes
Inline Side-by-side
Showing with 2492 additions and 10 deletions
go.mod
View file @ 34cb57c9
......@@ -10,7 +10,6 @@ require (
github.com/ethereum/go-ethereum v1.9.23
github.com/ethersphere/bmt v0.1.4
github.com/ethersphere/langos v1.0.0
github.com/ethersphere/manifest v0.3.6
github.com/ethersphere/sw3-bindings/v3 v3.0.3
github.com/foxcpp/go-mockdns v0.0.0-20201212160233-ede2f9158d15
github.com/gogo/protobuf v1.3.1
......
go.sum
View file @ 34cb57c9
......@@ -169,8 +169,6 @@ github.com/ethersphere/bmt v0.1.4 h1:+rkWYNtMgDx6bkNqGdWu+U9DgGI1rRZplpSW3YhBr1Q
github.com/ethersphere/bmt v0.1.4/go.mod h1:Yd8ft1U69WDuHevZc/rwPxUv1rzPSMpMnS6xbU53aY8=
github.com/ethersphere/langos v1.0.0 h1:NBtNKzXTTRSue95uOlzPN4py7Aofs0xWPzyj4AI1Vcc=
github.com/ethersphere/langos v1.0.0/go.mod h1:dlcN2j4O8sQ+BlCaxeBu43bgr4RQ+inJ+pHwLeZg5Tw=
github.com/ethersphere/manifest v0.3.6 h1:38WgYoXAQyC2lrSTArj+HM62AecX8JfUn1oVr1q+CVg=
github.com/ethersphere/manifest v0.3.6/go.mod h1:frSxQFT67hQvmTN5CBtgVuqHzGQpg0V0oIIm/B3Am+U=
github.com/ethersphere/sw3-bindings/v3 v3.0.3 h1:iENjwaFFqu9hM9LrL8H0yRgToq9xFwLAr9XXvOt9LFM=
github.com/ethersphere/sw3-bindings/v3 v3.0.3/go.mod h1:EEn7sxejLPj6p1oDT/YGrjDfNV8z6PWcd4DviE0hOIk=
github.com/fatih/color v1.3.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
......@@ -1147,8 +1145,6 @@ golang.org/x/crypto v0.0.0-20200510223506-06a226fb4e37/go.mod h1:LzIPMQfyMNhhGPh
golang.org/x/crypto v0.0.0-20200602180216-279210d13fed/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 h1:psW17arqaxU48Z5kZ0CQnkZWQJsqcURM6tKiBApRjXI=
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/crypto v0.0.0-20200728195943-123391ffb6de h1:ikNHVSjEfnvz6sxdSPCaPt572qowuyMDMJLLm3Db3ig=
golang.org/x/crypto v0.0.0-20200728195943-123391ffb6de/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/crypto v0.0.0-20201016220609-9e8e0b390897/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad h1:DN0cp81fZ3njFcrLCytUHRSUkqBjfTo4Tx9RJTWs0EY=
......
pkg/api/bzz.go
View file @ 34cb57c9
......@@ -26,11 +26,11 @@ import (
"github.com/ethersphere/bee/pkg/file/loadsave"
"github.com/ethersphere/bee/pkg/jsonhttp"
"github.com/ethersphere/bee/pkg/manifest"
"github.com/ethersphere/bee/pkg/manifest/mantaray"
"github.com/ethersphere/bee/pkg/sctx"
"github.com/ethersphere/bee/pkg/storage"
"github.com/ethersphere/bee/pkg/swarm"
"github.com/ethersphere/bee/pkg/tracing"
"github.com/ethersphere/manifest/mantaray"
)
func (s *server) bzzDownloadHandler(w http.ResponseWriter, r *http.Request) {
......
pkg/manifest/manifest.go
View file @ 34cb57c9
......@@ -3,8 +3,7 @@
// license that can be found in the LICENSE file.
// Package manifest contains the abstractions needed for
// collection representation in Swarm. It uses implementations
// in ethersphere/manifest repo under the hood.
// collection representation in Swarm.
package manifest
import (
......
pkg/manifest/mantaray.go
View file @ 34cb57c9
......@@ -10,8 +10,8 @@ import (
"fmt"
"github.com/ethersphere/bee/pkg/file"
"github.com/ethersphere/bee/pkg/manifest/mantaray"
"github.com/ethersphere/bee/pkg/swarm"
"github.com/ethersphere/manifest/mantaray"
)
const (
......
pkg/manifest/mantaray/docs/format/node.md 0 → 100644
View file @ 34cb57c9
# node binary format
The following describes the format of a node binary format.
```
┌────────────────────────────────┐
│ obfuscationKey <32 byte> │
├────────────────────────────────┤
│ hash("mantaray:0.1") <31 byte> │
├────────────────────────────────┤
│ refBytesSize <1 byte> │
├────────────────────────────────┤
│ entry <32/64 byte> │
├────────────────────────────────┤
│ forksIndexBytes <32 byte> │
├────────────────────────────────┤
│ ┌────────────────────────────┐ │
│ │ Fork 1 │ │
│ ├────────────────────────────┤ │
│ │ ... │ │
│ ├────────────────────────────┤ │
│ │ Fork N │ │
│ └────────────────────────────┘ │
└────────────────────────────────┘
```
## Fork
```
┌───────────────────┬───────────────────────┬──────────────────┐
│ nodeType <1 byte> │ prefixLength <1 byte> │ prefix <30 byte> │
├───────────────────┴───────────────────────┴──────────────────┤
│ reference <32/64 bytes> │
│ │
└──────────────────────────────────────────────────────────────┘
```
### Fork with metadata
```
┌───────────────────┬───────────────────────┬──────────────────┐
│ nodeType <1 byte> │ prefixLength <1 byte> │ prefix <30 byte> │
├───────────────────┴───────────────────────┴──────────────────┤
│ reference <32/64 bytes> │
│ │
├─────────────────────────────┬────────────────────────────────┤
│ metadataBytesSize <2 bytes> │ metadataBytes <varlen> │
├─────────────────────────────┘ │
│ │
└──────────────────────────────────────────────────────────────┘
```
pkg/manifest/mantaray/marshal.go 0 → 100644
View file @ 34cb57c9
This diff is collapsed.
pkg/manifest/mantaray/marshal_test.go 0 → 100644
View file @ 34cb57c9
// Copyright 2020 The Swarm Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package mantaray
import (
"bytes"
"context"
"encoding/hex"
mrand "math/rand"
"reflect"
"testing"
"golang.org/x/crypto/sha3"
)
const testMarshalOutput01 = "52fdfc072182654f163f5f0f9a621d729566c74d10037c4d7bbb0407d1e2c64950ac787fbce1061870e8d34e0a638bc7e812c7ca4ebd31d626a572ba47b06f6952fdfc072182654f163f5f0f9a621d729566c74d10037c4d7bbb0407d1e2c64952fdfc072102654f163f5f0fa0621d729566c74d10037c4d7bbb0407d1e2c64950fcd3072182654f163f5f0f9a621d729566c74d10037c4d7bbb0407d1e2c64952fdfc072182654f163f5f0f9a621d729566c74d10037c4d7bbb0407d1e2c64950f89d6640e3044f163f5f0f9a621d729566c74d10037c4d7bbb0407d1e2c64952fdfc072182654f163f5f0f9a621d729566c74d10037c4d7bbb0407d1e2c64850ff9f642182654f163f5f0f9a621d729566c74d10037c4d7bbb0407d1e2c64952fdfc072182654f163f5f0f9a621d729566c74d10037c4d7bbb0407d1e2c64b50fc98072182654f163f5f0f9a621d729566c74d10037c4d7bbb0407d1e2c64952fdfc072182654f163f5f0f9a621d729566c74d10037c4d7bbb0407d1e2c64a50ff99622182654f163f5f0f9a621d729566c74d10037c4d7bbb0407d1e2c64952fdfc072182654f163f5f0f9a621d729566c74d10037c4d7bbb0407d1e2c64d"
const testMarshalOutput02 = "52fdfc072182654f163f5f0f9a621d729566c74d10037c4d7bbb0407d1e2c64905954fb18659339d0b25e0fb9723d3cd5d528fb3c8d495fd157bd7b7a210496952fdfc072182654f163f5f0f9a621d729566c74d10037c4d7bbb0407d1e2c64952fdfc072102654f163f5f0fa0621d729566c74d10037c4d7bbb0407d1e2c64940fcd3072182654f163f5f0f9a621d729566c74d10037c4d7bbb0407d1e2c64952fdfc072182654f163f5f0f9a621d729566c74d10037c4d7bbb0407d1e2c64952e3872548ec012a6e123b60f9177017fb12e57732621d2c1ada267adbe8cc4350f89d6640e3044f163f5f0f9a621d729566c74d10037c4d7bbb0407d1e2c64952fdfc072182654f163f5f0f9a621d729566c74d10037c4d7bbb0407d1e2c64850ff9f642182654f163f5f0f9a621d729566c74d10037c4d7bbb0407d1e2c64952fdfc072182654f163f5f0f9a621d729566c74d10037c4d7bbb0407d1e2c64b50fc98072182654f163f5f0f9a621d729566c74d10037c4d7bbb0407d1e2c64952fdfc072182654f163f5f0f9a621d729566c74d10037c4d7bbb0407d1e2c64a50ff99622182654f163f5f0f9a621d729566c74d10037c4d7bbb0407d1e2c64952fdfc072182654f163f5f0f9a621d729566c74d10037c4d7bbb0407d1e2c64d"
var testEntries = []nodeEntry{
{
path: []byte("/"),
metadata: map[string]string{
"index-document": "aaaaa",
},
},
{
path: []byte("aaaaa"),
},
{
path: []byte("cc"),
},
{
path: []byte("d"),
},
{
path: []byte("ee"),
},
}
func init() {
obfuscationKeyFn = mrand.Read
}
func TestVersion01(t *testing.T) {
hasher := sha3.NewLegacyKeccak256()
_, err := hasher.Write([]byte(version01String))
if err != nil {
t.Fatal(err)
}
sum := hasher.Sum(nil)
sumHex := hex.EncodeToString(sum)
if version01HashString != sumHex {
t.Fatalf("expecting version hash '%s', got '%s'", version01String, sumHex)
}
}
func TestVersion02(t *testing.T) {
hasher := sha3.NewLegacyKeccak256()
_, err := hasher.Write([]byte(version02String))
if err != nil {
t.Fatal(err)
}
sum := hasher.Sum(nil)
sumHex := hex.EncodeToString(sum)
if version02HashString != sumHex {
t.Fatalf("expecting version hash '%s', got '%s'", version02String, sumHex)
}
}
func TestUnmarshal01(t *testing.T) {
input, _ := hex.DecodeString(testMarshalOutput01)
n := &Node{}
err := n.UnmarshalBinary(input)
if err != nil {
t.Fatalf("expected no error marshaling, got %v", err)
}
expEncrypted := testMarshalOutput01[128:192]
// perform XOR decryption
expEncryptedBytes, _ := hex.DecodeString(expEncrypted)
expBytes := encryptDecrypt(expEncryptedBytes, n.obfuscationKey)
exp := hex.EncodeToString(expBytes)
if hex.EncodeToString(n.entry) != exp {
t.Fatalf("expected %x, got %x", exp, n.entry)
}
if len(testEntries) != len(n.forks) {
t.Fatalf("expected %d forks, got %d", len(testEntries), len(n.forks))
}
for _, entry := range testEntries {
prefix := entry.path
f := n.forks[prefix[0]]
if f == nil {
t.Fatalf("expected to have fork on byte %x", prefix[:1])
}
if !bytes.Equal(f.prefix, prefix) {
t.Fatalf("expected prefix for byte %x to match %s, got %s", prefix[:1], prefix, f.prefix)
}
}
}
func TestUnmarshal02(t *testing.T) {
input, _ := hex.DecodeString(testMarshalOutput02)
n := &Node{}
err := n.UnmarshalBinary(input)
if err != nil {
t.Fatalf("expected no error marshaling, got %v", err)
}
expEncrypted := testMarshalOutput02[128:192]
// perform XOR decryption
expEncryptedBytes, _ := hex.DecodeString(expEncrypted)
expBytes := encryptDecrypt(expEncryptedBytes, n.obfuscationKey)
exp := hex.EncodeToString(expBytes)
if hex.EncodeToString(n.entry) != exp {
t.Fatalf("expected %x, got %x", exp, n.entry)
}
if len(testEntries) != len(n.forks) {
t.Fatalf("expected %d forks, got %d", len(testEntries), len(n.forks))
}
for _, entry := range testEntries {
prefix := entry.path
f := n.forks[prefix[0]]
if f == nil {
t.Fatalf("expected to have fork on byte %x", prefix[:1])
}
if !bytes.Equal(f.prefix, prefix) {
t.Fatalf("expected prefix for byte %x to match %s, got %s", prefix[:1], prefix, f.prefix)
}
if len(entry.metadata) > 0 {
if !reflect.DeepEqual(entry.metadata, f.metadata) {
t.Fatalf("expected metadata for byte %x to match %s, got %s", prefix[:1], entry.metadata, f.metadata)
}
}
}
}
func TestMarshal(t *testing.T) {
ctx := context.Background()
n := New()
defer func(r func(*fork) []byte) { refBytes = r }(refBytes)
i := uint8(0)
refBytes = func(*fork) []byte {
b := make([]byte, 32)
b[31] = i
i++
return b
}
for i := 0; i < len(testEntries); i++ {
c := testEntries[i].path
e := testEntries[i].entry
if len(e) == 0 {
e = append(make([]byte, 32-len(c)), c...)
}
m := testEntries[i].metadata
err := n.Add(ctx, c, e, m, nil)
if err != nil {
t.Fatalf("expected no error, got %v", err)
}
}
b, err := n.MarshalBinary()
if err != nil {
t.Fatalf("expected no error marshaling, got %v", err)
}
exp, _ := hex.DecodeString(testMarshalOutput02)
if !bytes.Equal(b, exp) {
t.Fatalf("expected marshalled output to match %x, got %x", exp, b)
}
// n = &Node{}
// err = n.UnmarshalBinary(b)
// if err != nil {
// t.Fatalf("expected no error unmarshaling, got %v", err)
// }
// for j := 0; j < len(testCases); j++ {
// d := testCases[j]
// m, err := n.Lookup(d, nil)
// if err != nil {
// t.Fatalf("expected no error, got %v", err)
// }
// if !bytes.Equal(m, d) {
// t.Fatalf("expected value %x, got %x", d, m)
// }
// }
}
pkg/manifest/mantaray/node.go 0 → 100644
View file @ 34cb57c9
// Copyright 2020 The Swarm Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package mantaray
import (
"bytes"
"context"
"errors"
"fmt"
)
const (
PathSeparator = '/' // path separator
)
var (
ZeroObfuscationKey []byte
)
func init() {
ZeroObfuscationKey = make([]byte, 32)
}
// Error used when lookup path does not match
var (
ErrNotFound = errors.New("not found")
ErrEmptyPath = errors.New("empty path")
ErrMetadataTooLarge = errors.New("metadata too large")
)
// Node represents a mantaray Node
type Node struct {
nodeType uint8
refBytesSize int
obfuscationKey []byte
ref []byte // reference to uninstantiated Node persisted serialised
entry []byte
metadata map[string]string
forks map[byte]*fork
}
type fork struct {
prefix []byte // the non-branching part of the subpath
*Node // in memory structure that represents the Node
}
const (
nodeTypeValue = uint8(2)
nodeTypeEdge = uint8(4)
nodeTypeWithPathSeparator = uint8(8)
nodeTypeWithMetadata = uint8(16)
nodeTypeMask = uint8(255)
)
func nodeTypeIsWithMetadataType(nodeType uint8) bool {
return nodeType&nodeTypeWithMetadata == nodeTypeWithMetadata
}
// NewNodeRef is the exported Node constructor used to represent manifests by reference
func NewNodeRef(ref []byte) *Node {
return &Node{ref: ref}
}
// New is the constructor for in-memory Node structure
func New() *Node {
return &Node{forks: make(map[byte]*fork)}
}
func notFound(path []byte) error {
return fmt.Errorf("entry on '%s' ('%x'): %w", path, path, ErrNotFound)
}
// IsValueType returns true if the node contains entry.
func (n *Node) IsValueType() bool {
return n.nodeType&nodeTypeValue == nodeTypeValue
}
// IsEdgeType returns true if the node forks into other nodes.
func (n *Node) IsEdgeType() bool {
return n.nodeType&nodeTypeEdge == nodeTypeEdge
}
// IsWithPathSeparatorType returns true if the node path contains separator character.
func (n *Node) IsWithPathSeparatorType() bool {
return n.nodeType&nodeTypeWithPathSeparator == nodeTypeWithPathSeparator
}
// IsWithMetadataType returns true if the node contains metadata.
func (n *Node) IsWithMetadataType() bool {
return n.nodeType&nodeTypeWithMetadata == nodeTypeWithMetadata
}
func (n *Node) makeValue() {
n.nodeType = n.nodeType | nodeTypeValue
}
func (n *Node) makeEdge() {
n.nodeType = n.nodeType | nodeTypeEdge
}
func (n *Node) makeWithPathSeparator() {
n.nodeType = n.nodeType | nodeTypeWithPathSeparator
}
func (n *Node) makeWithMetadata() {
n.nodeType = n.nodeType | nodeTypeWithMetadata
}
//nolint,unused
func (n *Node) makeNotValue() { // skipcq: SCC-U1000
n.nodeType = (nodeTypeMask ^ nodeTypeValue) & n.nodeType
}
//nolint,unused
func (n *Node) makeNotEdge() { // skipcq: SCC-U1000
n.nodeType = (nodeTypeMask ^ nodeTypeEdge) & n.nodeType
}
func (n *Node) makeNotWithPathSeparator() {
n.nodeType = (nodeTypeMask ^ nodeTypeWithPathSeparator) & n.nodeType
}
//nolint,unused
func (n *Node) makeNotWithMetadata() { // skipcq: SCC-U1000
n.nodeType = (nodeTypeMask ^ nodeTypeWithMetadata) & n.nodeType
}
func (n *Node) SetObfuscationKey(obfuscationKey []byte) {
bytes := make([]byte, 32)
copy(bytes, obfuscationKey)
n.obfuscationKey = bytes
}
// Reference returns the address of the mantaray node if saved.
func (n *Node) Reference() []byte {
return n.ref
}
// Entry returns the value stored on the specific path.
func (n *Node) Entry() []byte {
return n.entry
}
// Metadata returns the metadata stored on the specific path.
func (n *Node) Metadata() map[string]string {
return n.metadata
}
// LookupNode finds the node for a path or returns error if not found
func (n *Node) LookupNode(ctx context.Context, path []byte, l Loader) (*Node, error) {
select {
case <-ctx.Done():
return nil, ctx.Err()
default:
}
if n.forks == nil {
if err := n.load(ctx, l); err != nil {
return nil, err
}
}
if len(path) == 0 {
return n, nil
}
f := n.forks[path[0]]
if f == nil {
return nil, notFound(path)
}
c := common(f.prefix, path)
if len(c) == len(f.prefix) {
return f.Node.LookupNode(ctx, path[len(c):], l)
}
return nil, notFound(path)
}
// Lookup finds the entry for a path or returns error if not found
func (n *Node) Lookup(ctx context.Context, path []byte, l Loader) ([]byte, error) {
node, err := n.LookupNode(ctx, path, l)
if err != nil {
return nil, err
}
return node.entry, nil
}
// Add adds an entry to the path
func (n *Node) Add(ctx context.Context, path, entry []byte, metadata map[string]string, ls LoadSaver) error {
select {
case <-ctx.Done():
return ctx.Err()
default:
}
if n.refBytesSize == 0 {
if len(entry) > 256 {
return fmt.Errorf("node entry size > 256: %d", len(entry))
}
// empty entry for directories
if len(entry) > 0 {
n.refBytesSize = len(entry)
}
} else if len(entry) > 0 && n.refBytesSize != len(entry) {
return fmt.Errorf("invalid entry size: %d, expected: %d", len(entry), n.refBytesSize)
}
if len(path) == 0 {
n.entry = entry
if len(metadata) > 0 {
n.metadata = metadata
n.makeWithMetadata()
}
n.ref = nil
return nil
}
if n.forks == nil {
if err := n.load(ctx, ls); err != nil {
return err
}
n.ref = nil
}
f := n.forks[path[0]]
if f == nil {
nn := New()
if len(n.obfuscationKey) > 0 {
nn.SetObfuscationKey(n.obfuscationKey)
}
nn.refBytesSize = n.refBytesSize
// check for prefix size limit
if len(path) > nodePrefixMaxSize {
prefix := path[:nodePrefixMaxSize]
rest := path[nodePrefixMaxSize:]
err := nn.Add(ctx, rest, entry, metadata, ls)
if err != nil {
return err
}
nn.updateIsWithPathSeparator(prefix)
n.forks[path[0]] = &fork{prefix, nn}
n.makeEdge()
return nil
}
nn.entry = entry
if len(metadata) > 0 {
nn.metadata = metadata
nn.makeWithMetadata()
}
nn.makeValue()
nn.updateIsWithPathSeparator(path)
n.forks[path[0]] = &fork{path, nn}
n.makeEdge()
return nil
}
c := common(f.prefix, path)
rest := f.prefix[len(c):]
nn := f.Node
if len(rest) > 0 {
// move current common prefix node
nn = New()
if len(n.obfuscationKey) > 0 {
nn.SetObfuscationKey(n.obfuscationKey)
}
nn.refBytesSize = n.refBytesSize
f.Node.updateIsWithPathSeparator(rest)
nn.forks[rest[0]] = &fork{rest, f.Node}
nn.makeEdge()
// if common path is full path new node is value type
if len(path) == len(c) {
nn.makeValue()
}
}
// NOTE: special case on edge split
nn.updateIsWithPathSeparator(path)
// add new for shared prefix
err := nn.Add(ctx, path[len(c):], entry, metadata, ls)
if err != nil {
return err
}
n.forks[path[0]] = &fork{c, nn}
n.makeEdge()
return nil
}
func (n *Node) updateIsWithPathSeparator(path []byte) {
if bytes.IndexRune(path, PathSeparator) > 0 {
n.makeWithPathSeparator()
} else {
n.makeNotWithPathSeparator()
}
}
// Remove removes a path from the node
func (n *Node) Remove(ctx context.Context, path []byte, ls LoadSaver) error {
select {
case <-ctx.Done():
return ctx.Err()
default:
}
if len(path) == 0 {
return ErrEmptyPath
}
if n.forks == nil {
if err := n.load(ctx, ls); err != nil {
return err
}
}
f := n.forks[path[0]]
if f == nil {
return ErrNotFound
}
prefixIndex := bytes.Index(path, f.prefix)
if prefixIndex != 0 {
return ErrNotFound
}
rest := path[len(f.prefix):]
if len(rest) == 0 {
// full path matched
delete(n.forks, path[0])
return nil
}
return f.Node.Remove(ctx, rest, ls)
}
func common(a, b []byte) (c []byte) {
for i := 0; i < len(a) && i < len(b) && a[i] == b[i]; i++ {
c = append(c, a[i])
}
return c
}
// HasPrefix tests whether the node contains prefix path.
func (n *Node) HasPrefix(ctx context.Context, path []byte, l Loader) (bool, error) {
select {
case <-ctx.Done():
return false, ctx.Err()
default:
}
if n.forks == nil {
if err := n.load(ctx, l); err != nil {
return false, err
}
}
if len(path) == 0 {
return true, nil
}
f := n.forks[path[0]]
if f == nil {
return false, nil
}
c := common(f.prefix, path)
if len(c) == len(f.prefix) {
return f.Node.HasPrefix(ctx, path[len(c):], l)
}
if bytes.HasPrefix(f.prefix, path) {
return true, nil
}
return false, nil
}
pkg/manifest/mantaray/node_test.go 0 → 100644
View file @ 34cb57c9
// Copyright 2020 The Swarm Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package mantaray
import (
"bytes"
"context"
"errors"
"strconv"
"testing"
)
type nodeEntry struct {
path []byte
entry []byte
metadata map[string]string
}
func TestNilPath(t *testing.T) {
ctx := context.Background()
n := New()
_, err := n.Lookup(ctx, nil, nil)
if err != nil {
t.Fatalf("expected no error, got %v", err)
}
}
func TestAddAndLookup(t *testing.T) {
ctx := context.Background()
n := New()
testCases := [][]byte{
[]byte("aaaaaa"),
[]byte("aaaaab"),
[]byte("abbbb"),
[]byte("abbba"),
[]byte("bbbbba"),
[]byte("bbbaaa"),
[]byte("bbbaab"),
[]byte("aa"),
[]byte("b"),
}
for i := 0; i < len(testCases); i++ {
c := testCases[i]
e := append(make([]byte, 32-len(c)), c...)
err := n.Add(ctx, c, e, nil, nil)
if err != nil {
t.Fatalf("expected no error, got %v", err)
}
for j := 0; j < i; j++ {
d := testCases[j]
m, err := n.Lookup(ctx, d, nil)
if err != nil {
t.Fatalf("expected no error, got %v", err)
}
de := append(make([]byte, 32-len(d)), d...)
if !bytes.Equal(m, de) {
t.Fatalf("expected value %x, got %x", d, m)
}
}
}
}
func TestAddAndLookupNode(t *testing.T) {
for _, tc := range []struct {
name string
toAdd [][]byte
}{
{
name: "a",
toAdd: [][]byte{
[]byte("aaaaaa"),
[]byte("aaaaab"),
[]byte("abbbb"),
[]byte("abbba"),
[]byte("bbbbba"),
[]byte("bbbaaa"),
[]byte("bbbaab"),
[]byte("aa"),
[]byte("b"),
},
},
{
name: "simple",
toAdd: [][]byte{
[]byte("/"),
[]byte("index.html"),
[]byte("img/1.png"),
[]byte("img/2.png"),
[]byte("robots.txt"),
},
},
{
name: "nested-prefix-is-not-collapsed",
toAdd: [][]byte{
[]byte("index.html"),
[]byte("img/1.png"),
[]byte("img/2/test1.png"),
[]byte("img/2/test2.png"),
[]byte("robots.txt"),
},
},
{
name: "conflicting-path",
toAdd: [][]byte{
[]byte("app.js.map"),
[]byte("app.js"),
},
},
{
name: "spa-website",
toAdd: [][]byte{
[]byte("css/"),
[]byte("css/app.css"),
[]byte("favicon.ico"),
[]byte("img/"),
[]byte("img/logo.png"),
[]byte("index.html"),
[]byte("js/"),
[]byte("js/chunk-vendors.js.map"),
[]byte("js/chunk-vendors.js"),
[]byte("js/app.js.map"),
[]byte("js/app.js"),
},
},
} {
ctx := context.Background()
t.Run(tc.name, func(t *testing.T) {
n := New()
for i := 0; i < len(tc.toAdd); i++ {
c := tc.toAdd[i]
e := append(make([]byte, 32-len(c)), c...)
err := n.Add(ctx, c, e, nil, nil)
if err != nil {
t.Fatalf("expected no error, got %v", err)
}
for j := 0; j < i+1; j++ {
d := tc.toAdd[j]
node, err := n.LookupNode(ctx, d, nil)
if err != nil {
t.Fatalf("expected no error, got %v", err)
}
de := append(make([]byte, 32-len(d)), d...)
if !bytes.Equal(node.entry, de) {
t.Fatalf("expected value %x, got %x", d, node.entry)
}
if !node.IsValueType() {
t.Fatalf("expected value type, got %v", strconv.FormatInt(int64(node.nodeType), 2))
}
}
}
})
}
}
func TestRemove(t *testing.T) {
for _, tc := range []struct {
name string
toAdd []nodeEntry
toRemove [][]byte
}{
{
name: "simple",
toAdd: []nodeEntry{
{
path: []byte("/"),
metadata: map[string]string{
"index-document": "index.html",
},
},
{
path: []byte("index.html"),
},
{
path: []byte("img/1.png"),
},
{
path: []byte("img/2.png"),
},
{
path: []byte("robots.txt"),
},
},
toRemove: [][]byte{
[]byte("img/2.png"),
},
},
{
name: "nested-prefix-is-not-collapsed",
toAdd: []nodeEntry{
{
path: []byte("index.html"),
},
{
path: []byte("img/1.png"),
},
{
path: []byte("img/2/test1.png"),
},
{
path: []byte("img/2/test2.png"),
},
{
path: []byte("robots.txt"),
},
},
toRemove: [][]byte{
[]byte("img/2/test1.png"),
},
},
} {
ctx := context.Background()
t.Run(tc.name, func(t *testing.T) {
n := New()
for i := 0; i < len(tc.toAdd); i++ {
c := tc.toAdd[i].path
e := tc.toAdd[i].entry
if len(e) == 0 {
e = append(make([]byte, 32-len(c)), c...)
}
m := tc.toAdd[i].metadata
err := n.Add(ctx, c, e, m, nil)
if err != nil {
t.Fatalf("expected no error, got %v", err)
}
for j := 0; j < i; j++ {
d := tc.toAdd[j].path
m, err := n.Lookup(ctx, d, nil)
if err != nil {
t.Fatalf("expected no error, got %v", err)
}
de := append(make([]byte, 32-len(d)), d...)
if !bytes.Equal(m, de) {
t.Fatalf("expected value %x, got %x", d, m)
}
}
}
for i := 0; i < len(tc.toRemove); i++ {
c := tc.toRemove[i]
err := n.Remove(ctx, c, nil)
if err != nil {
t.Fatalf("expected no error, got %v", err)
}
_, err = n.Lookup(ctx, c, nil)
if !errors.Is(err, ErrNotFound) {
t.Fatalf("expected not found error, got %v", err)
}
}
})
}
}
func TestHasPrefix(t *testing.T) {
for _, tc := range []struct {
name string
toAdd [][]byte
testPrefix [][]byte
shouldExist []bool
}{
{
name: "simple",
toAdd: [][]byte{
[]byte("index.html"),
[]byte("img/1.png"),
[]byte("img/2.png"),
[]byte("robots.txt"),
},
testPrefix: [][]byte{
[]byte("img/"),
[]byte("images/"),
},
shouldExist: []bool{
true,
false,
},
},
{
name: "nested-single",
toAdd: [][]byte{
[]byte("some-path/file.ext"),
},
testPrefix: [][]byte{
[]byte("some-path/"),
[]byte("some-path/file"),
[]byte("some-other-path/"),
},
shouldExist: []bool{
true,
true,
false,
},
},
} {
ctx := context.Background()
t.Run(tc.name, func(t *testing.T) {
n := New()
for i := 0; i < len(tc.toAdd); i++ {
c := tc.toAdd[i]
e := append(make([]byte, 32-len(c)), c...)
err := n.Add(ctx, c, e, nil, nil)
if err != nil {
t.Fatalf("expected no error, got %v", err)
}
}
for i := 0; i < len(tc.testPrefix); i++ {
testPrefix := tc.testPrefix[i]
shouldExist := tc.shouldExist[i]
exists, err := n.HasPrefix(ctx, testPrefix, nil)
if err != nil {
t.Fatalf("expected no error, got %v", err)
}
if shouldExist != exists {
t.Errorf("expected prefix path %s to be %t, was %t", testPrefix, shouldExist, exists)
}
}
})
}
}
pkg/manifest/mantaray/persist.go 0 → 100644
View file @ 34cb57c9
// Copyright 2020 The Swarm Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package mantaray
import (
"context"
"errors"
"golang.org/x/sync/errgroup"
)
var (
// ErrNoSaver saver interface not given
ErrNoSaver = errors.New("Node is not persisted but no saver")
// ErrNoLoader saver interface not given
ErrNoLoader = errors.New("Node is reference but no loader")
)
// Loader defines a generic interface to retrieve nodes
// from a persistent storage
// for read only operations only
type Loader interface {
Load(ctx context.Context, reference []byte) (data []byte, err error)
}
// Saver defines a generic interface to persist nodes
// for write operations
type Saver interface {
Save(ctx context.Context, data []byte) (reference []byte, err error)
}
// LoadSaver is a composite interface of Loader and Saver
// it is meant to be implemented as thin wrappers around persistent storage like Swarm
type LoadSaver interface {
Loader
Saver
}
func (n *Node) load(ctx context.Context, l Loader) error {
if n == nil || n.ref == nil {
return nil
}
if l == nil {
return ErrNoLoader
}
b, err := l.Load(ctx, n.ref)
if err != nil {
return err
}
return n.UnmarshalBinary(b)
}
// Save persists a trie recursively traversing the nodes
func (n *Node) Save(ctx context.Context, s Saver) error {
if s == nil {
return ErrNoSaver
}
return n.save(ctx, s)
}
func (n *Node) save(ctx context.Context, s Saver) error {
if n != nil && n.ref != nil {
return nil
}
select {
case <-ctx.Done():
return ctx.Err()
default:
}
eg, ectx := errgroup.WithContext(ctx)
for _, f := range n.forks {
f := f
eg.Go(func() error {
return f.Node.save(ectx, s)
})
}
if err := eg.Wait(); err != nil {
return err
}
bytes, err := n.MarshalBinary()
if err != nil {
return err
}
n.ref, err = s.Save(ctx, bytes)
if err != nil {
return err
}
n.forks = nil
return nil
}
pkg/manifest/mantaray/persist_test.go 0 → 100644
View file @ 34cb57c9
// Copyright 2020 The Swarm Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package mantaray_test
import (
"bytes"
"context"
"crypto/sha256"
"sync"
"testing"
"github.com/ethersphere/bee/pkg/manifest/mantaray"
)
func TestPersistIdempotence(t *testing.T) {
n := mantaray.New()
paths := [][]byte{
[]byte("aa"),
[]byte("b"),
[]byte("aaaaaa"),
[]byte("aaaaab"),
[]byte("abbbb"),
[]byte("abbba"),
[]byte("bbbbba"),
[]byte("bbbaaa"),
[]byte("bbbaab"),
}
ctx := context.Background()
var ls mantaray.LoadSaver = newMockLoadSaver()
for i := 0; i < len(paths); i++ {
c := paths[i]
err := n.Save(ctx, ls)
if err != nil {
t.Fatalf("expected no error, got %v", err)
}
var v [32]byte
copy(v[:], c)
err = n.Add(ctx, c, v[:], nil, ls)
if err != nil {
t.Fatalf("expected no error, got %v", err)
}
}
err := n.Save(ctx, ls)
if err != nil {
t.Fatalf("expected no error, got %v", err)
}
for i := 0; i < len(paths); i++ {
c := paths[i]
m, err := n.Lookup(ctx, c, ls)
if err != nil {
t.Fatalf("expected no error, got %v", err)
}
var v [32]byte
copy(v[:], c)
if !bytes.Equal(m, v[:]) {
t.Fatalf("expected value %x, got %x", v[:], m)
}
}
}
type addr [32]byte
type mockLoadSaver struct {
mtx sync.Mutex
store map[addr][]byte
}
func newMockLoadSaver() *mockLoadSaver {
return &mockLoadSaver{
store: make(map[addr][]byte),
}
}
func (m *mockLoadSaver) Save(_ context.Context, b []byte) ([]byte, error) {
var a addr
hasher := sha256.New()
_, err := hasher.Write(b)
if err != nil {
return nil, err
}
copy(a[:], hasher.Sum(nil))
m.mtx.Lock()
defer m.mtx.Unlock()
m.store[a] = b
return a[:], nil
}
func (m *mockLoadSaver) Load(_ context.Context, ab []byte) ([]byte, error) {
var a addr
copy(a[:], ab)
m.mtx.Lock()
defer m.mtx.Unlock()
b, ok := m.store[a]
if !ok {
return nil, mantaray.ErrNotFound
}
return b, nil
}
pkg/manifest/mantaray/stringer.go 0 → 100644
View file @ 34cb57c9
// Copyright 2020 The Swarm Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package mantaray
import (
"bytes"
"fmt"
"io"
"strconv"
)
//nolint,errcheck
func (n *Node) String() string {
buf := bytes.NewBuffer(nil)
io.WriteString(buf, tableCharsMap["bottom-left"])
io.WriteString(buf, tableCharsMap["bottom"])
io.WriteString(buf, tableCharsMap["top-right"])
io.WriteString(buf, "\n")
nodeStringWithPrefix(n, " ", buf)
return buf.String()
}
//nolint,errcheck
func nodeStringWithPrefix(n *Node, prefix string, writer io.Writer) {
io.WriteString(writer, prefix)
io.WriteString(writer, tableCharsMap["left-mid"])
io.WriteString(writer, fmt.Sprintf("r: '%x'\n", n.ref))
io.WriteString(writer, prefix)
io.WriteString(writer, tableCharsMap["left-mid"])
io.WriteString(writer, fmt.Sprintf("t: '%s'", strconv.FormatInt(int64(n.nodeType), 2)))
io.WriteString(writer, " [")
if n.IsValueType() {
io.WriteString(writer, " Value")
}
if n.IsEdgeType() {
io.WriteString(writer, " Edge")
}
if n.IsWithPathSeparatorType() {
io.WriteString(writer, " PathSeparator")
}
io.WriteString(writer, " ]")
io.WriteString(writer, "\n")
io.WriteString(writer, prefix)
if len(n.forks) > 0 || len(n.metadata) > 0 {
io.WriteString(writer, tableCharsMap["left-mid"])
} else {
io.WriteString(writer, tableCharsMap["bottom-left"])
}
io.WriteString(writer, fmt.Sprintf("e: '%s'\n", string(n.entry)))
if len(n.metadata) > 0 {
io.WriteString(writer, prefix)
if len(n.forks) > 0 {
io.WriteString(writer, tableCharsMap["left-mid"])
} else {
io.WriteString(writer, tableCharsMap["bottom-left"])
}
io.WriteString(writer, fmt.Sprintf("m: '%s'\n", n.metadata))
}
counter := 0
for k, f := range n.forks {
isLast := counter != len(n.forks)-1
io.WriteString(writer, prefix)
if isLast {
io.WriteString(writer, tableCharsMap["left-mid"])
} else {
io.WriteString(writer, tableCharsMap["bottom-left"])
}
io.WriteString(writer, tableCharsMap["mid"])
io.WriteString(writer, fmt.Sprintf("[%s]", string(k)))
io.WriteString(writer, tableCharsMap["mid"])
io.WriteString(writer, tableCharsMap["top-mid"])
io.WriteString(writer, tableCharsMap["mid"])
io.WriteString(writer, fmt.Sprintf("`%s`\n", string(f.prefix)))
newPrefix := prefix
if isLast {
newPrefix += tableCharsMap["middle"]
} else {
newPrefix += " "
}
newPrefix += " "
nodeStringWithPrefix(f.Node, newPrefix, writer)
counter++
}
}
var tableCharsMap = map[string]string{
"top": "─",
"top-mid": "┬",
"top-left": "┌",
"top-right": "┐",
"bottom": "─",
"bottom-mid": "┴",
"bottom-left": "└",
"bottom-right": "┘",
"left": "│",
"left-mid": "├",
"mid": "─",
"mid-mid": "┼",
"right": "│",
"right-mid": "┤",
"middle": "│",
}
pkg/manifest/mantaray/walker.go 0 → 100644
View file @ 34cb57c9
// Copyright 2020 The Swarm Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package mantaray
import "context"
// WalkNodeFunc is the type of the function called for each node visited
// by WalkNode.
type WalkNodeFunc func(path []byte, node *Node, err error) error
func walkNodeFnCopyBytes(ctx context.Context, path []byte, node *Node, err error, walkFn WalkNodeFunc) error {
return walkFn(append(path[:0:0], path...), node, nil)
}
// walkNode recursively descends path, calling walkFn.
func walkNode(ctx context.Context, path []byte, l Loader, n *Node, walkFn WalkNodeFunc) error {
if n.forks == nil {
if err := n.load(ctx, l); err != nil {
return err
}
}
err := walkNodeFnCopyBytes(ctx, path, n, nil, walkFn)
if err != nil {
return err
}
for _, v := range n.forks {
nextPath := append(path[:0:0], path...)
nextPath = append(nextPath, v.prefix...)
err := walkNode(ctx, nextPath, l, v.Node, walkFn)
if err != nil {
return err
}
}
return nil
}
// WalkNode walks the node tree structure rooted at root, calling walkFn for
// each node in the tree, including root. All errors that arise visiting nodes
// are filtered by walkFn.
func (n *Node) WalkNode(ctx context.Context, root []byte, l Loader, walkFn WalkNodeFunc) error {
node, err := n.LookupNode(ctx, root, l)
if err != nil {
err = walkFn(root, nil, err)
} else {
err = walkNode(ctx, root, l, node, walkFn)
}
return err
}
// WalkFunc is the type of the function called for each file or directory
// visited by Walk.
type WalkFunc func(path []byte, isDir bool, err error) error
func walkFnCopyBytes(path []byte, isDir bool, err error, walkFn WalkFunc) error {
return walkFn(append(path[:0:0], path...), isDir, nil)
}
// walk recursively descends path, calling walkFn.
func walk(ctx context.Context, path, prefix []byte, l Loader, n *Node, walkFn WalkFunc) error {
if n.forks == nil {
if err := n.load(ctx, l); err != nil {
return err
}
}
nextPath := append(path[:0:0], path...)
for i := 0; i < len(prefix); i++ {
if prefix[i] == PathSeparator {
// path ends with separator
err := walkFnCopyBytes(nextPath, true, nil, walkFn)
if err != nil {
return err
}
}
nextPath = append(nextPath, prefix[i])
}
if n.IsValueType() {
if nextPath[len(nextPath)-1] == PathSeparator {
// path ends with separator; already reported
} else {
err := walkFnCopyBytes(nextPath, false, nil, walkFn)
if err != nil {
return err
}
}
}
if n.IsEdgeType() {
for _, v := range n.forks {
err := walk(ctx, nextPath, v.prefix, l, v.Node, walkFn)
if err != nil {
return err
}
}
}
return nil
}
// Walk walks the node tree structure rooted at root, calling walkFn for
// each file or directory in the tree, including root. All errors that arise
// visiting files and directories are filtered by walkFn.
func (n *Node) Walk(ctx context.Context, root []byte, l Loader, walkFn WalkFunc) error {
node, err := n.LookupNode(ctx, root, l)
if err != nil {
return walkFn(root, false, err)
}
return walk(ctx, root, []byte{}, l, node, walkFn)
}
pkg/manifest/mantaray/walker_test.go 0 → 100644
View file @ 34cb57c9
// Copyright 2020 The Swarm Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package mantaray
import (
"bytes"
"context"
"fmt"
"testing"
)
func TestWalkNode(t *testing.T) {
for _, tc := range []struct {
name string
toAdd [][]byte
expected [][]byte
}{
{
name: "simple",
toAdd: [][]byte{
[]byte("index.html"),
[]byte("img/1.png"),
[]byte("img/2.png"),
[]byte("robots.txt"),
},
expected: [][]byte{
[]byte(""),
[]byte("i"),
[]byte("index.html"),
[]byte("img/"),
[]byte("img/1.png"),
[]byte("img/2.png"),
[]byte("robots.txt"),
},
},
} {
ctx := context.Background()
t.Run(tc.name, func(t *testing.T) {
n := New()
for i := 0; i < len(tc.toAdd); i++ {
c := tc.toAdd[i]
e := append(make([]byte, 32-len(c)), c...)
err := n.Add(ctx, c, e, nil, nil)
if err != nil {
t.Fatalf("expected no error, got %v", err)
}
}
walkedCount := 0
walker := func(path []byte, node *Node, err error) error {
walkedCount++
pathFound := false
for i := 0; i < len(tc.expected); i++ {
c := tc.expected[i]
if bytes.Equal(path, c) {
pathFound = true
break
}
}
if !pathFound {
return fmt.Errorf("walkFn returned unknown path: %s", path)
}
return nil
}
// Expect no errors.
err := n.WalkNode(ctx, []byte{}, nil, walker)
if err != nil {
t.Fatalf("no error expected, found: %s", err)
}
if len(tc.expected) != walkedCount {
t.Errorf("expected %d nodes, got %d", len(tc.expected), walkedCount)
}
})
}
}
func TestWalk(t *testing.T) {
for _, tc := range []struct {
name string
toAdd [][]byte
expected [][]byte
}{
{
name: "simple",
toAdd: [][]byte{
[]byte("index.html"),
[]byte("img/test/"),
[]byte("img/test/oho.png"),
[]byte("img/test/old/test.png"),
[]byte("robots.txt"),
},
expected: [][]byte{
[]byte("index.html"),
[]byte("img"),
[]byte("img/test"),
[]byte("img/test/oho.png"),
[]byte("img/test/old"),
[]byte("img/test/old/test.png"),
[]byte("robots.txt"),
},
},
} {
ctx := context.Background()
t.Run(tc.name, func(t *testing.T) {
n := New()
for i := 0; i < len(tc.toAdd); i++ {
c := tc.toAdd[i]
e := append(make([]byte, 32-len(c)), c...)
err := n.Add(ctx, c, e, nil, nil)
if err != nil {
t.Fatalf("expected no error, got %v", err)
}
}
walkedCount := 0
walker := func(path []byte, isDir bool, err error) error {
walkedCount++
pathFound := false
for i := 0; i < len(tc.expected); i++ {
c := tc.expected[i]
if bytes.Equal(path, c) {
pathFound = true
break
}
}
if !pathFound {
return fmt.Errorf("walkFn returned unknown path: %s", path)
}
return nil
}
// Expect no errors.
err := n.Walk(ctx, []byte{}, nil, walker)
if err != nil {
t.Fatalf("no error expected, found: %s", err)
}
if len(tc.expected) != walkedCount {
t.Errorf("expected %d nodes, got %d", len(tc.expected), walkedCount)
}
})
}
}
pkg/manifest/simple.go
View file @ 34cb57c9
......@@ -10,8 +10,8 @@ import (
"fmt"
"github.com/ethersphere/bee/pkg/file"
"github.com/ethersphere/bee/pkg/manifest/simple"
"github.com/ethersphere/bee/pkg/swarm"
"github.com/ethersphere/manifest/simple"
)
const (
......
pkg/manifest/simple/entry.go 0 → 100644
View file @ 34cb57c9
// Copyright 2020 The Swarm Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package simple
// Entry is a representation of a single manifest entry.
type Entry interface {
// Reference returns the address of the file in the entry.
Reference() string
// Metadata returns the metadata for this entry.
Metadata() map[string]string
}
// entry is a JSON representation of a single manifest entry.
type entry struct {
Ref string `json:"reference"`
Meta map[string]string `json:"metadata,omitempty"`
}
// newEntry creates a new Entry struct and returns it.
func newEntry(reference string, metadata map[string]string) *entry {
return &entry{
Ref: reference,
Meta: metadata,
}
}
func (me *entry) Reference() string {
return me.Ref
}
func (me *entry) Metadata() map[string]string {
return me.Meta
}
pkg/manifest/simple/manifest.go 0 → 100644
View file @ 34cb57c9
// Copyright 2020 The Swarm Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package simple
import (
"encoding"
"encoding/json"
"errors"
"fmt"
"strings"
"sync"
)
// Error used when lookup path does not match
var (
ErrNotFound = errors.New("not found")
ErrEmptyPath = errors.New("empty path")
)
// Manifest is a representation of a manifest.
type Manifest interface {
// Add adds a manifest entry to the specified path.
Add(string, string, map[string]string) error
// Remove removes a manifest entry on the specified path.
Remove(string) error
// Lookup returns a manifest node entry if one is found in the specified path.
Lookup(string) (Entry, error)
// HasPrefix tests whether the specified prefix path exists.
HasPrefix(string) bool
// Length returns an implementation-specific count of elements in the manifest.
// For Manifest, this means the number of all the existing entries.
Length() int
// WalkEntry walks all entries, calling walkFn for each entry in the map.
// All errors that arise visiting entires are filtered by walkFn.
WalkEntry(string, WalkEntryFunc) error
encoding.BinaryMarshaler
encoding.BinaryUnmarshaler
}
// manifest is a JSON representation of a manifest.
// It stores manifest entries in a map based on string keys.
type manifest struct {
Entries map[string]*entry `json:"entries,omitempty"`
mu sync.RWMutex // mutex for accessing the entries map
}
// NewManifest creates a new Manifest struct and returns a pointer to it.
func NewManifest() Manifest {
return &manifest{
Entries: make(map[string]*entry),
}
}
func notFound(path string) error {
return fmt.Errorf("entry on '%s': %w", path, ErrNotFound)
}
func (m *manifest) Add(path, entry string, metadata map[string]string) error {
if path == "" {
return ErrEmptyPath
}
m.mu.Lock()
defer m.mu.Unlock()
m.Entries[path] = newEntry(entry, metadata)
return nil
}
func (m *manifest) Remove(path string) error {
if path == "" {
return ErrEmptyPath
}
m.mu.Lock()
defer m.mu.Unlock()
delete(m.Entries, path)
return nil
}
func (m *manifest) Lookup(path string) (Entry, error) {
m.mu.RLock()
defer m.mu.RUnlock()
entry, ok := m.Entries[path]
if !ok {
return nil, notFound(path)
}
// return a copy to prevent external modification
return newEntry(entry.Reference(), entry.Metadata()), nil
}
func (m *manifest) HasPrefix(path string) bool {
m.mu.RLock()
defer m.mu.RUnlock()
for k := range m.Entries {
if strings.HasPrefix(k, path) {
return true
}
}
return false
}
func (m *manifest) Length() int {
m.mu.RLock()
defer m.mu.RUnlock()
return len(m.Entries)
}
// MarshalBinary implements encoding.BinaryMarshaler.
func (m *manifest) MarshalBinary() ([]byte, error) {
m.mu.RLock()
defer m.mu.RUnlock()
return json.Marshal(m)
}
// UnmarshalBinary implements encoding.BinaryUnmarshaler.
func (m *manifest) UnmarshalBinary(b []byte) error {
m.mu.Lock()
defer m.mu.Unlock()
return json.Unmarshal(b, m)
}
pkg/manifest/simple/manifest_test.go 0 → 100644
View file @ 34cb57c9
// Copyright 2020 The Swarm Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package simple_test
import (
"crypto/rand"
"encoding/hex"
"errors"
"reflect"
"testing"
"github.com/ethersphere/bee/pkg/manifest/simple"
)
// randomAddress generates a random address.
func randomAddress() string {
b := make([]byte, 32)
_, err := rand.Read(b)
if err != nil {
panic(err)
}
return hex.EncodeToString(b)
}
func TestNilPath(t *testing.T) {
m := simple.NewManifest()
n, err := m.Lookup("")
if err == nil {
t.Fatalf("expected error, got reference %s", n.Reference())
}
}
// struct for manifest entries for test cases
type e struct {
path string
reference string
metadata map[string]string
}
var testCases = []struct {
name string
entries []e // entries to add to manifest
}{
{
name: "empty-manifest",
entries: nil,
},
{
name: "one-entry",
entries: []e{
{
path: "entry-1",
reference: randomAddress(),
},
},
},
{
name: "two-entries",
entries: []e{
{
path: "entry-1.txt",
reference: randomAddress(),
},
{
path: "entry-2.png",
reference: randomAddress(),
},
},
},
{
name: "nested-entries",
entries: []e{
{
path: "text/robots.txt",
reference: randomAddress(),
},
{
path: "img/1.png",
reference: randomAddress(),
},
{
path: "img/2.jpg",
reference: randomAddress(),
},
{
path: "readme.md",
reference: randomAddress(),
},
{
path: "/",
metadata: map[string]string{
"index-document": "readme.md",
"error-document": "404.html",
},
},
},
},
}
func TestEntries(t *testing.T) {
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
m := simple.NewManifest()
checkLength(t, m, 0)
// add entries
for i, e := range tc.entries {
err := m.Add(e.path, e.reference, e.metadata)
if err != nil {
t.Fatal(err)
}
checkLength(t, m, i+1)
checkEntry(t, m, e.reference, e.path)
}
manifestLen := m.Length()
if len(tc.entries) != manifestLen {
t.Fatalf("expected %d entries, found %d", len(tc.entries), manifestLen)
}
if manifestLen == 0 {
// special case for empty manifest
return
}
// replace entry
lastEntry := tc.entries[len(tc.entries)-1]
newReference := randomAddress()
err := m.Add(lastEntry.path, newReference, map[string]string{})
if err != nil {
t.Fatal(err)
}
checkLength(t, m, manifestLen) // length should not have changed
checkEntry(t, m, newReference, lastEntry.path)
// remove entries
err = m.Remove("invalid/path.ext") // try removing inexistent entry
if err != nil {
t.Fatal(err)
}
checkLength(t, m, manifestLen) // length should not have changed
for i, e := range tc.entries {
err = m.Remove(e.path)
if err != nil {
t.Fatal(err)
}
entry, err := m.Lookup(e.path)
if entry != nil || !errors.Is(err, simple.ErrNotFound) {
t.Fatalf("expected path %v not to be present in the manifest, but it was found", e.path)
}
checkLength(t, m, manifestLen-i-1)
}
})
}
}
// checkLength verifies that the given manifest length and integer match.
func checkLength(t *testing.T, m simple.Manifest, length int) {
t.Helper()
if m.Length() != length {
t.Fatalf("expected length to be %d, but is %d instead", length, m.Length())
}
}
// checkEntry verifies that an entry is equal to the one retrieved from the given manifest and path.
func checkEntry(t *testing.T, m simple.Manifest, reference, path string) {
t.Helper()
n, err := m.Lookup(path)
if err != nil {
t.Fatal(err)
}
if n.Reference() != reference {
t.Fatalf("expected reference %s, got: %s", reference, n.Reference())
}
}
// TestMarshal verifies that created manifests are successfully marshalled and unmarshalled.
// This function wil add all test case entries to a manifest and marshal it.
// After, it will unmarshal the result, and verify that it is equal to the original manifest.
func TestMarshal(t *testing.T) {
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
m := simple.NewManifest()
for _, e := range tc.entries {
err := m.Add(e.path, e.reference, e.metadata)
if err != nil {
t.Fatal(err)
}
}
b, err := m.MarshalBinary()
if err != nil {
t.Fatal(err)
}
um := simple.NewManifest()
if err := um.UnmarshalBinary(b); err != nil {
t.Fatal(err)
}
if !reflect.DeepEqual(m, um) {
t.Fatalf("marshalled and unmarshalled manifests are not equal: %v, %v", m, um)
}
})
}
}
func TestHasPrefix(t *testing.T) {
for _, tc := range []struct {
name string
toAdd []string
testPrefix []string
shouldExist []bool
}{
{
name: "simple",
toAdd: []string{
"index.html",
"img/1.png",
"img/2.png",
"robots.txt",
},
testPrefix: []string{
"img/",
"images/",
},
shouldExist: []bool{
true,
false,
},
},
{
name: "nested-single",
toAdd: []string{
"some-path/file.ext",
},
testPrefix: []string{
"some-path/",
"some-path/file",
"some-other-path/",
},
shouldExist: []bool{
true,
true,
false,
},
},
} {
t.Run(tc.name, func(t *testing.T) {
m := simple.NewManifest()
for _, e := range tc.toAdd {
err := m.Add(e, "", nil)
if err != nil {
t.Fatalf("expected no error, got %v", err)
}
}
for i := 0; i < len(tc.testPrefix); i++ {
testPrefix := tc.testPrefix[i]
shouldExist := tc.shouldExist[i]
exists := m.HasPrefix(testPrefix)
if shouldExist != exists {
t.Errorf("expected prefix path %s to be %t, was %t", testPrefix, shouldExist, exists)
}
}
})
}
}
pkg/manifest/simple/walker.go 0 → 100644
View file @ 34cb57c9
// Copyright 2020 The Swarm Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package simple
// WalkEntryFunc is the type of the function called for each entry visited
// by WalkEntry.
type WalkEntryFunc func(path string, entry Entry, err error) error
func (m *manifest) WalkEntry(root string, walkFn WalkEntryFunc) (err error) {
m.mu.Lock()
defer m.mu.Unlock()
for k, v := range m.Entries {
entry := newEntry(v.Ref, v.Meta)
err = walkFn(k, entry, nil)
if err != nil {
return err
}
}
return nil
}
pkg/manifest/simple/walker_test.go 0 → 100644
View file @ 34cb57c9
// Copyright 2020 The Swarm Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package simple_test
import (
"fmt"
"testing"
"github.com/ethersphere/bee/pkg/manifest/simple"
)
func TestWalkEntry(t *testing.T) {
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
m := simple.NewManifest()
// add entries
for _, e := range tc.entries {
err := m.Add(e.path, e.reference, e.metadata)
if err != nil {
t.Fatal(err)
}
}
manifestLen := m.Length()
if len(tc.entries) != manifestLen {
t.Fatalf("expected %d entries, found %d", len(tc.entries), manifestLen)
}
walkedCount := 0
walker := func(path string, entry simple.Entry, err error) error {
walkedCount++
pathFound := false
for i := 0; i < len(tc.entries); i++ {
p := tc.entries[i].path
if path == p {
pathFound = true
break
}
}
if !pathFound {
return fmt.Errorf("walkFn returned unknown path: %s", path)
}
return nil
}
// Expect no errors.
err := m.WalkEntry("", walker)
if err != nil {
t.Fatalf("no error expected, found: %s", err)
}
if len(tc.entries) != walkedCount {
t.Errorf("expected %d nodes, got %d", len(tc.entries), walkedCount)
}
})
}
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment