Skip to content

M
mybee
Commit d5f15317 authored by aloknerurkar's avatar aloknerurkar Committed by GitHub
Browse files
Options

fix(mantaray): mantaray lookup issue (#2380)

parent 2cb6e13f
Hide whitespace changes
Inline Side-by-side
Showing with 234 additions and 84 deletions
pkg/manifest/mantaray/export_test.go 0 → 100644
View file @ d5f15317
// Copyright 2020 The Swarm Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package mantaray
func (n *Node) NodeType() uint8 {
return n.nodeType
}
pkg/manifest/mantaray/marshal.go
View file @ d5f15317
......@@ -57,11 +57,13 @@ const (
var (
version01HashBytes []byte
version02HashBytes []byte
zero32 []byte
)
func init() {
initVersion(version01HashString, &version01HashBytes)
initVersion(version02HashString, &version02HashBytes)
zero32 = make([]byte, 32)
}
func initVersion(hash string, bytes *[]byte) {
......@@ -274,8 +276,8 @@ func (n *Node) UnmarshalBinary(data []byte) error {
// the root nodeType information is lost on Unmarshal. This causes issues when we want to
// perform a path 'Walk' on the root. If there is more than 1 fork, the root node type
// is an edge, so we will deduce this information from index byte array
if !bytes.Equal(data[offset:offset+32], make([]byte, 32)) {
n.nodeType = nodeTypeEdge
if !bytes.Equal(data[offset:offset+32], zero32) && !n.IsEdgeType() {
n.makeEdge()
}
n.forks = make(map[byte]*fork)
bb := &bitsForBytes{}
......
pkg/manifest/mantaray/marshal_test.go
View file @ d5f15317
......@@ -20,27 +20,33 @@ const testMarshalOutput01 = "52fdfc072182654f163f5f0f9a621d729566c74d10037c4d7bb
const testMarshalOutput02 = "52fdfc072182654f163f5f0f9a621d729566c74d10037c4d7bbb0407d1e2c64905954fb18659339d0b25e0fb9723d3cd5d528fb3c8d495fd157bd7b7a210496952fdfc072182654f163f5f0f9a621d729566c74d10037c4d7bbb0407d1e2c64952fdfc072102654f163f5f0fa0621d729566c74d10037c4d7bbb0407d1e2c64940fcd3072182654f163f5f0f9a621d729566c74d10037c4d7bbb0407d1e2c64952fdfc072182654f163f5f0f9a621d729566c74d10037c4d7bbb0407d1e2c64952e3872548ec012a6e123b60f9177017fb12e57732621d2c1ada267adbe8cc4350f89d6640e3044f163f5f0f9a621d729566c74d10037c4d7bbb0407d1e2c64952fdfc072182654f163f5f0f9a621d729566c74d10037c4d7bbb0407d1e2c64850ff9f642182654f163f5f0f9a621d729566c74d10037c4d7bbb0407d1e2c64952fdfc072182654f163f5f0f9a621d729566c74d10037c4d7bbb0407d1e2c64b50fc98072182654f163f5f0f9a621d729566c74d10037c4d7bbb0407d1e2c64952fdfc072182654f163f5f0f9a621d729566c74d10037c4d7bbb0407d1e2c64a50ff99622182654f163f5f0f9a621d729566c74d10037c4d7bbb0407d1e2c64952fdfc072182654f163f5f0f9a621d729566c74d10037c4d7bbb0407d1e2c64d"
var testEntries = []nodeEntry{
var testEntries = []NodeEntry{
{
path: []byte("/"),
metadata: map[string]string{
Path: []byte("/"),
Metadata: map[string]string{
"index-document": "aaaaa",
},
},
{
path: []byte("aaaaa"),
Path: []byte("aaaaa"),
},
{
path: []byte("cc"),
Path: []byte("cc"),
},
{
path: []byte("d"),
Path: []byte("d"),
},
{
path: []byte("ee"),
Path: []byte("ee"),
},
}
type NodeEntry struct {
Path []byte
Entry []byte
Metadata map[string]string
}
func init() {
obfuscationKeyFn = mrand.Read
}
......@@ -98,7 +104,7 @@ func TestUnmarshal01(t *testing.T) {
t.Fatalf("expected %d forks, got %d", len(testEntries), len(n.forks))
}
for _, entry := range testEntries {
prefix := entry.path
prefix := entry.Path
f := n.forks[prefix[0]]
if f == nil {
t.Fatalf("expected to have fork on byte %x", prefix[:1])
......@@ -130,7 +136,7 @@ func TestUnmarshal02(t *testing.T) {
t.Fatalf("expected %d forks, got %d", len(testEntries), len(n.forks))
}
for _, entry := range testEntries {
prefix := entry.path
prefix := entry.Path
f := n.forks[prefix[0]]
if f == nil {
t.Fatalf("expected to have fork on byte %x", prefix[:1])
......@@ -138,9 +144,9 @@ func TestUnmarshal02(t *testing.T) {
if !bytes.Equal(f.prefix, prefix) {
t.Fatalf("expected prefix for byte %x to match %s, got %s", prefix[:1], prefix, f.prefix)
}
if len(entry.metadata) > 0 {
if !reflect.DeepEqual(entry.metadata, f.metadata) {
t.Fatalf("expected metadata for byte %x to match %s, got %s", prefix[:1], entry.metadata, f.metadata)
if len(entry.Metadata) > 0 {
if !reflect.DeepEqual(entry.Metadata, f.metadata) {
t.Fatalf("expected metadata for byte %x to match %s, got %s", prefix[:1], entry.Metadata, f.metadata)
}
}
}
......@@ -159,12 +165,12 @@ func TestMarshal(t *testing.T) {
return b
}
for i := 0; i < len(testEntries); i++ {
c := testEntries[i].path
e := testEntries[i].entry
c := testEntries[i].Path
e := testEntries[i].Entry
if len(e) == 0 {
e = append(make([]byte, 32-len(c)), c...)
}
m := testEntries[i].metadata
m := testEntries[i].Metadata
err := n.Add(ctx, c, e, m, nil)
if err != nil {
t.Fatalf("expected no error, got %v", err)
......
pkg/manifest/mantaray/node.go
View file @ d5f15317
......@@ -181,6 +181,9 @@ func (n *Node) Lookup(ctx context.Context, path []byte, l Loader) ([]byte, error
if err != nil {
return nil, err
}
if !node.IsValueType() && len(path) > 0 {
return nil, notFound(path)
}
return node.entry, nil
}
......@@ -205,6 +208,7 @@ func (n *Node) Add(ctx context.Context, path, entry []byte, metadata map[string]
if len(path) == 0 {
n.entry = entry
n.makeValue()
if len(metadata) > 0 {
n.metadata = metadata
n.makeWithMetadata()
......
pkg/manifest/mantaray/node_test.go
View file @ d5f15317
......@@ -2,7 +2,7 @@
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package mantaray
package mantaray_test
import (
"bytes"
......@@ -10,17 +10,13 @@ import (
"errors"
"strconv"
"testing"
)
type nodeEntry struct {
path []byte
entry []byte
metadata map[string]string
}
"github.com/ethersphere/bee/pkg/manifest/mantaray"
)
func TestNilPath(t *testing.T) {
ctx := context.Background()
n := New()
n := mantaray.New()
_, err := n.Lookup(ctx, nil, nil)
if err != nil {
t.Fatalf("expected no error, got %v", err)
......@@ -29,7 +25,7 @@ func TestNilPath(t *testing.T) {
func TestAddAndLookup(t *testing.T) {
ctx := context.Background()
n := New()
n := mantaray.New()
testCases := [][]byte{
[]byte("aaaaaa"),
[]byte("aaaaab"),
......@@ -91,6 +87,14 @@ func TestAddAndLookupNode(t *testing.T) {
[]byte("robots.txt"),
},
},
{
// mantaray.nodePrefixMaxSize number of '.'
name: "nested-value-node-is-recognized",
toAdd: [][]byte{
[]byte("..............................@"),
[]byte(".............................."),
},
},
{
name: "nested-prefix-is-not-collapsed",
toAdd: [][]byte{
......@@ -127,7 +131,7 @@ func TestAddAndLookupNode(t *testing.T) {
} {
ctx := context.Background()
t.Run(tc.name, func(t *testing.T) {
n := New()
n := mantaray.New()
for i := 0; i < len(tc.toAdd); i++ {
c := tc.toAdd[i]
......@@ -142,45 +146,79 @@ func TestAddAndLookupNode(t *testing.T) {
if err != nil {
t.Fatalf("expected no error, got %v", err)
}
de := append(make([]byte, 32-len(d)), d...)
if !bytes.Equal(node.entry, de) {
t.Fatalf("expected value %x, got %x", d, node.entry)
}
if !node.IsValueType() {
t.Fatalf("expected value type, got %v", strconv.FormatInt(int64(node.nodeType), 2))
t.Fatalf("expected value type, got %v", strconv.FormatInt(int64(node.NodeType()), 2))
}
de := append(make([]byte, 32-len(d)), d...)
if !bytes.Equal(node.Entry(), de) {
t.Fatalf("expected value %x, got %x", d, node.Entry())
}
}
}
})
t.Run(tc.name+"/with load save", func(t *testing.T) {
n := mantaray.New()
for i := 0; i < len(tc.toAdd); i++ {
c := tc.toAdd[i]
e := append(make([]byte, 32-len(c)), c...)
err := n.Add(ctx, c, e, nil, nil)
if err != nil {
t.Fatalf("expected no error, got %v", err)
}
}
ls := newMockLoadSaver()
err := n.Save(ctx, ls)
if err != nil {
t.Fatal(err)
}
n2 := mantaray.NewNodeRef(n.Reference())
for j := 0; j < len(tc.toAdd); j++ {
d := tc.toAdd[j]
node, err := n2.LookupNode(ctx, d, ls)
if err != nil {
t.Fatalf("expected no error, got %v", err)
}
if !node.IsValueType() {
t.Fatalf("expected value type, got %v", strconv.FormatInt(int64(node.NodeType()), 2))
}
de := append(make([]byte, 32-len(d)), d...)
if !bytes.Equal(node.Entry(), de) {
t.Fatalf("expected value %x, got %x", d, node.Entry())
}
}
})
}
}
func TestRemove(t *testing.T) {
for _, tc := range []struct {
name string
toAdd []nodeEntry
toAdd []mantaray.NodeEntry
toRemove [][]byte
}{
{
name: "simple",
toAdd: []nodeEntry{
toAdd: []mantaray.NodeEntry{
{
path: []byte("/"),
metadata: map[string]string{
Path: []byte("/"),
Metadata: map[string]string{
"index-document": "index.html",
},
},
{
path: []byte("index.html"),
Path: []byte("index.html"),
},
{
path: []byte("img/1.png"),
Path: []byte("img/1.png"),
},
{
path: []byte("img/2.png"),
Path: []byte("img/2.png"),
},
{
path: []byte("robots.txt"),
Path: []byte("robots.txt"),
},
},
toRemove: [][]byte{
......@@ -189,21 +227,21 @@ func TestRemove(t *testing.T) {
},
{
name: "nested-prefix-is-not-collapsed",
toAdd: []nodeEntry{
toAdd: []mantaray.NodeEntry{
{
path: []byte("index.html"),
Path: []byte("index.html"),
},
{
path: []byte("img/1.png"),
Path: []byte("img/1.png"),
},
{
path: []byte("img/2/test1.png"),
Path: []byte("img/2/test1.png"),
},
{
path: []byte("img/2/test2.png"),
Path: []byte("img/2/test2.png"),
},
{
path: []byte("robots.txt"),
Path: []byte("robots.txt"),
},
},
toRemove: [][]byte{
......@@ -213,21 +251,21 @@ func TestRemove(t *testing.T) {
} {
ctx := context.Background()
t.Run(tc.name, func(t *testing.T) {
n := New()
n := mantaray.New()
for i := 0; i < len(tc.toAdd); i++ {
c := tc.toAdd[i].path
e := tc.toAdd[i].entry
c := tc.toAdd[i].Path
e := tc.toAdd[i].Entry
if len(e) == 0 {
e = append(make([]byte, 32-len(c)), c...)
}
m := tc.toAdd[i].metadata
m := tc.toAdd[i].Metadata
err := n.Add(ctx, c, e, m, nil)
if err != nil {
t.Fatalf("expected no error, got %v", err)
}
for j := 0; j < i; j++ {
d := tc.toAdd[j].path
d := tc.toAdd[j].Path
m, err := n.Lookup(ctx, d, nil)
if err != nil {
t.Fatalf("expected no error, got %v", err)
......@@ -246,7 +284,7 @@ func TestRemove(t *testing.T) {
t.Fatalf("expected no error, got %v", err)
}
_, err = n.Lookup(ctx, c, nil)
if !errors.Is(err, ErrNotFound) {
if !errors.Is(err, mantaray.ErrNotFound) {
t.Fatalf("expected not found error, got %v", err)
}
}
......@@ -298,7 +336,7 @@ func TestHasPrefix(t *testing.T) {
} {
ctx := context.Background()
t.Run(tc.name, func(t *testing.T) {
n := New()
n := mantaray.New()
for i := 0; i < len(tc.toAdd); i++ {
c := tc.toAdd[i]
......
pkg/manifest/mantaray/walker_test.go
View file @ d5f15317
......@@ -2,13 +2,15 @@
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package mantaray
package mantaray_test
import (
"bytes"
"context"
"fmt"
"testing"
"github.com/ethersphere/bee/pkg/manifest/mantaray"
)
func TestWalkNode(t *testing.T) {
......@@ -37,41 +39,83 @@ func TestWalkNode(t *testing.T) {
},
} {
ctx := context.Background()
t.Run(tc.name, func(t *testing.T) {
n := New()
for i := 0; i < len(tc.toAdd); i++ {
c := tc.toAdd[i]
createTree := func(t *testing.T, toAdd [][]byte) *mantaray.Node {
n := mantaray.New()
for i := 0; i < len(toAdd); i++ {
c := toAdd[i]
e := append(make([]byte, 32-len(c)), c...)
err := n.Add(ctx, c, e, nil, nil)
if err != nil {
t.Fatalf("expected no error, got %v", err)
}
}
return n
}
pathExists := func(found []byte, expected [][]byte) bool {
pathFound := false
for i := 0; i < len(tc.expected); i++ {
c := tc.expected[i]
if bytes.Equal(found, c) {
pathFound = true
break
}
}
return pathFound
}
t.Run(tc.name, func(t *testing.T) {
n := createTree(t, tc.toAdd)
walkedCount := 0
walker := func(path []byte, node *Node, err error) error {
walker := func(path []byte, node *mantaray.Node, err error) error {
walkedCount++
pathFound := false
for i := 0; i < len(tc.expected); i++ {
c := tc.expected[i]
if bytes.Equal(path, c) {
pathFound = true
break
}
if !pathExists(path, tc.expected) {
return fmt.Errorf("walkFn returned unknown path: %s", path)
}
return nil
}
// Expect no errors.
err := n.WalkNode(ctx, []byte{}, nil, walker)
if err != nil {
t.Fatalf("no error expected, found: %s", err)
}
if !pathFound {
if len(tc.expected) != walkedCount {
t.Errorf("expected %d nodes, got %d", len(tc.expected), walkedCount)
}
})
t.Run(tc.name+"/with load save", func(t *testing.T) {
n := createTree(t, tc.toAdd)
ls := newMockLoadSaver()
err := n.Save(ctx, ls)
if err != nil {
t.Fatal(err)
}
n2 := mantaray.NewNodeRef(n.Reference())
walkedCount := 0
walker := func(path []byte, node *mantaray.Node, err error) error {
walkedCount++
if !pathExists(path, tc.expected) {
return fmt.Errorf("walkFn returned unknown path: %s", path)
}
return nil
}
// Expect no errors.
err := n.WalkNode(ctx, []byte{}, nil, walker)
err = n2.WalkNode(ctx, []byte{}, ls, walker)
if err != nil {
t.Fatalf("no error expected, found: %s", err)
}
......@@ -79,7 +123,6 @@ func TestWalkNode(t *testing.T) {
if len(tc.expected) != walkedCount {
t.Errorf("expected %d nodes, got %d", len(tc.expected), walkedCount)
}
})
}
}
......@@ -97,6 +140,8 @@ func TestWalk(t *testing.T) {
[]byte("img/test/"),
[]byte("img/test/oho.png"),
[]byte("img/test/old/test.png"),
// file with same prefix but not a directory prefix
[]byte("img/test/old/test.png.backup"),
[]byte("robots.txt"),
},
expected: [][]byte{
......@@ -106,46 +151,93 @@ func TestWalk(t *testing.T) {
[]byte("img/test/oho.png"),
[]byte("img/test/old"),
[]byte("img/test/old/test.png"),
[]byte("img/test/old/test.png.backup"),
[]byte("robots.txt"),
},
},
} {
ctx := context.Background()
t.Run(tc.name, func(t *testing.T) {
n := New()
for i := 0; i < len(tc.toAdd); i++ {
c := tc.toAdd[i]
createTree := func(t *testing.T, toAdd [][]byte) *mantaray.Node {
n := mantaray.New()
for i := 0; i < len(toAdd); i++ {
c := toAdd[i]
e := append(make([]byte, 32-len(c)), c...)
err := n.Add(ctx, c, e, nil, nil)
if err != nil {
t.Fatalf("expected no error, got %v", err)
}
}
return n
}
pathExists := func(found []byte, expected [][]byte) bool {
pathFound := false
for i := 0; i < len(tc.expected); i++ {
c := tc.expected[i]
if bytes.Equal(found, c) {
pathFound = true
break
}
}
return pathFound
}
t.Run(tc.name, func(t *testing.T) {
n := createTree(t, tc.toAdd)
walkedCount := 0
walker := func(path []byte, isDir bool, err error) error {
walkedCount++
pathFound := false
for i := 0; i < len(tc.expected); i++ {
c := tc.expected[i]
if bytes.Equal(path, c) {
pathFound = true
break
}
if !pathExists(path, tc.expected) {
return fmt.Errorf("walkFn returned unknown path: %s", path)
}
if !pathFound {
return nil
}
// Expect no errors.
err := n.Walk(ctx, []byte{}, nil, walker)
if err != nil {
t.Fatalf("no error expected, found: %s", err)
}
if len(tc.expected) != walkedCount {
t.Errorf("expected %d nodes, got %d", len(tc.expected), walkedCount)
}
})
t.Run(tc.name+"/with load save", func(t *testing.T) {
n := createTree(t, tc.toAdd)
ls := newMockLoadSaver()
err := n.Save(ctx, ls)
if err != nil {
t.Fatal(err)
}
n2 := mantaray.NewNodeRef(n.Reference())
walkedCount := 0
walker := func(path []byte, isDir bool, err error) error {
walkedCount++
if !pathExists(path, tc.expected) {
return fmt.Errorf("walkFn returned unknown path: %s", path)
}
return nil
}
// Expect no errors.
err := n.Walk(ctx, []byte{}, nil, walker)
err = n2.Walk(ctx, []byte{}, ls, walker)
if err != nil {
t.Fatalf("no error expected, found: %s", err)
}
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment