fix(mantaray): mantaray lookup issue (#2380)

pkg/manifest/mantaray/export_test.go

+// Copyright 2020 The Swarm Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+package mantaray
+
+func (n *Node) NodeType() uint8 {
+	return n.nodeType
+}

pkg/manifest/mantaray/marshal.go

@@ -57,11 +57,13 @@ const (
 var (
 	version01HashBytes []byte
 	version02HashBytes []byte
+	zero32             []byte
 )
 
 func init() {
 	initVersion(version01HashString, &version01HashBytes)
 	initVersion(version02HashString, &version02HashBytes)
+	zero32 = make([]byte, 32)
 }
 
 func initVersion(hash string, bytes *[]byte) {
@@ -274,8 +276,8 @@ func (n *Node) UnmarshalBinary(data []byte) error {
 		// the root nodeType information is lost on Unmarshal. This causes issues when we want to
 		// perform a path 'Walk' on the root. If there is more than 1 fork, the root node type
 		// is an edge, so we will deduce this information from index byte array
-		if !bytes.Equal(data[offset:offset+32], make([]byte, 32)) {
-			n.nodeType = nodeTypeEdge
+		if !bytes.Equal(data[offset:offset+32], zero32) && !n.IsEdgeType() {
+			n.makeEdge()
 		}
 		n.forks = make(map[byte]*fork)
 		bb := &bitsForBytes{}

pkg/manifest/mantaray/marshal_test.go

@@ -20,27 +20,33 @@ const testMarshalOutput01 = "52fdfc072182654f163f5f0f9a621d729566c74d10037c4d7bb
 
 const testMarshalOutput02 = "52fdfc072182654f163f5f0f9a621d729566c74d10037c4d7bbb0407d1e2c64905954fb18659339d0b25e0fb9723d3cd5d528fb3c8d495fd157bd7b7a210496952fdfc072182654f163f5f0f9a621d729566c74d10037c4d7bbb0407d1e2c64952fdfc072102654f163f5f0fa0621d729566c74d10037c4d7bbb0407d1e2c64940fcd3072182654f163f5f0f9a621d729566c74d10037c4d7bbb0407d1e2c64952fdfc072182654f163f5f0f9a621d729566c74d10037c4d7bbb0407d1e2c64952e3872548ec012a6e123b60f9177017fb12e57732621d2c1ada267adbe8cc4350f89d6640e3044f163f5f0f9a621d729566c74d10037c4d7bbb0407d1e2c64952fdfc072182654f163f5f0f9a621d729566c74d10037c4d7bbb0407d1e2c64850ff9f642182654f163f5f0f9a621d729566c74d10037c4d7bbb0407d1e2c64952fdfc072182654f163f5f0f9a621d729566c74d10037c4d7bbb0407d1e2c64b50fc98072182654f163f5f0f9a621d729566c74d10037c4d7bbb0407d1e2c64952fdfc072182654f163f5f0f9a621d729566c74d10037c4d7bbb0407d1e2c64a50ff99622182654f163f5f0f9a621d729566c74d10037c4d7bbb0407d1e2c64952fdfc072182654f163f5f0f9a621d729566c74d10037c4d7bbb0407d1e2c64d"
 
-var testEntries = []nodeEntry{
+var testEntries = []NodeEntry{
 	{
-		path: []byte("/"),
-		metadata: map[string]string{
+		Path: []byte("/"),
+		Metadata: map[string]string{
 			"index-document": "aaaaa",
 		},
 	},
 	{
-		path: []byte("aaaaa"),
+		Path: []byte("aaaaa"),
 	},
 	{
-		path: []byte("cc"),
+		Path: []byte("cc"),
 	},
 	{
-		path: []byte("d"),
+		Path: []byte("d"),
 	},
 	{
-		path: []byte("ee"),
+		Path: []byte("ee"),
 	},
 }
 
+type NodeEntry struct {
+	Path     []byte
+	Entry    []byte
+	Metadata map[string]string
+}
+
 func init() {
 	obfuscationKeyFn = mrand.Read
 }
@@ -98,7 +104,7 @@ func TestUnmarshal01(t *testing.T) {
 		t.Fatalf("expected %d forks, got %d", len(testEntries), len(n.forks))
 	}
 	for _, entry := range testEntries {
-		prefix := entry.path
+		prefix := entry.Path
 		f := n.forks[prefix[0]]
 		if f == nil {
 			t.Fatalf("expected to have  fork on byte %x", prefix[:1])
@@ -130,7 +136,7 @@ func TestUnmarshal02(t *testing.T) {
 		t.Fatalf("expected %d forks, got %d", len(testEntries), len(n.forks))
 	}
 	for _, entry := range testEntries {
-		prefix := entry.path
+		prefix := entry.Path
 		f := n.forks[prefix[0]]
 		if f == nil {
 			t.Fatalf("expected to have  fork on byte %x", prefix[:1])
@@ -138,9 +144,9 @@ func TestUnmarshal02(t *testing.T) {
 		if !bytes.Equal(f.prefix, prefix) {
 			t.Fatalf("expected prefix for byte %x to match %s, got %s", prefix[:1], prefix, f.prefix)
 		}
-		if len(entry.metadata) > 0 {
-			if !reflect.DeepEqual(entry.metadata, f.metadata) {
-				t.Fatalf("expected metadata for byte %x to match %s, got %s", prefix[:1], entry.metadata, f.metadata)
+		if len(entry.Metadata) > 0 {
+			if !reflect.DeepEqual(entry.Metadata, f.metadata) {
+				t.Fatalf("expected metadata for byte %x to match %s, got %s", prefix[:1], entry.Metadata, f.metadata)
 			}
 		}
 	}
@@ -159,12 +165,12 @@ func TestMarshal(t *testing.T) {
 		return b
 	}
 	for i := 0; i < len(testEntries); i++ {
-		c := testEntries[i].path
-		e := testEntries[i].entry
+		c := testEntries[i].Path
+		e := testEntries[i].Entry
 		if len(e) == 0 {
 			e = append(make([]byte, 32-len(c)), c...)
 		}
-		m := testEntries[i].metadata
+		m := testEntries[i].Metadata
 		err := n.Add(ctx, c, e, m, nil)
 		if err != nil {
 			t.Fatalf("expected no error, got %v", err)

pkg/manifest/mantaray/node.go

@@ -181,6 +181,9 @@ func (n *Node) Lookup(ctx context.Context, path []byte, l Loader) ([]byte, error
 	if err != nil {
 		return nil, err
 	}
+	if !node.IsValueType() && len(path) > 0 {
+		return nil, notFound(path)
+	}
 	return node.entry, nil
 }
 
@@ -205,6 +208,7 @@ func (n *Node) Add(ctx context.Context, path, entry []byte, metadata map[string]
 
 	if len(path) == 0 {
 		n.entry = entry
+		n.makeValue()
 		if len(metadata) > 0 {
 			n.metadata = metadata
 			n.makeWithMetadata()

pkg/manifest/mantaray/node_test.go

@@ -2,7 +2,7 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
-package mantaray
+package mantaray_test
 
 import (
 	"bytes"
@@ -10,17 +10,13 @@ import (
 	"errors"
 	"strconv"
 	"testing"
-)
 
-type nodeEntry struct {
-	path     []byte
-	entry    []byte
-	metadata map[string]string
-}
+	"github.com/ethersphere/bee/pkg/manifest/mantaray"
+)
 
 func TestNilPath(t *testing.T) {
 	ctx := context.Background()
-	n := New()
+	n := mantaray.New()
 	_, err := n.Lookup(ctx, nil, nil)
 	if err != nil {
 		t.Fatalf("expected no error, got %v", err)
@@ -29,7 +25,7 @@ func TestNilPath(t *testing.T) {
 
 func TestAddAndLookup(t *testing.T) {
 	ctx := context.Background()
-	n := New()
+	n := mantaray.New()
 	testCases := [][]byte{
 		[]byte("aaaaaa"),
 		[]byte("aaaaab"),
@@ -91,6 +87,14 @@ func TestAddAndLookupNode(t *testing.T) {
 				[]byte("robots.txt"),
 			},
 		},
+		{
+			// mantaray.nodePrefixMaxSize number of '.'
+			name: "nested-value-node-is-recognized",
+			toAdd: [][]byte{
+				[]byte("..............................@"),
+				[]byte(".............................."),
+			},
+		},
 		{
 			name: "nested-prefix-is-not-collapsed",
 			toAdd: [][]byte{
@@ -127,7 +131,7 @@ func TestAddAndLookupNode(t *testing.T) {
 	} {
 		ctx := context.Background()
 		t.Run(tc.name, func(t *testing.T) {
-			n := New()
+			n := mantaray.New()
 
 			for i := 0; i < len(tc.toAdd); i++ {
 				c := tc.toAdd[i]
@@ -142,45 +146,79 @@ func TestAddAndLookupNode(t *testing.T) {
 					if err != nil {
 						t.Fatalf("expected no error, got %v", err)
 					}
-					de := append(make([]byte, 32-len(d)), d...)
-					if !bytes.Equal(node.entry, de) {
-						t.Fatalf("expected value %x, got %x", d, node.entry)
-					}
 					if !node.IsValueType() {
-						t.Fatalf("expected value type, got %v", strconv.FormatInt(int64(node.nodeType), 2))
+						t.Fatalf("expected value type, got %v", strconv.FormatInt(int64(node.NodeType()), 2))
+					}
+					de := append(make([]byte, 32-len(d)), d...)
+					if !bytes.Equal(node.Entry(), de) {
+						t.Fatalf("expected value %x, got %x", d, node.Entry())
 					}
 				}
 			}
 		})
+		t.Run(tc.name+"/with load save", func(t *testing.T) {
+			n := mantaray.New()
+
+			for i := 0; i < len(tc.toAdd); i++ {
+				c := tc.toAdd[i]
+				e := append(make([]byte, 32-len(c)), c...)
+				err := n.Add(ctx, c, e, nil, nil)
+				if err != nil {
+					t.Fatalf("expected no error, got %v", err)
+				}
+			}
+			ls := newMockLoadSaver()
+			err := n.Save(ctx, ls)
+			if err != nil {
+				t.Fatal(err)
+			}
+
+			n2 := mantaray.NewNodeRef(n.Reference())
+
+			for j := 0; j < len(tc.toAdd); j++ {
+				d := tc.toAdd[j]
+				node, err := n2.LookupNode(ctx, d, ls)
+				if err != nil {
+					t.Fatalf("expected no error, got %v", err)
+				}
+				if !node.IsValueType() {
+					t.Fatalf("expected value type, got %v", strconv.FormatInt(int64(node.NodeType()), 2))
+				}
+				de := append(make([]byte, 32-len(d)), d...)
+				if !bytes.Equal(node.Entry(), de) {
+					t.Fatalf("expected value %x, got %x", d, node.Entry())
+				}
+			}
+		})
 	}
 }
 
 func TestRemove(t *testing.T) {
 	for _, tc := range []struct {
 		name     string
-		toAdd    []nodeEntry
+		toAdd    []mantaray.NodeEntry
 		toRemove [][]byte
 	}{
 		{
 			name: "simple",
-			toAdd: []nodeEntry{
+			toAdd: []mantaray.NodeEntry{
 				{
-					path: []byte("/"),
-					metadata: map[string]string{
+					Path: []byte("/"),
+					Metadata: map[string]string{
 						"index-document": "index.html",
 					},
 				},
 				{
-					path: []byte("index.html"),
+					Path: []byte("index.html"),
 				},
 				{
-					path: []byte("img/1.png"),
+					Path: []byte("img/1.png"),
 				},
 				{
-					path: []byte("img/2.png"),
+					Path: []byte("img/2.png"),
 				},
 				{
-					path: []byte("robots.txt"),
+					Path: []byte("robots.txt"),
 				},
 			},
 			toRemove: [][]byte{
@@ -189,21 +227,21 @@ func TestRemove(t *testing.T) {
 		},
 		{
 			name: "nested-prefix-is-not-collapsed",
-			toAdd: []nodeEntry{
+			toAdd: []mantaray.NodeEntry{
 				{
-					path: []byte("index.html"),
+					Path: []byte("index.html"),
 				},
 				{
-					path: []byte("img/1.png"),
+					Path: []byte("img/1.png"),
 				},
 				{
-					path: []byte("img/2/test1.png"),
+					Path: []byte("img/2/test1.png"),
 				},
 				{
-					path: []byte("img/2/test2.png"),
+					Path: []byte("img/2/test2.png"),
 				},
 				{
-					path: []byte("robots.txt"),
+					Path: []byte("robots.txt"),
 				},
 			},
 			toRemove: [][]byte{
@@ -213,21 +251,21 @@ func TestRemove(t *testing.T) {
 	} {
 		ctx := context.Background()
 		t.Run(tc.name, func(t *testing.T) {
-			n := New()
+			n := mantaray.New()
 
 			for i := 0; i < len(tc.toAdd); i++ {
-				c := tc.toAdd[i].path
-				e := tc.toAdd[i].entry
+				c := tc.toAdd[i].Path
+				e := tc.toAdd[i].Entry
 				if len(e) == 0 {
 					e = append(make([]byte, 32-len(c)), c...)
 				}
-				m := tc.toAdd[i].metadata
+				m := tc.toAdd[i].Metadata
 				err := n.Add(ctx, c, e, m, nil)
 				if err != nil {
 					t.Fatalf("expected no error, got %v", err)
 				}
 				for j := 0; j < i; j++ {
-					d := tc.toAdd[j].path
+					d := tc.toAdd[j].Path
 					m, err := n.Lookup(ctx, d, nil)
 					if err != nil {
 						t.Fatalf("expected no error, got %v", err)
@@ -246,7 +284,7 @@ func TestRemove(t *testing.T) {
 					t.Fatalf("expected no error, got %v", err)
 				}
 				_, err = n.Lookup(ctx, c, nil)
-				if !errors.Is(err, ErrNotFound) {
+				if !errors.Is(err, mantaray.ErrNotFound) {
 					t.Fatalf("expected not found error, got %v", err)
 				}
 			}
@@ -298,7 +336,7 @@ func TestHasPrefix(t *testing.T) {
 	} {
 		ctx := context.Background()
 		t.Run(tc.name, func(t *testing.T) {
-			n := New()
+			n := mantaray.New()
 
 			for i := 0; i < len(tc.toAdd); i++ {
 				c := tc.toAdd[i]

pkg/manifest/mantaray/walker_test.go

@@ -2,13 +2,15 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
-package mantaray
+package mantaray_test
 
 import (
 	"bytes"
 	"context"
 	"fmt"
 	"testing"
+
+	"github.com/ethersphere/bee/pkg/manifest/mantaray"
 )
 
 func TestWalkNode(t *testing.T) {
@@ -37,41 +39,83 @@ func TestWalkNode(t *testing.T) {
 		},
 	} {
 		ctx := context.Background()
-		t.Run(tc.name, func(t *testing.T) {
-			n := New()
 
-			for i := 0; i < len(tc.toAdd); i++ {
-				c := tc.toAdd[i]
+		createTree := func(t *testing.T, toAdd [][]byte) *mantaray.Node {
+			n := mantaray.New()
+
+			for i := 0; i < len(toAdd); i++ {
+				c := toAdd[i]
 				e := append(make([]byte, 32-len(c)), c...)
 				err := n.Add(ctx, c, e, nil, nil)
 				if err != nil {
 					t.Fatalf("expected no error, got %v", err)
 				}
 			}
+			return n
+		}
+
+		pathExists := func(found []byte, expected [][]byte) bool {
+			pathFound := false
+
+			for i := 0; i < len(tc.expected); i++ {
+				c := tc.expected[i]
+				if bytes.Equal(found, c) {
+					pathFound = true
+					break
+				}
+			}
+			return pathFound
+		}
+
+		t.Run(tc.name, func(t *testing.T) {
+			n := createTree(t, tc.toAdd)
 
 			walkedCount := 0
 
-			walker := func(path []byte, node *Node, err error) error {
+			walker := func(path []byte, node *mantaray.Node, err error) error {
 				walkedCount++
 
-				pathFound := false
-
-				for i := 0; i < len(tc.expected); i++ {
-					c := tc.expected[i]
-					if bytes.Equal(path, c) {
-						pathFound = true
-						break
-					}
+				if !pathExists(path, tc.expected) {
+					return fmt.Errorf("walkFn returned unknown path: %s", path)
 				}
+				return nil
+			}
+			// Expect no errors.
+			err := n.WalkNode(ctx, []byte{}, nil, walker)
+			if err != nil {
+				t.Fatalf("no error expected, found: %s", err)
+			}
 
-				if !pathFound {
+			if len(tc.expected) != walkedCount {
+				t.Errorf("expected %d nodes, got %d", len(tc.expected), walkedCount)
+			}
+		})
+
+		t.Run(tc.name+"/with load save", func(t *testing.T) {
+			n := createTree(t, tc.toAdd)
+
+			ls := newMockLoadSaver()
+
+			err := n.Save(ctx, ls)
+			if err != nil {
+				t.Fatal(err)
+			}
+
+			n2 := mantaray.NewNodeRef(n.Reference())
+
+			walkedCount := 0
+
+			walker := func(path []byte, node *mantaray.Node, err error) error {
+				walkedCount++
+
+				if !pathExists(path, tc.expected) {
 					return fmt.Errorf("walkFn returned unknown path: %s", path)
 				}
 
 				return nil
 			}
 			// Expect no errors.
-			err := n.WalkNode(ctx, []byte{}, nil, walker)
+			err = n2.WalkNode(ctx, []byte{}, ls, walker)
 			if err != nil {
 				t.Fatalf("no error expected, found: %s", err)
 			}
@@ -79,7 +123,6 @@ func TestWalkNode(t *testing.T) {
 			if len(tc.expected) != walkedCount {
 				t.Errorf("expected %d nodes, got %d", len(tc.expected), walkedCount)
 			}
-
 		})
 	}
 }
@@ -97,6 +140,8 @@ func TestWalk(t *testing.T) {
 				[]byte("img/test/"),
 				[]byte("img/test/oho.png"),
 				[]byte("img/test/old/test.png"),
+				// file with same prefix but not a directory prefix
+				[]byte("img/test/old/test.png.backup"),
 				[]byte("robots.txt"),
 			},
 			expected: [][]byte{
@@ -106,46 +151,93 @@ func TestWalk(t *testing.T) {
 				[]byte("img/test/oho.png"),
 				[]byte("img/test/old"),
 				[]byte("img/test/old/test.png"),
+				[]byte("img/test/old/test.png.backup"),
 				[]byte("robots.txt"),
 			},
 		},
 	} {
 		ctx := context.Background()
-		t.Run(tc.name, func(t *testing.T) {
-			n := New()
 
-			for i := 0; i < len(tc.toAdd); i++ {
-				c := tc.toAdd[i]
+		createTree := func(t *testing.T, toAdd [][]byte) *mantaray.Node {
+			n := mantaray.New()
+
+			for i := 0; i < len(toAdd); i++ {
+				c := toAdd[i]
 				e := append(make([]byte, 32-len(c)), c...)
 				err := n.Add(ctx, c, e, nil, nil)
 				if err != nil {
 					t.Fatalf("expected no error, got %v", err)
 				}
 			}
+			return n
+		}
+
+		pathExists := func(found []byte, expected [][]byte) bool {
+			pathFound := false
+
+			for i := 0; i < len(tc.expected); i++ {
+				c := tc.expected[i]
+				if bytes.Equal(found, c) {
+					pathFound = true
+					break
+				}
+			}
+			return pathFound
+		}
+
+		t.Run(tc.name, func(t *testing.T) {
+
+			n := createTree(t, tc.toAdd)
 
 			walkedCount := 0
 
 			walker := func(path []byte, isDir bool, err error) error {
 				walkedCount++
 
-				pathFound := false
-
-				for i := 0; i < len(tc.expected); i++ {
-					c := tc.expected[i]
-					if bytes.Equal(path, c) {
-						pathFound = true
-						break
-					}
+				if !pathExists(path, tc.expected) {
+					return fmt.Errorf("walkFn returned unknown path: %s", path)
 				}
 
-				if !pathFound {
+				return nil
+			}
+			// Expect no errors.
+			err := n.Walk(ctx, []byte{}, nil, walker)
+			if err != nil {
+				t.Fatalf("no error expected, found: %s", err)
+			}
+
+			if len(tc.expected) != walkedCount {
+				t.Errorf("expected %d nodes, got %d", len(tc.expected), walkedCount)
+			}
+
+		})
+
+		t.Run(tc.name+"/with load save", func(t *testing.T) {
+
+			n := createTree(t, tc.toAdd)
+
+			ls := newMockLoadSaver()
+
+			err := n.Save(ctx, ls)
+			if err != nil {
+				t.Fatal(err)
+			}
+
+			n2 := mantaray.NewNodeRef(n.Reference())
+
+			walkedCount := 0
+
+			walker := func(path []byte, isDir bool, err error) error {
+				walkedCount++
+
+				if !pathExists(path, tc.expected) {
 					return fmt.Errorf("walkFn returned unknown path: %s", path)
 				}
 
 				return nil
 			}
 			// Expect no errors.
-			err := n.Walk(ctx, []byte{}, nil, walker)
+			err = n2.Walk(ctx, []byte{}, ls, walker)
 			if err != nil {
 				t.Fatalf("no error expected, found: %s", err)
 			}