add retrieval protocol (#4)

pkg/p2p/p2p.go

@@ -27,6 +27,11 @@ type Stream interface {
 	io.Closer
 }
 
+// PeerSuggester suggests a peer to retrieve a chunk from
+type PeerSuggester interface {
+	SuggestPeer(addr swarm.Address) (peerAddr swarm.Address, err error)
+}
+
 type ProtocolSpec struct {
 	Name        string
 	StreamSpecs []StreamSpec

pkg/retrieval/pb/doc.go

+// Copyright 2020 The Swarm Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+//go:generate sh -c "protoc -I . -I \"$(go list -f '{{ .Dir }}' -m github.com/gogo/protobuf)/protobuf\" --gogofaster_out=. retrieval.proto"
+
+package pb

pkg/retrieval/pb/retrieval.proto

+// Copyright 2020 The Swarm Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+syntax = "proto3";
+
+package pb;
+
+message Request {
+    bytes Addr = 1;
+}
+
+message Delivery {
+    bytes Data = 1;
+}

pkg/retrieval/retrieval.go

+// Copyright 2020 The Swarm Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package retrieval
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/ethersphere/bee/pkg/logging"
+	"github.com/ethersphere/bee/pkg/p2p"
+	"github.com/ethersphere/bee/pkg/p2p/protobuf"
+	pb "github.com/ethersphere/bee/pkg/retrieval/pb"
+	"github.com/ethersphere/bee/pkg/storage"
+	"github.com/ethersphere/bee/pkg/swarm"
+)
+
+const (
+	protocolName  = "retrieval"
+	streamName    = "retrieval"
+	streamVersion = "1.0.0"
+)
+
+type Service struct {
+	streamer      p2p.Streamer
+	peerSuggester p2p.PeerSuggester
+	storer        storage.Storer
+	logger        logging.Logger
+}
+
+type Options struct {
+	Streamer      p2p.Streamer
+	PeerSuggester p2p.PeerSuggester
+	Storer        storage.Storer
+	Logger        logging.Logger
+}
+
+type Storer interface {
+}
+
+func New(o Options) *Service {
+	return &Service{
+		streamer:      o.Streamer,
+		peerSuggester: o.PeerSuggester,
+		storer:        o.Storer,
+		logger:        o.Logger,
+	}
+}
+
+func (s *Service) Protocol() p2p.ProtocolSpec {
+	return p2p.ProtocolSpec{
+		Name: protocolName,
+		StreamSpecs: []p2p.StreamSpec{
+			{
+				Name:    streamName,
+				Version: streamVersion,
+				Handler: s.Handler,
+			},
+		},
+	}
+}
+
+func (s *Service) RetrieveChunk(ctx context.Context, addr swarm.Address) (data []byte, err error) {
+	peerID, err := s.peerSuggester.SuggestPeer(addr)
+	if err != nil {
+		return nil, err
+	}
+	stream, err := s.streamer.NewStream(ctx, peerID, protocolName, streamName, streamVersion)
+	if err != nil {
+		return nil, fmt.Errorf("new stream: %w", err)
+	}
+	defer stream.Close()
+
+	w, r := protobuf.NewWriterAndReader(stream)
+
+	if err := w.WriteMsg(&pb.Request{
+		Addr: addr.Bytes(),
+	}); err != nil {
+		return nil, fmt.Errorf("stream write: %w", err)
+	}
+
+	var d pb.Delivery
+	if err := r.ReadMsg(&d); err != nil {
+		return nil, err
+	}
+
+	return d.Data, nil
+}
+
+func (s *Service) Handler(p p2p.Peer, stream p2p.Stream) error {
+	w, r := protobuf.NewWriterAndReader(stream)
+	defer stream.Close()
+	var req pb.Request
+	if err := r.ReadMsg(&req); err != nil {
+		return err
+	}
+
+	data, err := s.storer.Get(context.TODO(), swarm.NewAddress(req.Addr))
+	if err != nil {
+		return err
+	}
+
+	if err := w.WriteMsg(&pb.Delivery{
+		Data: data,
+	}); err != nil {
+		return err
+	}
+
+	return nil
+}

pkg/retrieval/retrieval_test.go

+// Copyright 2020 The Swarm Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package retrieval_test
+
+import (
+	"bytes"
+	"context"
+	"encoding/hex"
+	"io/ioutil"
+	"testing"
+	"time"
+
+	"github.com/ethersphere/bee/pkg/logging"
+	"github.com/ethersphere/bee/pkg/p2p/protobuf"
+	"github.com/ethersphere/bee/pkg/p2p/streamtest"
+	"github.com/ethersphere/bee/pkg/retrieval"
+	pb "github.com/ethersphere/bee/pkg/retrieval/pb"
+	storemock "github.com/ethersphere/bee/pkg/storage/mock"
+	"github.com/ethersphere/bee/pkg/swarm"
+)
+
+var testTimeout = 5 * time.Second
+
+// TestDelivery tests that a naive request -> delivery flow works.
+func TestDelivery(t *testing.T) {
+	logger := logging.New(ioutil.Discard, 0)
+
+	mockStorer := storemock.NewStorer()
+	reqAddr, err := swarm.ParseHexAddress("00112233")
+	if err != nil {
+		t.Fatal(err)
+	}
+	reqData := []byte("data data data")
+
+	// put testdata in the mock store of the server
+	_ = mockStorer.Put(context.TODO(), reqAddr, reqData)
+
+	// create the server that will handle the request and will serve the response
+	server := retrieval.New(retrieval.Options{
+		Storer: mockStorer,
+		Logger: logger,
+	})
+	recorder := streamtest.New(
+		streamtest.WithProtocols(server.Protocol()),
+	)
+
+	// client mock storer does not store any data at this point
+	// but should be checked at at the end of the test for the
+	// presence of the reqAddr key and value to ensure delivery
+	// was successful
+	clientMockStorer := storemock.NewStorer()
+
+	ps := mockPeerSuggester{spFunc: func(_ swarm.Address) (swarm.Address, error) {
+		v, err := swarm.ParseHexAddress("9ee7add7")
+		return v, err
+	}}
+	client := retrieval.New(retrieval.Options{
+		Streamer:      recorder,
+		PeerSuggester: ps,
+		Storer:        clientMockStorer,
+		Logger:        logger,
+	})
+	ctx, cancel := context.WithTimeout(context.Background(), testTimeout)
+	defer cancel()
+	v, err := client.RetrieveChunk(ctx, reqAddr)
+	if err != nil {
+		return
+	}
+	if !bytes.Equal(v, reqData) {
+		t.Fatalf("request and response data not equal. got %s want %s", v, reqData)
+	}
+	peerID, _ := ps.SuggestPeer(swarm.ZeroAddress)
+	records, err := recorder.Records(peerID, "retrieval", "retrieval", "1.0.0")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if l := len(records); l != 1 {
+		t.Fatalf("got %v records, want %v", l, 1)
+	}
+
+	record := records[0]
+
+	messages, err := protobuf.ReadMessages(
+		bytes.NewReader(record.In()),
+		func() protobuf.Message { return new(pb.Request) },
+	)
+	if err != nil {
+		t.Fatal(err)
+	}
+	var reqs []string
+	for _, m := range messages {
+		reqs = append(reqs, hex.EncodeToString(m.(*pb.Request).Addr))
+	}
+
+	if len(reqs) != 1 {
+		t.Fatalf("got too many requests. want 1 got %d", len(reqs))
+	}
+
+	messages, err = protobuf.ReadMessages(
+		bytes.NewReader(record.Out()),
+		func() protobuf.Message { return new(pb.Delivery) },
+	)
+	if err != nil {
+		t.Fatal(err)
+	}
+	var gotDeliveries []string
+	for _, m := range messages {
+		gotDeliveries = append(gotDeliveries, string(m.(*pb.Delivery).Data))
+	}
+
+	if len(gotDeliveries) != 1 {
+		t.Fatalf("got too many deliveries. want 1 got %d", len(gotDeliveries))
+	}
+
+}
+
+type mockPeerSuggester struct {
+	spFunc func(swarm.Address) (swarm.Address, error)
+}
+
+func (v mockPeerSuggester) SuggestPeer(addr swarm.Address) (swarm.Address, error) {
+	return v.spFunc(addr)
+}

pkg/storage/mock/storer.go

+// Copyright 2020 The Swarm Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package mock
+
+import (
+	"context"
+
+	"github.com/ethersphere/bee/pkg/storage"
+	"github.com/ethersphere/bee/pkg/swarm"
+)
+
+type mockStorer struct {
+	store map[string][]byte
+}
+
+func NewStorer() storage.Storer {
+	s := &mockStorer{
+		store: make(map[string][]byte),
+	}
+
+	return s
+}
+
+func (m *mockStorer) Get(ctx context.Context, addr swarm.Address) (data []byte, err error) {
+	v, has := m.store[addr.String()]
+	if !has {
+		return nil, storage.ErrNotFound
+	}
+	return v, nil
+}
+
+func (m *mockStorer) Put(ctx context.Context, addr swarm.Address, data []byte) error {
+	m.store[addr.String()] = data
+	return nil
+}

pkg/storage/mock/storer_test.go

+package mock_test
+
+import (
+	"bytes"
+	"context"
+	"testing"
+
+	"github.com/ethersphere/bee/pkg/storage"
+	"github.com/ethersphere/bee/pkg/storage/mock"
+	"github.com/ethersphere/bee/pkg/swarm"
+)
+
+func TestMockStorer(t *testing.T) {
+	s := mock.NewStorer()
+
+	keyFound, err := swarm.ParseHexAddress("aabbcc")
+	if err != nil {
+		t.Fatal(err)
+	}
+	keyNotFound, err := swarm.ParseHexAddress("bbccdd")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	valueFound := []byte("data data data")
+
+	ctx := context.Background()
+	if _, err := s.Get(ctx, keyFound); err != storage.ErrNotFound {
+		t.Fatalf("expected ErrNotFound, got %v", err)
+	}
+
+	if _, err := s.Get(ctx, keyNotFound); err != storage.ErrNotFound {
+		t.Fatalf("expected ErrNotFound, got %v", err)
+	}
+
+	if err := s.Put(ctx, keyFound, valueFound); err != nil {
+		t.Fatalf("expected not error but got: %v", err)
+	}
+
+	if data, err := s.Get(ctx, keyFound); err != nil {
+		t.Fatalf("expected not error but got: %v", err)
+
+	} else {
+		if !bytes.Equal(data, valueFound) {
+			t.Fatalf("expected value %s but got %s", valueFound, data)
+		}
+	}
+}

pkg/storage/store.go

+// Copyright 2020 The Swarm Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.package storage
+
+package storage
+
+import (
+	"context"
+	"errors"
+
+	"github.com/ethersphere/bee/pkg/swarm"
+)
+
+var ErrNotFound = errors.New("storage: not found")
+
+type Storer interface {
+	Get(ctx context.Context, addr swarm.Address) (data []byte, err error)
+	Put(ctx context.Context, addr swarm.Address, data []byte) error
+}