Trojan chunk support (#429)

* Trojan chunk support

pkg/crypto/crypto.go

@@ -56,14 +56,14 @@ func NewEthereumAddress(p ecdsa.PublicKey) ([]byte, error) {
 		return nil, errors.New("invalid public key")
 	}
 	pubBytes := elliptic.Marshal(btcec.S256(), p.X, p.Y)
-	pubHash, err := legacyKeccak256(pubBytes[1:])
+	pubHash, err := LegacyKeccak256(pubBytes[1:])
 	if err != nil {
 		return nil, err
 	}
 	return pubHash[12:], err
 }
 
-func legacyKeccak256(data []byte) ([]byte, error) {
+func LegacyKeccak256(data []byte) ([]byte, error) {
 	var err error
 	hasher := sha3.NewLegacyKeccak256()
 	_, err = hasher.Write(data)

pkg/file/file.go

@@ -14,10 +14,6 @@ import (
 	"github.com/ethersphere/bee/pkg/swarm"
 )
 
-var (
-	ChunkWithLengthSize = swarm.ChunkSize + 8
-)
-
 // Joiner returns file data referenced by the given Swarm Address to the given io.Reader.
 //
 // The call returns when the chunk for the given Swarm Address is found,

pkg/file/splitter/internal/job.go

@@ -68,7 +68,7 @@ func NewSimpleSplitterJob(ctx context.Context, putter storage.Putter, spanLength
 		sumCounts:  make([]int, levelBufferLimit),
 		cursors:    make([]int, levelBufferLimit),
 		hasher:     bmtlegacy.New(p),
-		buffer:     make([]byte, file.ChunkWithLengthSize*levelBufferLimit*2), // double size as temp workaround for weak calculation of needed buffer space
+		buffer:     make([]byte, swarm.ChunkWithSpanSize*levelBufferLimit*2), // double size as temp workaround for weak calculation of needed buffer space
 		toEncrypt:  toEncrypt,
 		refSize:    refSize,
 	}

pkg/swarm/swarm.go

@@ -10,6 +10,8 @@ import (
 	"encoding/hex"
 	"encoding/json"
 	"fmt"
+
+	"golang.org/x/crypto/sha3"
 )
 
 const (
@@ -19,6 +21,12 @@ const (
 	HashSize                = 32
 	MaxPO             uint8 = 15
 	MaxBins                 = MaxPO + 1
+	SpanSize                = 8
+	ChunkWithSpanSize       = ChunkSize + SpanSize
+)
+
+var (
+	NewHasher = sha3.NewLegacyKeccak256
 )
 
 // Address represents an address in Swarm metric space of
@@ -94,6 +102,14 @@ func (a Address) MarshalJSON() ([]byte, error) {
 // ZeroAddress is the address that has no value.
 var ZeroAddress = NewAddress(nil)
 
+// Type describes a kind of chunk, whether it is content-addressed or other
+type Type int
+
+const (
+	Unknown Type = iota
+	ContentAddressed
+)
+
 type Chunk interface {
 	Address() Address
 	Data() []byte
@@ -102,6 +118,8 @@ type Chunk interface {
 	TagID() uint32
 	WithTagID(t uint32) Chunk
 	Equal(Chunk) bool
+	Type() Type
+	WithType(t Type) Chunk
 }
 
 type chunk struct {
@@ -109,6 +127,7 @@ type chunk struct {
 	sdata      []byte
 	pinCounter uint64
 	tagID      uint32
+	typ        Type
 }
 
 func NewChunk(addr Address, data []byte) Chunk {
@@ -152,6 +171,15 @@ func (c *chunk) Equal(cp Chunk) bool {
 	return c.Address().Equal(cp.Address()) && bytes.Equal(c.Data(), cp.Data())
 }
 
+func (c *chunk) Type() Type {
+	return c.typ
+}
+
+func (c *chunk) WithType(t Type) Chunk {
+	c.typ = t
+	return c
+}
+
 type ChunkValidator interface {
 	Validate(ch Chunk) (valid bool)
 }

pkg/trojan/error.go

+package trojan
+
+import (
+	"errors"
+	"fmt"
+)
+
+var (
+	// ErrPayloadTooBig is returned when a given payload for a Message type is longer than the maximum amount allowed
+	ErrPayloadTooBig = fmt.Errorf("message payload size cannot be greater than %d bytes", MaxPayloadSize)
+
+	// ErrEmptyTargets is returned when the given target list for a trojan chunk is empty
+	ErrEmptyTargets = errors.New("target list cannot be empty")
+
+	// ErrVarLenTargets is returned when the given target list for a trojan chunk has addresses of different lengths
+	ErrVarLenTargets = errors.New("target list cannot have targets of different length")
+
+	// ErrUnMarshallingTrojanMessage is returned when a trojan message could not be de-serialized
+	ErrUnmarshal = errors.New("trojan message unmarshall error")
+
+	// ErrMinerTimeout is returned when mining a new nonce takes more time than swarm.TrojanMinerTimeout seconds
+	ErrMinerTimeout = errors.New("miner timeout error")
+)

pkg/trojan/export_test.go

+package trojan
+
+var (
+	Contains    = contains
+	HashBytes   = hashBytes
+	PadBytes    = padBytesLeft
+	IsPotential = isPotential
+)

pkg/trojan/message.go

+// Copyright 2020 The Swarm Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package trojan
+
+import (
+	"bytes"
+	"crypto/rand"
+	"encoding/binary"
+	"math/big"
+	"time"
+
+	"github.com/ethersphere/bee/pkg/swarm"
+	bmtlegacy "github.com/ethersphere/bmt/legacy"
+)
+
+// Topic is an alias for a 32 byte fixed-size array which contains an encoding of a message topic
+type Topic [32]byte
+
+// Target is an alias for an address which can be mined to construct a trojan message.
+// Target is like partial address which helps to send message to a particular PO.
+type Target []byte
+
+// Targets is an alias for a collection of targets
+type Targets []Target
+
+// Message represents a trojan message, which is a message that will be hidden within a chunk payload as part of its data
+type Message struct {
+	length  [2]byte // big-endian encoding of Message payload length
+	Topic   Topic
+	Payload []byte // contains the chunk address to be repaired
+	padding []byte
+}
+
+const (
+	// MaxPayloadSize is the maximum allowed payload size for the Message type, in bytes
+	// MaxPayloadSize + Topic + Length + Nonce = Default ChunkSize
+	//    (4030)      +  (32) +   (2)  +  (32) = 4096 Bytes
+	MaxPayloadSize = swarm.ChunkSize - NonceSize - LengthSize - TopicSize
+	NonceSize      = 32
+	LengthSize     = 2
+	TopicSize      = 32
+	MinerTimeout   = 2 // seconds
+)
+
+// NewTopic creates a new Topic variable with the given input string
+// the input string is taken as a byte slice and hashed
+func NewTopic(topic string) Topic {
+	var tpc Topic
+	hasher := swarm.NewHasher()
+	_, err := hasher.Write([]byte(topic))
+	if err != nil {
+		return tpc
+	}
+	sum := hasher.Sum(nil)
+	copy(tpc[:], sum)
+	return tpc
+}
+
+// NewMessage creates a new Message variable with the given topic and payload
+// it finds a length and nonce for the message according to the given input and maximum payload size
+func NewMessage(topic Topic, payload []byte) (Message, error) {
+	if len(payload) > MaxPayloadSize {
+		return Message{}, ErrPayloadTooBig
+	}
+
+	// get length as array of 2 bytes
+	payloadSize := uint16(len(payload))
+
+	// set random bytes as padding
+	paddingLen := MaxPayloadSize - payloadSize
+	padding := make([]byte, paddingLen)
+	if _, err := rand.Read(padding); err != nil {
+		return Message{}, err
+	}
+
+	// create new Message var and set fields
+	m := new(Message)
+	binary.BigEndian.PutUint16(m.length[:], payloadSize)
+	m.Topic = topic
+	m.Payload = payload
+	m.padding = padding
+	return *m, nil
+}
+
+// Wrap creates a new trojan chunk for the given targets and Message
+// a trojan chunk is a content-addressed chunk made up of span, a nonce, and a payload which contains the Message
+// the chunk address will have one of the targets as its prefix and thus will be forwarded to the neighbourhood of the recipient overlay address the target is derived from
+func (m *Message) Wrap(targets Targets) (swarm.Chunk, error) {
+	if err := checkTargets(targets); err != nil {
+		return nil, err
+	}
+
+	span := make([]byte, 8)
+	binary.LittleEndian.PutUint64(span, swarm.ChunkSize)
+	return m.toChunk(targets, span)
+}
+
+// Unwrap creates a new trojan message from the given chunk payload
+// this function assumes the chunk has been validated as a content-addressed chunk
+// it will return the resulting message if the unwrapping is successful, and an error otherwise
+func Unwrap(c swarm.Chunk) (*Message, error) {
+	d := c.Data()
+
+	// unmarshal chunk payload into message
+	m := new(Message)
+	// first 40 bytes are span + nonce
+	err := m.UnmarshalBinary(d[40:])
+
+	return m, err
+}
+
+// IsPotential returns true if the given chunk is a potential trojan
+func isPotential(c swarm.Chunk) bool {
+	// chunk must be content-addressed to be trojan
+	if c.Type() != swarm.ContentAddressed {
+		return false
+	}
+
+	data := c.Data()
+	// check for minimum chunk data length
+	trojanChunkMinDataLen := swarm.SpanSize + NonceSize + TopicSize + LengthSize
+	if len(data) < trojanChunkMinDataLen {
+		return false
+	}
+
+	// check for valid trojan message length in bytes #41 and #42
+	messageLen := int(binary.BigEndian.Uint16(data[40:42]))
+	return trojanChunkMinDataLen+messageLen <= len(data)
+}
+
+// checkTargets verifies that the list of given targets is non empty and with elements of matching size
+func checkTargets(targets Targets) error {
+	if len(targets) == 0 {
+		return ErrEmptyTargets
+	}
+	validLen := len(targets[0]) // take first element as allowed length
+	for i := 1; i < len(targets); i++ {
+		if len(targets[i]) != validLen {
+			return ErrVarLenTargets
+		}
+	}
+	return nil
+}
+
+// toChunk finds a nonce so that when the given trojan chunk fields are hashed, the result will fall in the neighbourhood of one of the given targets
+// this is done by iteratively enumerating different nonces until the BMT hash of the serialization of the trojan chunk fields results in a chunk address that has one of the targets as its prefix
+// the function returns a new chunk, with the found matching hash to be used as its address,
+// and its data set to the serialization of the trojan chunk fields which correctly hash into the matching address
+func (m *Message) toChunk(targets Targets, span []byte) (swarm.Chunk, error) {
+	// start out with random nonce
+	nonce := make([]byte, NonceSize)
+	if _, err := rand.Read(nonce); err != nil {
+		return nil, err
+	}
+	nonceInt := new(big.Int).SetBytes(nonce)
+	targetsLen := len(targets[0])
+
+	// serialize message
+	b, err := m.MarshalBinary() // TODO: this should be encrypted
+	if err != nil {
+		return nil, err
+	}
+
+	// hash chunk fields with different nonces until an acceptable one is found
+	for start := time.Now(); ; {
+		s := append(append(span, nonce...), b...) // serialize chunk fields
+		hash1, err := hashBytes(s)
+		if err != nil {
+			return nil, err
+		}
+
+		// take as much of the hash as the targets are long
+		if contains(targets, hash1[:targetsLen]) {
+			// if nonce found, stop loop and return chunk
+			return swarm.NewChunk(swarm.NewAddress(hash1), s), nil
+		}
+		// else, add 1 to nonce and try again
+		nonceInt.Add(nonceInt, big.NewInt(1))
+		// loop around in case of overflow after 256 bits
+		if nonceInt.BitLen() > (NonceSize * swarm.SpanSize) {
+			// Test if timeout after after every 256 iteration
+			if time.Since(start) > (MinerTimeout * time.Second) {
+				break
+			}
+			nonceInt = big.NewInt(0)
+		}
+		nonce = padBytesLeft(nonceInt.Bytes()) // pad in case Bytes call is not 32 bytes long
+	}
+	return nil, ErrMinerTimeout
+}
+
+// hashBytes hashes the given serialization of chunk fields with the hashing func
+func hashBytes(s []byte) ([]byte, error) {
+	hashPool := bmtlegacy.NewTreePool(swarm.NewHasher, swarm.Branches, bmtlegacy.PoolSize)
+	hasher := bmtlegacy.New(hashPool)
+	hasher.Reset()
+	span := binary.LittleEndian.Uint64(s[:8])
+	err := hasher.SetSpan(int64(span))
+	if err != nil {
+		return nil, err
+	}
+	if _, err := hasher.Write(s[8:]); err != nil {
+		return nil, err
+	}
+	return hasher.Sum(nil), nil
+}
+
+// contains returns whether the given collection contains the given element
+func contains(col Targets, elem []byte) bool {
+	for i := range col {
+		if bytes.Equal(elem, col[i]) {
+			return true
+		}
+	}
+	return false
+}
+
+// padBytesLeft adds 0s to the given byte slice as left padding,
+// returning this as a new byte slice with a length of exactly 32
+// given param is assumed to be at most 32 bytes long
+func padBytesLeft(b []byte) []byte {
+	l := len(b)
+	if l == 32 {
+		return b
+	}
+	bb := make([]byte, 32)
+	copy(bb[32-l:], b)
+	return bb
+}
+
+// MarshalBinary serializes a message struct
+func (m *Message) MarshalBinary() (data []byte, err error) {
+	data = append(m.length[:], m.Topic[:]...)
+	data = append(data, m.Payload...)
+	data = append(data, m.padding...)
+	return data, nil
+}
+
+// UnmarshalBinary deserializes a message struct
+func (m *Message) UnmarshalBinary(data []byte) (err error) {
+	if len(data) < LengthSize+TopicSize {
+		return ErrUnmarshal
+	}
+
+	copy(m.length[:], data[:LengthSize])                    // first 2 bytes are length
+	copy(m.Topic[:], data[LengthSize:LengthSize+TopicSize]) // following 32 bytes are topic
+
+	length := binary.BigEndian.Uint16(m.length[:])
+	if (len(data) - LengthSize - TopicSize) < int(length) {
+		return ErrUnmarshal
+	}
+
+	// rest of the bytes are payload and padding
+	payloadEnd := LengthSize + TopicSize + length
+	m.Payload = data[LengthSize+TopicSize : payloadEnd]
+	m.padding = data[payloadEnd:]
+	return nil
+}

pkg/trojan/message_test.go

+// Copyright 2020 The Swarm Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package trojan_test
+
+import (
+	"bytes"
+	"encoding/binary"
+	"reflect"
+	"testing"
+
+	chunktesting "github.com/ethersphere/bee/pkg/storage/testing"
+	"github.com/ethersphere/bee/pkg/swarm"
+	"github.com/ethersphere/bee/pkg/trojan"
+)
+
+// arbitrary targets for tests
+var t1 = trojan.Target([]byte{57, 120})
+var t2 = trojan.Target([]byte{209, 156})
+var t3 = trojan.Target([]byte{156, 38})
+var t4 = trojan.Target([]byte{89, 19})
+var t5 = trojan.Target([]byte{22, 129})
+var testTargets = trojan.Targets([]trojan.Target{t1, t2, t3, t4, t5})
+
+// arbitrary topic for tests
+var testTopic = trojan.NewTopic("foo")
+
+// newTestMessage creates an arbitrary Message for tests
+func newTestMessage(t *testing.T) trojan.Message {
+	payload := []byte("foopayload")
+	m, err := trojan.NewMessage(testTopic, payload)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	return m
+}
+
+// TestNewMessage tests the correct and incorrect creation of a Message struct
+func TestNewMessage(t *testing.T) {
+	smallPayload := make([]byte, 32)
+	m, err := trojan.NewMessage(testTopic, smallPayload)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// verify topic
+	if m.Topic != testTopic {
+		t.Fatalf("expected message topic to be %v but is %v instead", testTopic, m.Topic)
+	}
+
+	maxPayload := make([]byte, trojan.MaxPayloadSize)
+	if _, err := trojan.NewMessage(testTopic, maxPayload); err != nil {
+		t.Fatal(err)
+	}
+
+	// the creation should fail if the payload is too big
+	invalidPayload := make([]byte, trojan.MaxPayloadSize+1)
+	if _, err := trojan.NewMessage(testTopic, invalidPayload); err != trojan.ErrPayloadTooBig {
+		t.Fatalf("expected error when creating message of invalid payload size to be %q, but got %v", trojan.ErrPayloadTooBig, err)
+	}
+}
+
+// TestWrap tests the creation of a chunk from a list of targets
+// its address length and span should be correct
+// its resulting address should have a prefix which matches one of the given targets
+// its resulting data should have a hash that matches its address exactly
+func TestWrap(t *testing.T) {
+	m := newTestMessage(t)
+	c, err := m.Wrap(testTargets)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	addr := c.Address()
+	addrLen := len(addr.Bytes())
+	if addrLen != swarm.HashSize {
+		t.Fatalf("chunk has an unexpected address length of %d rather than %d", addrLen, swarm.HashSize)
+	}
+
+	addrPrefix := addr.Bytes()[:len(testTargets[0])]
+	if !trojan.Contains(testTargets, addrPrefix) {
+		t.Fatal("chunk address prefix does not match any of the targets")
+	}
+
+	data := c.Data()
+	dataSize := len(data)
+	expectedSize := swarm.ChunkWithSpanSize // span + payload
+	if dataSize != expectedSize {
+		t.Fatalf("chunk data has an unexpected size of %d rather than %d", dataSize, expectedSize)
+	}
+
+	span := binary.LittleEndian.Uint64(data[:8])
+	remainingDataLen := len(data[8:])
+	if int(span) != remainingDataLen {
+		t.Fatalf("chunk span set to %d, but rest of chunk data is of size %d", span, remainingDataLen)
+	}
+
+	dataHash, err := trojan.HashBytes(data)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !bytes.Equal(addr.Bytes(), dataHash) {
+		t.Fatal("chunk address does not match its data hash")
+	}
+}
+
+// TestWrapFail tests that the creation of a chunk fails when given targets are invalid
+func TestWrapFail(t *testing.T) {
+	m := newTestMessage(t)
+
+	emptyTargets := trojan.Targets([]trojan.Target{})
+	if _, err := m.Wrap(emptyTargets); err != trojan.ErrEmptyTargets {
+		t.Fatalf("expected error when creating chunk for empty targets to be %q, but got %v", trojan.ErrEmptyTargets, err)
+	}
+
+	t1 := trojan.Target([]byte{34})
+	t2 := trojan.Target([]byte{25, 120})
+	t3 := trojan.Target([]byte{180, 18, 255})
+	varLenTargets := trojan.Targets([]trojan.Target{t1, t2, t3})
+	if _, err := m.Wrap(varLenTargets); err != trojan.ErrVarLenTargets {
+		t.Fatalf("expected error when creating chunk for variable-length targets to be %q, but got %v", trojan.ErrVarLenTargets, err)
+	}
+}
+
+// TestPadBytes tests that different types of byte slices are correctly padded with leading 0s
+// all slices are interpreted as big-endian
+func TestPadBytes(t *testing.T) {
+	s := make([]byte, 32)
+
+	// empty slice should be unchanged
+	p := trojan.PadBytes(s)
+	if !bytes.Equal(p, s) {
+		t.Fatalf("expected byte padding to result in %x, but is %x", s, p)
+	}
+
+	// slice of length 3
+	s = []byte{255, 128, 64}
+	p = trojan.PadBytes(s)
+	e := append(make([]byte, 29), s...) // 29 zeros plus the 3 original bytes
+	if !bytes.Equal(p, e) {
+		t.Fatalf("expected byte padding to result in %x, but is %x", e, p)
+	}
+
+	// simulate toChunk behavior
+	s = []byte{1, 0, 0, 0}
+
+	p = trojan.PadBytes(s)
+	e = append(make([]byte, 28), s...) // 28 zeros plus the 4 original bytes
+	if !bytes.Equal(p, e) {
+		t.Fatalf("expected byte padding to result in %x, but is %x", e, p)
+	}
+}
+
+// TestUnwrap tests the correct unwrapping of a trojan chunk to obtain a message
+func TestUnwrap(t *testing.T) {
+	m := newTestMessage(t)
+	c, err := m.Wrap(testTargets)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	um, err := trojan.Unwrap(c)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if !reflect.DeepEqual(m, *um) {
+		t.Fatalf("original message does not match unwrapped one")
+	}
+}
+
+// TestIsPotential tests if chunks are correctly interpreted as potentially trojan
+func TestIsPotential(t *testing.T) {
+	c := chunktesting.GenerateTestRandomChunk()
+
+	// invalid type
+	c.WithType(swarm.Unknown)
+	if trojan.IsPotential(c) {
+		t.Fatal("non content-addressed chunk marked as potential trojan")
+	}
+
+	// valid type, but invalid trojan message length
+	c.WithType(swarm.ContentAddressed)
+	length := len(c.Data()) - 73 // go 1 byte over the maximum allowed
+	lengthBuf := make([]byte, 2)
+	binary.BigEndian.PutUint16(lengthBuf, uint16(length))
+	// put invalid length into bytes #41 and #42
+	copy(c.Data()[40:42], lengthBuf)
+	if trojan.IsPotential(c) {
+		t.Fatal("chunk with invalid trojan message length marked as potential trojan")
+	}
+
+	// valid type, but invalid chunk data length
+	data := make([]byte, 10)
+	c = swarm.NewChunk(swarm.ZeroAddress, data)
+	c.WithType(swarm.ContentAddressed)
+	if trojan.IsPotential(c) {
+		t.Fatal("chunk with invalid data length marked as potential trojan")
+	}
+
+	// valid potential trojan
+	m := newTestMessage(t)
+	c, err := m.Wrap(testTargets)
+	if err != nil {
+		t.Fatal(err)
+	}
+	c.WithType(swarm.ContentAddressed)
+	if !trojan.IsPotential(c) {
+		t.Fatal("valid test trojan chunk not marked as potential trojan")
+	}
+}
+
+// TestMessageSerialization tests that the Message type can be correctly serialized and deserialized
+func TestMessageSerialization(t *testing.T) {
+	m := newTestMessage(t)
+
+	sm, err := m.MarshalBinary()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	dsm := new(trojan.Message)
+	err = dsm.UnmarshalBinary(sm)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if !reflect.DeepEqual(m, *dsm) {
+		t.Fatalf("original message does not match deserialized one")
+	}
+}