add trie support to minigeth

minigeth/ethdb/batch.go

+// Copyright 2018 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package ethdb
+
+// IdealBatchSize defines the size of the data batches should ideally add in one
+// write.
+const IdealBatchSize = 100 * 1024
+
+// Batch is a write-only database that commits changes to its host database
+// when Write is called. A batch cannot be used concurrently.
+type Batch interface {
+	KeyValueWriter
+
+	// ValueSize retrieves the amount of data queued up for writing.
+	ValueSize() int
+
+	// Write flushes any accumulated data to disk.
+	Write() error
+
+	// Reset resets the batch for reuse.
+	Reset()
+
+	// Replay replays the batch contents.
+	Replay(w KeyValueWriter) error
+}
+
+// Batcher wraps the NewBatch method of a backing data store.
+type Batcher interface {
+	// NewBatch creates a write-only database that buffers changes to its host db
+	// until a final write is called.
+	NewBatch() Batch
+}
+
+// HookedBatch wraps an arbitrary batch where each operation may be hooked into
+// to monitor from black box code.
+type HookedBatch struct {
+	Batch
+
+	OnPut    func(key []byte, value []byte) // Callback if a key is inserted
+	OnDelete func(key []byte)               // Callback if a key is deleted
+}
+
+// Put inserts the given value into the key-value data store.
+func (b HookedBatch) Put(key []byte, value []byte) error {
+	if b.OnPut != nil {
+		b.OnPut(key, value)
+	}
+	return b.Batch.Put(key, value)
+}
+
+// Delete removes the key from the key-value data store.
+func (b HookedBatch) Delete(key []byte) error {
+	if b.OnDelete != nil {
+		b.OnDelete(key)
+	}
+	return b.Batch.Delete(key)
+}

minigeth/ethdb/database.go

+// Copyright 2014 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+// Package ethdb defines the interfaces for an Ethereum data store.
+package ethdb
+
+import "io"
+
+// KeyValueReader wraps the Has and Get method of a backing data store.
+type KeyValueReader interface {
+	// Has retrieves if a key is present in the key-value data store.
+	Has(key []byte) (bool, error)
+
+	// Get retrieves the given key if it's present in the key-value data store.
+	Get(key []byte) ([]byte, error)
+}
+
+// KeyValueWriter wraps the Put method of a backing data store.
+type KeyValueWriter interface {
+	// Put inserts the given value into the key-value data store.
+	Put(key []byte, value []byte) error
+
+	// Delete removes the key from the key-value data store.
+	Delete(key []byte) error
+}
+
+// Stater wraps the Stat method of a backing data store.
+type Stater interface {
+	// Stat returns a particular internal stat of the database.
+	Stat(property string) (string, error)
+}
+
+// Compacter wraps the Compact method of a backing data store.
+type Compacter interface {
+	// Compact flattens the underlying data store for the given key range. In essence,
+	// deleted and overwritten versions are discarded, and the data is rearranged to
+	// reduce the cost of operations needed to access them.
+	//
+	// A nil start is treated as a key before all keys in the data store; a nil limit
+	// is treated as a key after all keys in the data store. If both is nil then it
+	// will compact entire data store.
+	Compact(start []byte, limit []byte) error
+}
+
+// KeyValueStore contains all the methods required to allow handling different
+// key-value data stores backing the high level database.
+type KeyValueStore interface {
+	KeyValueReader
+	KeyValueWriter
+	Batcher
+	Iteratee
+	Stater
+	Compacter
+	io.Closer
+}
+
+// AncientReader contains the methods required to read from immutable ancient data.
+type AncientReader interface {
+	// HasAncient returns an indicator whether the specified data exists in the
+	// ancient store.
+	HasAncient(kind string, number uint64) (bool, error)
+
+	// Ancient retrieves an ancient binary blob from the append-only immutable files.
+	Ancient(kind string, number uint64) ([]byte, error)
+
+	// ReadAncients retrieves multiple items in sequence, starting from the index 'start'.
+	// It will return
+	//  - at most 'count' items,
+	//  - at least 1 item (even if exceeding the maxBytes), but will otherwise
+	//   return as many items as fit into maxBytes.
+	ReadAncients(kind string, start, count, maxBytes uint64) ([][]byte, error)
+
+	// Ancients returns the ancient item numbers in the ancient store.
+	Ancients() (uint64, error)
+
+	// AncientSize returns the ancient size of the specified category.
+	AncientSize(kind string) (uint64, error)
+}
+
+// AncientWriter contains the methods required to write to immutable ancient data.
+type AncientWriter interface {
+	// AppendAncient injects all binary blobs belong to block at the end of the
+	// append-only immutable table files.
+	AppendAncient(number uint64, hash, header, body, receipt, td []byte) error
+
+	// TruncateAncients discards all but the first n ancient data from the ancient store.
+	TruncateAncients(n uint64) error
+
+	// Sync flushes all in-memory ancient store data to disk.
+	Sync() error
+}
+
+// Reader contains the methods required to read data from both key-value as well as
+// immutable ancient data.
+type Reader interface {
+	KeyValueReader
+	AncientReader
+}
+
+// Writer contains the methods required to write data to both key-value as well as
+// immutable ancient data.
+type Writer interface {
+	KeyValueWriter
+	AncientWriter
+}
+
+// AncientStore contains all the methods required to allow handling different
+// ancient data stores backing immutable chain data store.
+type AncientStore interface {
+	AncientReader
+	AncientWriter
+	io.Closer
+}
+
+// Database contains all the methods required by the high level database to not
+// only access the key-value data store but also the chain freezer.
+type Database interface {
+	Reader
+	Writer
+	Batcher
+	Iteratee
+	Stater
+	Compacter
+	io.Closer
+}

minigeth/ethdb/iterator.go

+// Copyright 2018 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package ethdb
+
+// Iterator iterates over a database's key/value pairs in ascending key order.
+//
+// When it encounters an error any seek will return false and will yield no key/
+// value pairs. The error can be queried by calling the Error method. Calling
+// Release is still necessary.
+//
+// An iterator must be released after use, but it is not necessary to read an
+// iterator until exhaustion. An iterator is not safe for concurrent use, but it
+// is safe to use multiple iterators concurrently.
+type Iterator interface {
+	// Next moves the iterator to the next key/value pair. It returns whether the
+	// iterator is exhausted.
+	Next() bool
+
+	// Error returns any accumulated error. Exhausting all the key/value pairs
+	// is not considered to be an error.
+	Error() error
+
+	// Key returns the key of the current key/value pair, or nil if done. The caller
+	// should not modify the contents of the returned slice, and its contents may
+	// change on the next call to Next.
+	Key() []byte
+
+	// Value returns the value of the current key/value pair, or nil if done. The
+	// caller should not modify the contents of the returned slice, and its contents
+	// may change on the next call to Next.
+	Value() []byte
+
+	// Release releases associated resources. Release should always succeed and can
+	// be called multiple times without causing error.
+	Release()
+}
+
+// Iteratee wraps the NewIterator methods of a backing data store.
+type Iteratee interface {
+	// NewIterator creates a binary-alphabetical iterator over a subset
+	// of database content with a particular key prefix, starting at a particular
+	// initial key (or after, if it does not exist).
+	//
+	// Note: This method assumes that the prefix is NOT part of the start, so there's
+	// no need for the caller to prepend the prefix to the start
+	NewIterator(prefix []byte, start []byte) Iterator
+}

minigeth/main.go

@@ -10,6 +10,7 @@ import (
 	"github.com/ethereum/go-ethereum/core/vm"
 	"github.com/ethereum/go-ethereum/params"
 	"github.com/ethereum/go-ethereum/rlp"
+	"github.com/ethereum/go-ethereum/trie"
 )
 
 func main() {
@@ -24,7 +25,10 @@ func main() {
 	var header types.Header
 	rlpheader := rlp.NewStream(f, 0)
 	rlpheader.Decode(&header)
-	block := types.NewBlockWithHeader(&header)
+	var txs []*types.Transaction
+	var uncles []*types.Header
+	var receipts []*types.Receipt
+	block := types.NewBlock(&header, txs, uncles, receipts, trie.NewStackTrie(nil))
 	fmt.Println("made block, parent:", header.ParentHash)
 
 	processor.Process(block, statedb, vmconfig)

minigeth/trie/encoding.go

+// Copyright 2014 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package trie
+
+// Trie keys are dealt with in three distinct encodings:
+//
+// KEYBYTES encoding contains the actual key and nothing else. This encoding is the
+// input to most API functions.
+//
+// HEX encoding contains one byte for each nibble of the key and an optional trailing
+// 'terminator' byte of value 0x10 which indicates whether or not the node at the key
+// contains a value. Hex key encoding is used for nodes loaded in memory because it's
+// convenient to access.
+//
+// COMPACT encoding is defined by the Ethereum Yellow Paper (it's called "hex prefix
+// encoding" there) and contains the bytes of the key and a flag. The high nibble of the
+// first byte contains the flag; the lowest bit encoding the oddness of the length and
+// the second-lowest encoding whether the node at the key is a value node. The low nibble
+// of the first byte is zero in the case of an even number of nibbles and the first nibble
+// in the case of an odd number. All remaining nibbles (now an even number) fit properly
+// into the remaining bytes. Compact encoding is used for nodes stored on disk.
+
+func hexToCompact(hex []byte) []byte {
+	terminator := byte(0)
+	if hasTerm(hex) {
+		terminator = 1
+		hex = hex[:len(hex)-1]
+	}
+	buf := make([]byte, len(hex)/2+1)
+	buf[0] = terminator << 5 // the flag byte
+	if len(hex)&1 == 1 {
+		buf[0] |= 1 << 4 // odd flag
+		buf[0] |= hex[0] // first nibble is contained in the first byte
+		hex = hex[1:]
+	}
+	decodeNibbles(hex, buf[1:])
+	return buf
+}
+
+// hexToCompactInPlace places the compact key in input buffer, returning the length
+// needed for the representation
+func hexToCompactInPlace(hex []byte) int {
+	var (
+		hexLen    = len(hex) // length of the hex input
+		firstByte = byte(0)
+	)
+	// Check if we have a terminator there
+	if hexLen > 0 && hex[hexLen-1] == 16 {
+		firstByte = 1 << 5
+		hexLen-- // last part was the terminator, ignore that
+	}
+	var (
+		binLen = hexLen/2 + 1
+		ni     = 0 // index in hex
+		bi     = 1 // index in bin (compact)
+	)
+	if hexLen&1 == 1 {
+		firstByte |= 1 << 4 // odd flag
+		firstByte |= hex[0] // first nibble is contained in the first byte
+		ni++
+	}
+	for ; ni < hexLen; bi, ni = bi+1, ni+2 {
+		hex[bi] = hex[ni]<<4 | hex[ni+1]
+	}
+	hex[0] = firstByte
+	return binLen
+}
+
+func compactToHex(compact []byte) []byte {
+	if len(compact) == 0 {
+		return compact
+	}
+	base := keybytesToHex(compact)
+	// delete terminator flag
+	if base[0] < 2 {
+		base = base[:len(base)-1]
+	}
+	// apply odd flag
+	chop := 2 - base[0]&1
+	return base[chop:]
+}
+
+func keybytesToHex(str []byte) []byte {
+	l := len(str)*2 + 1
+	var nibbles = make([]byte, l)
+	for i, b := range str {
+		nibbles[i*2] = b / 16
+		nibbles[i*2+1] = b % 16
+	}
+	nibbles[l-1] = 16
+	return nibbles
+}
+
+// hexToKeybytes turns hex nibbles into key bytes.
+// This can only be used for keys of even length.
+func hexToKeybytes(hex []byte) []byte {
+	if hasTerm(hex) {
+		hex = hex[:len(hex)-1]
+	}
+	if len(hex)&1 != 0 {
+		panic("can't convert hex key of odd length")
+	}
+	key := make([]byte, len(hex)/2)
+	decodeNibbles(hex, key)
+	return key
+}
+
+func decodeNibbles(nibbles []byte, bytes []byte) {
+	for bi, ni := 0, 0; ni < len(nibbles); bi, ni = bi+1, ni+2 {
+		bytes[bi] = nibbles[ni]<<4 | nibbles[ni+1]
+	}
+}
+
+// prefixLen returns the length of the common prefix of a and b.
+func prefixLen(a, b []byte) int {
+	var i, length = 0, len(a)
+	if len(b) < length {
+		length = len(b)
+	}
+	for ; i < length; i++ {
+		if a[i] != b[i] {
+			break
+		}
+	}
+	return i
+}
+
+// hasTerm returns whether a hex key has the terminator flag.
+func hasTerm(s []byte) bool {
+	return len(s) > 0 && s[len(s)-1] == 16
+}

minigeth/trie/extras.go

+package trie
+
+import (
+	"io"
+
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/crypto"
+)
+
+// rawNode is a simple binary blob used to differentiate between collapsed trie
+// nodes and already encoded RLP binary blobs (while at the same time store them
+// in the same cache fields).
+type rawNode []byte
+
+func (n rawNode) cache() (hashNode, bool)   { panic("this should never end up in a live trie") }
+func (n rawNode) fstring(ind string) string { panic("this should never end up in a live trie") }
+
+func (n rawNode) EncodeRLP(w io.Writer) error {
+	_, err := w.Write(n)
+	return err
+}
+
+var (
+	// emptyRoot is the known root hash of an empty trie.
+	emptyRoot = common.HexToHash("56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421")
+
+	// emptyState is the known hash of an empty state trie entry.
+	emptyState = crypto.Keccak256Hash(nil)
+)

minigeth/trie/hasher.go

+// Copyright 2019 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package trie
+
+import (
+	"sync"
+
+	"github.com/ethereum/go-ethereum/crypto"
+	"github.com/ethereum/go-ethereum/rlp"
+	"golang.org/x/crypto/sha3"
+)
+
+type sliceBuffer []byte
+
+func (b *sliceBuffer) Write(data []byte) (n int, err error) {
+	*b = append(*b, data...)
+	return len(data), nil
+}
+
+func (b *sliceBuffer) Reset() {
+	*b = (*b)[:0]
+}
+
+// hasher is a type used for the trie Hash operation. A hasher has some
+// internal preallocated temp space
+type hasher struct {
+	sha      crypto.KeccakState
+	tmp      sliceBuffer
+	parallel bool // Whether to use paralallel threads when hashing
+}
+
+// hasherPool holds pureHashers
+var hasherPool = sync.Pool{
+	New: func() interface{} {
+		return &hasher{
+			tmp: make(sliceBuffer, 0, 550), // cap is as large as a full fullNode.
+			sha: sha3.NewLegacyKeccak256().(crypto.KeccakState),
+		}
+	},
+}
+
+func newHasher(parallel bool) *hasher {
+	h := hasherPool.Get().(*hasher)
+	h.parallel = parallel
+	return h
+}
+
+func returnHasherToPool(h *hasher) {
+	hasherPool.Put(h)
+}
+
+// hash collapses a node down into a hash node, also returning a copy of the
+// original node initialized with the computed hash to replace the original one.
+func (h *hasher) hash(n node, force bool) (hashed node, cached node) {
+	// Return the cached hash if it's available
+	if hash, _ := n.cache(); hash != nil {
+		return hash, n
+	}
+	// Trie not processed yet, walk the children
+	switch n := n.(type) {
+	case *shortNode:
+		collapsed, cached := h.hashShortNodeChildren(n)
+		hashed := h.shortnodeToHash(collapsed, force)
+		// We need to retain the possibly _not_ hashed node, in case it was too
+		// small to be hashed
+		if hn, ok := hashed.(hashNode); ok {
+			cached.flags.hash = hn
+		} else {
+			cached.flags.hash = nil
+		}
+		return hashed, cached
+	case *fullNode:
+		collapsed, cached := h.hashFullNodeChildren(n)
+		hashed = h.fullnodeToHash(collapsed, force)
+		if hn, ok := hashed.(hashNode); ok {
+			cached.flags.hash = hn
+		} else {
+			cached.flags.hash = nil
+		}
+		return hashed, cached
+	default:
+		// Value and hash nodes don't have children so they're left as were
+		return n, n
+	}
+}
+
+// hashShortNodeChildren collapses the short node. The returned collapsed node
+// holds a live reference to the Key, and must not be modified.
+// The cached
+func (h *hasher) hashShortNodeChildren(n *shortNode) (collapsed, cached *shortNode) {
+	// Hash the short node's child, caching the newly hashed subtree
+	collapsed, cached = n.copy(), n.copy()
+	// Previously, we did copy this one. We don't seem to need to actually
+	// do that, since we don't overwrite/reuse keys
+	//cached.Key = common.CopyBytes(n.Key)
+	collapsed.Key = hexToCompact(n.Key)
+	// Unless the child is a valuenode or hashnode, hash it
+	switch n.Val.(type) {
+	case *fullNode, *shortNode:
+		collapsed.Val, cached.Val = h.hash(n.Val, false)
+	}
+	return collapsed, cached
+}
+
+func (h *hasher) hashFullNodeChildren(n *fullNode) (collapsed *fullNode, cached *fullNode) {
+	// Hash the full node's children, caching the newly hashed subtrees
+	cached = n.copy()
+	collapsed = n.copy()
+	if h.parallel {
+		var wg sync.WaitGroup
+		wg.Add(16)
+		for i := 0; i < 16; i++ {
+			go func(i int) {
+				hasher := newHasher(false)
+				if child := n.Children[i]; child != nil {
+					collapsed.Children[i], cached.Children[i] = hasher.hash(child, false)
+				} else {
+					collapsed.Children[i] = nilValueNode
+				}
+				returnHasherToPool(hasher)
+				wg.Done()
+			}(i)
+		}
+		wg.Wait()
+	} else {
+		for i := 0; i < 16; i++ {
+			if child := n.Children[i]; child != nil {
+				collapsed.Children[i], cached.Children[i] = h.hash(child, false)
+			} else {
+				collapsed.Children[i] = nilValueNode
+			}
+		}
+	}
+	return collapsed, cached
+}
+
+// shortnodeToHash creates a hashNode from a shortNode. The supplied shortnode
+// should have hex-type Key, which will be converted (without modification)
+// into compact form for RLP encoding.
+// If the rlp data is smaller than 32 bytes, `nil` is returned.
+func (h *hasher) shortnodeToHash(n *shortNode, force bool) node {
+	h.tmp.Reset()
+	if err := rlp.Encode(&h.tmp, n); err != nil {
+		panic("encode error: " + err.Error())
+	}
+
+	if len(h.tmp) < 32 && !force {
+		return n // Nodes smaller than 32 bytes are stored inside their parent
+	}
+	return h.hashData(h.tmp)
+}
+
+// shortnodeToHash is used to creates a hashNode from a set of hashNodes, (which
+// may contain nil values)
+func (h *hasher) fullnodeToHash(n *fullNode, force bool) node {
+	h.tmp.Reset()
+	// Generate the RLP encoding of the node
+	if err := n.EncodeRLP(&h.tmp); err != nil {
+		panic("encode error: " + err.Error())
+	}
+
+	if len(h.tmp) < 32 && !force {
+		return n // Nodes smaller than 32 bytes are stored inside their parent
+	}
+	return h.hashData(h.tmp)
+}
+
+// hashData hashes the provided data
+func (h *hasher) hashData(data []byte) hashNode {
+	n := make(hashNode, 32)
+	h.sha.Reset()
+	h.sha.Write(data)
+	h.sha.Read(n)
+	return n
+}
+
+// proofHash is used to construct trie proofs, and returns the 'collapsed'
+// node (for later RLP encoding) aswell as the hashed node -- unless the
+// node is smaller than 32 bytes, in which case it will be returned as is.
+// This method does not do anything on value- or hash-nodes.
+func (h *hasher) proofHash(original node) (collapsed, hashed node) {
+	switch n := original.(type) {
+	case *shortNode:
+		sn, _ := h.hashShortNodeChildren(n)
+		return sn, h.shortnodeToHash(sn, false)
+	case *fullNode:
+		fn, _ := h.hashFullNodeChildren(n)
+		return fn, h.fullnodeToHash(fn, false)
+	default:
+		// Value and hash nodes don't have children so they're left as were
+		return n, n
+	}
+}

minigeth/trie/node.go

+// Copyright 2014 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package trie
+
+import (
+	"fmt"
+	"io"
+	"strings"
+
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/rlp"
+)
+
+var indices = []string{"0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "a", "b", "c", "d", "e", "f", "[17]"}
+
+type node interface {
+	fstring(string) string
+	cache() (hashNode, bool)
+}
+
+type (
+	fullNode struct {
+		Children [17]node // Actual trie node data to encode/decode (needs custom encoder)
+		flags    nodeFlag
+	}
+	shortNode struct {
+		Key   []byte
+		Val   node
+		flags nodeFlag
+	}
+	hashNode  []byte
+	valueNode []byte
+)
+
+// nilValueNode is used when collapsing internal trie nodes for hashing, since
+// unset children need to serialize correctly.
+var nilValueNode = valueNode(nil)
+
+// EncodeRLP encodes a full node into the consensus RLP format.
+func (n *fullNode) EncodeRLP(w io.Writer) error {
+	var nodes [17]node
+
+	for i, child := range &n.Children {
+		if child != nil {
+			nodes[i] = child
+		} else {
+			nodes[i] = nilValueNode
+		}
+	}
+	return rlp.Encode(w, nodes)
+}
+
+func (n *fullNode) copy() *fullNode   { copy := *n; return &copy }
+func (n *shortNode) copy() *shortNode { copy := *n; return &copy }
+
+// nodeFlag contains caching-related metadata about a node.
+type nodeFlag struct {
+	hash  hashNode // cached hash of the node (may be nil)
+	dirty bool     // whether the node has changes that must be written to the database
+}
+
+func (n *fullNode) cache() (hashNode, bool)  { return n.flags.hash, n.flags.dirty }
+func (n *shortNode) cache() (hashNode, bool) { return n.flags.hash, n.flags.dirty }
+func (n hashNode) cache() (hashNode, bool)   { return nil, true }
+func (n valueNode) cache() (hashNode, bool)  { return nil, true }
+
+// Pretty printing.
+func (n *fullNode) String() string  { return n.fstring("") }
+func (n *shortNode) String() string { return n.fstring("") }
+func (n hashNode) String() string   { return n.fstring("") }
+func (n valueNode) String() string  { return n.fstring("") }
+
+func (n *fullNode) fstring(ind string) string {
+	resp := fmt.Sprintf("[\n%s  ", ind)
+	for i, node := range &n.Children {
+		if node == nil {
+			resp += fmt.Sprintf("%s: <nil> ", indices[i])
+		} else {
+			resp += fmt.Sprintf("%s: %v", indices[i], node.fstring(ind+"  "))
+		}
+	}
+	return resp + fmt.Sprintf("\n%s] ", ind)
+}
+func (n *shortNode) fstring(ind string) string {
+	return fmt.Sprintf("{%x: %v} ", n.Key, n.Val.fstring(ind+"  "))
+}
+func (n hashNode) fstring(ind string) string {
+	return fmt.Sprintf("<%x> ", []byte(n))
+}
+func (n valueNode) fstring(ind string) string {
+	return fmt.Sprintf("%x ", []byte(n))
+}
+
+func mustDecodeNode(hash, buf []byte) node {
+	n, err := decodeNode(hash, buf)
+	if err != nil {
+		panic(fmt.Sprintf("node %x: %v", hash, err))
+	}
+	return n
+}
+
+// decodeNode parses the RLP encoding of a trie node.
+func decodeNode(hash, buf []byte) (node, error) {
+	if len(buf) == 0 {
+		return nil, io.ErrUnexpectedEOF
+	}
+	elems, _, err := rlp.SplitList(buf)
+	if err != nil {
+		return nil, fmt.Errorf("decode error: %v", err)
+	}
+	switch c, _ := rlp.CountValues(elems); c {
+	case 2:
+		n, err := decodeShort(hash, elems)
+		return n, wrapError(err, "short")
+	case 17:
+		n, err := decodeFull(hash, elems)
+		return n, wrapError(err, "full")
+	default:
+		return nil, fmt.Errorf("invalid number of list elements: %v", c)
+	}
+}
+
+func decodeShort(hash, elems []byte) (node, error) {
+	kbuf, rest, err := rlp.SplitString(elems)
+	if err != nil {
+		return nil, err
+	}
+	flag := nodeFlag{hash: hash}
+	key := compactToHex(kbuf)
+	if hasTerm(key) {
+		// value node
+		val, _, err := rlp.SplitString(rest)
+		if err != nil {
+			return nil, fmt.Errorf("invalid value node: %v", err)
+		}
+		return &shortNode{key, append(valueNode{}, val...), flag}, nil
+	}
+	r, _, err := decodeRef(rest)
+	if err != nil {
+		return nil, wrapError(err, "val")
+	}
+	return &shortNode{key, r, flag}, nil
+}
+
+func decodeFull(hash, elems []byte) (*fullNode, error) {
+	n := &fullNode{flags: nodeFlag{hash: hash}}
+	for i := 0; i < 16; i++ {
+		cld, rest, err := decodeRef(elems)
+		if err != nil {
+			return n, wrapError(err, fmt.Sprintf("[%d]", i))
+		}
+		n.Children[i], elems = cld, rest
+	}
+	val, _, err := rlp.SplitString(elems)
+	if err != nil {
+		return n, err
+	}
+	if len(val) > 0 {
+		n.Children[16] = append(valueNode{}, val...)
+	}
+	return n, nil
+}
+
+const hashLen = len(common.Hash{})
+
+func decodeRef(buf []byte) (node, []byte, error) {
+	kind, val, rest, err := rlp.Split(buf)
+	if err != nil {
+		return nil, buf, err
+	}
+	switch {
+	case kind == rlp.List:
+		// 'embedded' node reference. The encoding must be smaller
+		// than a hash in order to be valid.
+		if size := len(buf) - len(rest); size > hashLen {
+			err := fmt.Errorf("oversized embedded node (size is %d bytes, want size < %d)", size, hashLen)
+			return nil, buf, err
+		}
+		n, err := decodeNode(nil, buf)
+		return n, rest, err
+	case kind == rlp.String && len(val) == 0:
+		// empty node
+		return nil, rest, nil
+	case kind == rlp.String && len(val) == 32:
+		return append(hashNode{}, val...), rest, nil
+	default:
+		return nil, nil, fmt.Errorf("invalid RLP string size %d (want 0 or 32)", len(val))
+	}
+}
+
+// wraps a decoding error with information about the path to the
+// invalid child node (for debugging encoding issues).
+type decodeError struct {
+	what  error
+	stack []string
+}
+
+func wrapError(err error, ctx string) error {
+	if err == nil {
+		return nil
+	}
+	if decErr, ok := err.(*decodeError); ok {
+		decErr.stack = append(decErr.stack, ctx)
+		return decErr
+	}
+	return &decodeError{err, []string{ctx}}
+}
+
+func (err *decodeError) Error() string {
+	return fmt.Sprintf("%v (decode path: %s)", err.what, strings.Join(err.stack, "<-"))
+}

minigeth/trie/stacktrie.go

+// Copyright 2020 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package trie
+
+import (
+	"bufio"
+	"bytes"
+	"encoding/gob"
+	"errors"
+	"io"
+	"sync"
+
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/ethdb"
+
+	//"github.com/ethereum/go-ethereum/log"
+	"github.com/ethereum/go-ethereum/rlp"
+)
+
+var ErrCommitDisabled = errors.New("no database for committing")
+
+var stPool = sync.Pool{
+	New: func() interface{} {
+		return NewStackTrie(nil)
+	},
+}
+
+func stackTrieFromPool(db ethdb.KeyValueWriter) *StackTrie {
+	st := stPool.Get().(*StackTrie)
+	st.db = db
+	return st
+}
+
+func returnToPool(st *StackTrie) {
+	st.Reset()
+	stPool.Put(st)
+}
+
+// StackTrie is a trie implementation that expects keys to be inserted
+// in order. Once it determines that a subtree will no longer be inserted
+// into, it will hash it and free up the memory it uses.
+type StackTrie struct {
+	nodeType  uint8                // node type (as in branch, ext, leaf)
+	val       []byte               // value contained by this node if it's a leaf
+	key       []byte               // key chunk covered by this (full|ext) node
+	keyOffset int                  // offset of the key chunk inside a full key
+	children  [16]*StackTrie       // list of children (for fullnodes and exts)
+	db        ethdb.KeyValueWriter // Pointer to the commit db, can be nil
+}
+
+// NewStackTrie allocates and initializes an empty trie.
+func NewStackTrie(db ethdb.KeyValueWriter) *StackTrie {
+	return &StackTrie{
+		nodeType: emptyNode,
+		db:       db,
+	}
+}
+
+// NewFromBinary initialises a serialized stacktrie with the given db.
+func NewFromBinary(data []byte, db ethdb.KeyValueWriter) (*StackTrie, error) {
+	var st StackTrie
+	if err := st.UnmarshalBinary(data); err != nil {
+		return nil, err
+	}
+	// If a database is used, we need to recursively add it to every child
+	if db != nil {
+		st.setDb(db)
+	}
+	return &st, nil
+}
+
+// MarshalBinary implements encoding.BinaryMarshaler
+func (st *StackTrie) MarshalBinary() (data []byte, err error) {
+	var (
+		b bytes.Buffer
+		w = bufio.NewWriter(&b)
+	)
+	if err := gob.NewEncoder(w).Encode(struct {
+		Nodetype  uint8
+		Val       []byte
+		Key       []byte
+		KeyOffset uint8
+	}{
+		st.nodeType,
+		st.val,
+		st.key,
+		uint8(st.keyOffset),
+	}); err != nil {
+		return nil, err
+	}
+	for _, child := range st.children {
+		if child == nil {
+			w.WriteByte(0)
+			continue
+		}
+		w.WriteByte(1)
+		if childData, err := child.MarshalBinary(); err != nil {
+			return nil, err
+		} else {
+			w.Write(childData)
+		}
+	}
+	w.Flush()
+	return b.Bytes(), nil
+}
+
+// UnmarshalBinary implements encoding.BinaryUnmarshaler
+func (st *StackTrie) UnmarshalBinary(data []byte) error {
+	r := bytes.NewReader(data)
+	return st.unmarshalBinary(r)
+}
+
+func (st *StackTrie) unmarshalBinary(r io.Reader) error {
+	var dec struct {
+		Nodetype  uint8
+		Val       []byte
+		Key       []byte
+		KeyOffset uint8
+	}
+	gob.NewDecoder(r).Decode(&dec)
+	st.nodeType = dec.Nodetype
+	st.val = dec.Val
+	st.key = dec.Key
+	st.keyOffset = int(dec.KeyOffset)
+
+	var hasChild = make([]byte, 1)
+	for i := range st.children {
+		if _, err := r.Read(hasChild); err != nil {
+			return err
+		} else if hasChild[0] == 0 {
+			continue
+		}
+		var child StackTrie
+		child.unmarshalBinary(r)
+		st.children[i] = &child
+	}
+	return nil
+}
+
+func (st *StackTrie) setDb(db ethdb.KeyValueWriter) {
+	st.db = db
+	for _, child := range st.children {
+		if child != nil {
+			child.setDb(db)
+		}
+	}
+}
+
+func newLeaf(ko int, key, val []byte, db ethdb.KeyValueWriter) *StackTrie {
+	st := stackTrieFromPool(db)
+	st.nodeType = leafNode
+	st.keyOffset = ko
+	st.key = append(st.key, key[ko:]...)
+	st.val = val
+	return st
+}
+
+func newExt(ko int, key []byte, child *StackTrie, db ethdb.KeyValueWriter) *StackTrie {
+	st := stackTrieFromPool(db)
+	st.nodeType = extNode
+	st.keyOffset = ko
+	st.key = append(st.key, key[ko:]...)
+	st.children[0] = child
+	return st
+}
+
+// List all values that StackTrie#nodeType can hold
+const (
+	emptyNode = iota
+	branchNode
+	extNode
+	leafNode
+	hashedNode
+)
+
+// TryUpdate inserts a (key, value) pair into the stack trie
+func (st *StackTrie) TryUpdate(key, value []byte) error {
+	k := keybytesToHex(key)
+	if len(value) == 0 {
+		panic("deletion not supported")
+	}
+	st.insert(k[:len(k)-1], value)
+	return nil
+}
+
+func (st *StackTrie) Update(key, value []byte) {
+	if err := st.TryUpdate(key, value); err != nil {
+		//log.Error(fmt.Sprintf("Unhandled trie error: %v", err))
+	}
+}
+
+func (st *StackTrie) Reset() {
+	st.db = nil
+	st.key = st.key[:0]
+	st.val = nil
+	for i := range st.children {
+		st.children[i] = nil
+	}
+	st.nodeType = emptyNode
+	st.keyOffset = 0
+}
+
+// Helper function that, given a full key, determines the index
+// at which the chunk pointed by st.keyOffset is different from
+// the same chunk in the full key.
+func (st *StackTrie) getDiffIndex(key []byte) int {
+	diffindex := 0
+	for ; diffindex < len(st.key) && st.key[diffindex] == key[st.keyOffset+diffindex]; diffindex++ {
+	}
+	return diffindex
+}
+
+// Helper function to that inserts a (key, value) pair into
+// the trie.
+func (st *StackTrie) insert(key, value []byte) {
+	switch st.nodeType {
+	case branchNode: /* Branch */
+		idx := int(key[st.keyOffset])
+		// Unresolve elder siblings
+		for i := idx - 1; i >= 0; i-- {
+			if st.children[i] != nil {
+				if st.children[i].nodeType != hashedNode {
+					st.children[i].hash()
+				}
+				break
+			}
+		}
+		// Add new child
+		if st.children[idx] == nil {
+			st.children[idx] = stackTrieFromPool(st.db)
+			st.children[idx].keyOffset = st.keyOffset + 1
+		}
+		st.children[idx].insert(key, value)
+	case extNode: /* Ext */
+		// Compare both key chunks and see where they differ
+		diffidx := st.getDiffIndex(key)
+
+		// Check if chunks are identical. If so, recurse into
+		// the child node. Otherwise, the key has to be split
+		// into 1) an optional common prefix, 2) the fullnode
+		// representing the two differing path, and 3) a leaf
+		// for each of the differentiated subtrees.
+		if diffidx == len(st.key) {
+			// Ext key and key segment are identical, recurse into
+			// the child node.
+			st.children[0].insert(key, value)
+			return
+		}
+		// Save the original part. Depending if the break is
+		// at the extension's last byte or not, create an
+		// intermediate extension or use the extension's child
+		// node directly.
+		var n *StackTrie
+		if diffidx < len(st.key)-1 {
+			n = newExt(diffidx+1, st.key, st.children[0], st.db)
+		} else {
+			// Break on the last byte, no need to insert
+			// an extension node: reuse the current node
+			n = st.children[0]
+		}
+		// Convert to hash
+		n.hash()
+		var p *StackTrie
+		if diffidx == 0 {
+			// the break is on the first byte, so
+			// the current node is converted into
+			// a branch node.
+			st.children[0] = nil
+			p = st
+			st.nodeType = branchNode
+		} else {
+			// the common prefix is at least one byte
+			// long, insert a new intermediate branch
+			// node.
+			st.children[0] = stackTrieFromPool(st.db)
+			st.children[0].nodeType = branchNode
+			st.children[0].keyOffset = st.keyOffset + diffidx
+			p = st.children[0]
+		}
+		// Create a leaf for the inserted part
+		o := newLeaf(st.keyOffset+diffidx+1, key, value, st.db)
+
+		// Insert both child leaves where they belong:
+		origIdx := st.key[diffidx]
+		newIdx := key[diffidx+st.keyOffset]
+		p.children[origIdx] = n
+		p.children[newIdx] = o
+		st.key = st.key[:diffidx]
+
+	case leafNode: /* Leaf */
+		// Compare both key chunks and see where they differ
+		diffidx := st.getDiffIndex(key)
+
+		// Overwriting a key isn't supported, which means that
+		// the current leaf is expected to be split into 1) an
+		// optional extension for the common prefix of these 2
+		// keys, 2) a fullnode selecting the path on which the
+		// keys differ, and 3) one leaf for the differentiated
+		// component of each key.
+		if diffidx >= len(st.key) {
+			panic("Trying to insert into existing key")
+		}
+
+		// Check if the split occurs at the first nibble of the
+		// chunk. In that case, no prefix extnode is necessary.
+		// Otherwise, create that
+		var p *StackTrie
+		if diffidx == 0 {
+			// Convert current leaf into a branch
+			st.nodeType = branchNode
+			p = st
+			st.children[0] = nil
+		} else {
+			// Convert current node into an ext,
+			// and insert a child branch node.
+			st.nodeType = extNode
+			st.children[0] = NewStackTrie(st.db)
+			st.children[0].nodeType = branchNode
+			st.children[0].keyOffset = st.keyOffset + diffidx
+			p = st.children[0]
+		}
+
+		// Create the two child leaves: the one containing the
+		// original value and the one containing the new value
+		// The child leave will be hashed directly in order to
+		// free up some memory.
+		origIdx := st.key[diffidx]
+		p.children[origIdx] = newLeaf(diffidx+1, st.key, st.val, st.db)
+		p.children[origIdx].hash()
+
+		newIdx := key[diffidx+st.keyOffset]
+		p.children[newIdx] = newLeaf(p.keyOffset+1, key, value, st.db)
+
+		// Finally, cut off the key part that has been passed
+		// over to the children.
+		st.key = st.key[:diffidx]
+		st.val = nil
+	case emptyNode: /* Empty */
+		st.nodeType = leafNode
+		st.key = key[st.keyOffset:]
+		st.val = value
+	case hashedNode:
+		panic("trying to insert into hash")
+	default:
+		panic("invalid type")
+	}
+}
+
+// hash() hashes the node 'st' and converts it into 'hashedNode', if possible.
+// Possible outcomes:
+// 1. The rlp-encoded value was >= 32 bytes:
+//  - Then the 32-byte `hash` will be accessible in `st.val`.
+//  - And the 'st.type' will be 'hashedNode'
+// 2. The rlp-encoded value was < 32 bytes
+//  - Then the <32 byte rlp-encoded value will be accessible in 'st.val'.
+//  - And the 'st.type' will be 'hashedNode' AGAIN
+//
+// This method will also:
+// set 'st.type' to hashedNode
+// clear 'st.key'
+func (st *StackTrie) hash() {
+	/* Shortcut if node is already hashed */
+	if st.nodeType == hashedNode {
+		return
+	}
+	// The 'hasher' is taken from a pool, but we don't actually
+	// claim an instance until all children are done with their hashing,
+	// and we actually need one
+	var h *hasher
+
+	switch st.nodeType {
+	case branchNode:
+		var nodes [17]node
+		for i, child := range st.children {
+			if child == nil {
+				nodes[i] = nilValueNode
+				continue
+			}
+			child.hash()
+			if len(child.val) < 32 {
+				nodes[i] = rawNode(child.val)
+			} else {
+				nodes[i] = hashNode(child.val)
+			}
+			st.children[i] = nil // Reclaim mem from subtree
+			returnToPool(child)
+		}
+		nodes[16] = nilValueNode
+		h = newHasher(false)
+		defer returnHasherToPool(h)
+		h.tmp.Reset()
+		if err := rlp.Encode(&h.tmp, nodes); err != nil {
+			panic(err)
+		}
+	case extNode:
+		st.children[0].hash()
+		h = newHasher(false)
+		defer returnHasherToPool(h)
+		h.tmp.Reset()
+		var valuenode node
+		if len(st.children[0].val) < 32 {
+			valuenode = rawNode(st.children[0].val)
+		} else {
+			valuenode = hashNode(st.children[0].val)
+		}
+		n := struct {
+			Key []byte
+			Val node
+		}{
+			Key: hexToCompact(st.key),
+			Val: valuenode,
+		}
+		if err := rlp.Encode(&h.tmp, n); err != nil {
+			panic(err)
+		}
+		returnToPool(st.children[0])
+		st.children[0] = nil // Reclaim mem from subtree
+	case leafNode:
+		h = newHasher(false)
+		defer returnHasherToPool(h)
+		h.tmp.Reset()
+		st.key = append(st.key, byte(16))
+		sz := hexToCompactInPlace(st.key)
+		n := [][]byte{st.key[:sz], st.val}
+		if err := rlp.Encode(&h.tmp, n); err != nil {
+			panic(err)
+		}
+	case emptyNode:
+		st.val = emptyRoot.Bytes()
+		st.key = st.key[:0]
+		st.nodeType = hashedNode
+		return
+	default:
+		panic("Invalid node type")
+	}
+	st.key = st.key[:0]
+	st.nodeType = hashedNode
+	if len(h.tmp) < 32 {
+		st.val = common.CopyBytes(h.tmp)
+		return
+	}
+	// Write the hash to the 'val'. We allocate a new val here to not mutate
+	// input values
+	st.val = make([]byte, 32)
+	h.sha.Reset()
+	h.sha.Write(h.tmp)
+	h.sha.Read(st.val)
+	if st.db != nil {
+		// TODO! Is it safe to Put the slice here?
+		// Do all db implementations copy the value provided?
+		st.db.Put(st.val, h.tmp)
+	}
+}
+
+// Hash returns the hash of the current node
+func (st *StackTrie) Hash() (h common.Hash) {
+	st.hash()
+	if len(st.val) != 32 {
+		// If the node's RLP isn't 32 bytes long, the node will not
+		// be hashed, and instead contain the  rlp-encoding of the
+		// node. For the top level node, we need to force the hashing.
+		ret := make([]byte, 32)
+		h := newHasher(false)
+		defer returnHasherToPool(h)
+		h.sha.Reset()
+		h.sha.Write(st.val)
+		h.sha.Read(ret)
+		return common.BytesToHash(ret)
+	}
+	return common.BytesToHash(st.val)
+}
+
+// Commit will firstly hash the entrie trie if it's still not hashed
+// and then commit all nodes to the associated database. Actually most
+// of the trie nodes MAY have been committed already. The main purpose
+// here is to commit the root node.
+//
+// The associated database is expected, otherwise the whole commit
+// functionality should be disabled.
+func (st *StackTrie) Commit() (common.Hash, error) {
+	if st.db == nil {
+		return common.Hash{}, ErrCommitDisabled
+	}
+	st.hash()
+	if len(st.val) != 32 {
+		// If the node's RLP isn't 32 bytes long, the node will not
+		// be hashed (and committed), and instead contain the  rlp-encoding of the
+		// node. For the top level node, we need to force the hashing+commit.
+		ret := make([]byte, 32)
+		h := newHasher(false)
+		defer returnHasherToPool(h)
+		h.sha.Reset()
+		h.sha.Write(st.val)
+		h.sha.Read(ret)
+		st.db.Put(ret, st.val)
+		return common.BytesToHash(ret), nil
+	}
+	return common.BytesToHash(st.val), nil
+}