stock trie

helpers/get_block.ts

@@ -87,8 +87,7 @@ function getBlockRlp(block : any): Buffer {
   return rdat
 }
 
-async function main() {
-  const blockNumber = 13247502;
+async function getBlock(blockNumber: Number) {
   let blockData = await provider.send("eth_getBlockByNumber", ["0x"+(blockNumber).toString(16), true])
   const blockHeaderRlp = getBlockRlp(blockData)
   //console.log(blockData)
@@ -98,4 +97,10 @@ async function main() {
   fs.writeFileSync(`data/txs_${blockNumber}`, rlp.encode(txsRlp))
 }
 
+
+async function main() {
+  await getBlock(13247501)
+  await getBlock(13247502)
+}
+
 main().then(() => process.exit(0))

minigeth/core/state/database.go

@@ -20,7 +20,9 @@ type Database struct {
 }
 
 func NewDatabase(header types.Header) Database {
-	triedb := trie.Database{BlockNumber: header.Number, Root: header.Root}
+	//triedb := trie.Database{BlockNumber: header.Number, Root: header.Root}
+	//triedb.Preseed()
+	triedb := trie.NewDatabase(header)
 	return Database{db: &triedb, BlockNumber: header.Number, StateRoot: header.Root}
 }
 
@@ -30,21 +32,22 @@ func (db *Database) ContractCode(addrHash common.Hash, codeHash common.Hash) ([]
 	return code, nil
 }
 
-func (db *Database) CopyTrie(trie Trie) Trie {
-	// TODO: this is wrong
-	return trie
-}
-
 // ContractCodeSize retrieves a particular contracts code's size.
 func (db *Database) ContractCodeSize(addrHash common.Hash, codeHash common.Hash) (int, error) {
 	code := oracle.GetProvedCodeBytes(db.BlockNumber, addrHash, codeHash)
 	return len(code), nil
 }
 
+func (db *Database) CopyTrie(trie Trie) Trie {
+	// TODO: this is wrong
+	//return trie
+	panic("don't copy tries")
+}
+
 // OpenTrie opens the main account trie at a specific root hash.
 func (db *Database) OpenTrie(root common.Hash) (Trie, error) {
 	//tr, err := trie.NewSecure(root, db.db)
-	return trie.New(root, db.db)
+	return trie.NewSecure(root, db.db)
 }
 
 // OpenStorageTrie opens the storage trie of an account.

minigeth/main.go

@@ -17,7 +17,7 @@ func main() {
 	// read header
 	var header types.Header
 	{
-		f, _ := os.Open("data/block_13247502")
+		f, _ := os.Open("data/block_13247501")
 		defer f.Close()
 		rlpheader := rlp.NewStream(f, 0)
 		rlpheader.Decode(&header)
@@ -46,9 +46,10 @@ func main() {
 	fmt.Println("made block, parent:", header.ParentHash)
 
 	// if this is correct, the trie is working
-	if header.TxHash != block.Header().TxHash {
+	// TODO: it's the previous block now
+	/*if header.TxHash != block.Header().TxHash {
 		panic("wrong transactions for block")
-	}
+	}*/
 
 	_, _, _, err := processor.Process(block, statedb, vmconfig)
 	fmt.Println(err)

minigeth/oracle/proved.go

@@ -91,6 +91,23 @@ func unhash(addrHash common.Hash) common.Address {
 	return unhashMap[addrHash]
 }
 
+func GetProofAccount(blockNumber *big.Int, stateRoot common.Hash, addr common.Address) []string {
+	addrHash := crypto.Keccak256Hash(addr[:])
+	unhashMap[addrHash] = addr
+
+	r := jsonreq{Jsonrpc: "2.0", Method: "eth_getProof", Id: 1}
+	r.Params = make([]interface{}, 3)
+	r.Params[0] = addr
+	r.Params[1] = []common.Hash{}
+	r.Params[2] = fmt.Sprintf("0x%x", blockNumber.Int64())
+	jsonData, _ := json.Marshal(r)
+	resp, _ := http.Post(nodeUrl, "application/json", bytes.NewBuffer(jsonData))
+	defer resp.Body.Close()
+	jr := jsonresp{}
+	json.NewDecoder(resp.Body).Decode(&jr)
+	return jr.Result.AccountProof
+}
+
 func GetProvedAccountBytes(blockNumber *big.Int, stateRoot common.Hash, addr common.Address) []byte {
 	fmt.Println("ORACLE GetProvedAccountBytes:", blockNumber, stateRoot, addr)
 	key := fmt.Sprintf("accounts_%d_%s", blockNumber, addr)
@@ -103,7 +120,7 @@ func GetProvedAccountBytes(blockNumber *big.Int, stateRoot common.Hash, addr com
 		r.Params = make([]interface{}, 3)
 		r.Params[0] = addr
 		r.Params[1] = []common.Hash{}
-		r.Params[2] = fmt.Sprintf("0x%x", blockNumber.Int64()-1)
+		r.Params[2] = fmt.Sprintf("0x%x", blockNumber.Int64())
 		jsonData, _ := json.Marshal(r)
 		resp, _ := http.Post(nodeUrl, "application/json", bytes.NewBuffer(jsonData))
 		defer resp.Body.Close()
@@ -137,7 +154,7 @@ func GetProvedCodeBytes(blockNumber *big.Int, addrHash common.Hash, codehash com
 		r := jsonreq{Jsonrpc: "2.0", Method: "eth_getCode", Id: 1}
 		r.Params = make([]interface{}, 2)
 		r.Params[0] = addr
-		r.Params[1] = fmt.Sprintf("0x%x", blockNumber.Int64()-1)
+		r.Params[1] = fmt.Sprintf("0x%x", blockNumber.Int64())
 		jsonData, _ := json.Marshal(r)
 		//fmt.Println(string(jsonData))
 		resp, _ := http.Post(nodeUrl, "application/json", bytes.NewBuffer(jsonData))
@@ -174,7 +191,7 @@ func GetProvedStorage(blockNumber *big.Int, addrHash common.Hash, root common.Ha
 		r.Params = make([]interface{}, 3)
 		r.Params[0] = addr
 		r.Params[1] = [1]common.Hash{skey}
-		r.Params[2] = fmt.Sprintf("0x%x", blockNumber.Int64()-1)
+		r.Params[2] = fmt.Sprintf("0x%x", blockNumber.Int64())
 		jsonData, _ := json.Marshal(r)
 		resp, _ := http.Post(nodeUrl, "application/json", bytes.NewBuffer(jsonData))
 		defer resp.Body.Close()

minigeth/trie/database.go

 package trie
 
 import (
+	"encoding/hex"
 	"fmt"
 	"io"
 	"math/big"
 	"sync"
 
 	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/ethereum/go-ethereum/crypto"
+	"github.com/ethereum/go-ethereum/oracle"
 )
 
 // rawNode is a simple binary blob used to differentiate between collapsed trie
@@ -26,6 +30,62 @@ type Database struct {
 	BlockNumber *big.Int
 	Root        common.Hash
 	lock        sync.RWMutex
+	lookup      map[common.Hash][]byte
+
+	preimages     map[common.Hash][]byte // Preimages of nodes from the secure trie
+	preimagesSize common.StorageSize     // Storage size of the preimages cache
+}
+
+// insertPreimage writes a new trie node pre-image to the memory database if it's
+// yet unknown. The method will NOT make a copy of the slice,
+// only use if the preimage will NOT be changed later on.
+//
+// Note, this method assumes that the database's lock is held!
+func (db *Database) insertPreimage(hash common.Hash, preimage []byte) {
+	// Short circuit if preimage collection is disabled
+	if db.preimages == nil {
+		return
+	}
+	// Track the preimage if a yet unknown one
+	if _, ok := db.preimages[hash]; ok {
+		return
+	}
+	db.preimages[hash] = preimage
+	db.preimagesSize += common.StorageSize(common.HashLength + len(preimage))
+}
+
+// preimage retrieves a cached trie node pre-image from memory. If it cannot be
+// found cached, the method queries the persistent database for the content.
+func (db *Database) preimage(hash common.Hash) []byte {
+	// Short circuit if preimage collection is disabled
+	if db.preimages == nil {
+		return nil
+	}
+	return db.preimages[hash]
+}
+
+func (db *Database) Fetch(addr common.Address) {
+	fmt.Println("prefetch", addr)
+	ap := oracle.GetProofAccount(db.BlockNumber, db.Root, addr)
+
+	for i, s := range ap {
+		ret, _ := hex.DecodeString(s[2:])
+		hash := crypto.Keccak256Hash(ret)
+		db.lookup[hash] = ret
+		fmt.Println(i, hash)
+	}
+}
+
+func NewDatabase(header types.Header) Database {
+	triedb := Database{BlockNumber: header.Number, Root: header.Root}
+	triedb.lookup = make(map[common.Hash][]byte)
+	fmt.Println("init database")
+
+	// fetch the root node
+	triedb.Fetch(common.Address{})
+
+	//panic("preseed")
+	return triedb
 }
 
 // Node retrieves an encoded cached trie node from memory. If it cannot be found
@@ -40,11 +100,17 @@ func (db *Database) Node(hash common.Hash) ([]byte, error) {
 func (db *Database) node(hash common.Hash) node {
 	fmt.Println("trie node", hash)
 	emptyHash := common.Hash{}
+	if hash == emptyHash {
+		panic("empty")
+	}
+	/*emptyHash := common.Hash{}
 	if hash == emptyHash {
 		return nilValueNode
 	}
-	//return hashNode(hash.Bytes())
-	return hashNode(nil)
+	//return hashNode(hash.Bytes())*/
+
+	enc := db.lookup[hash]
+	return mustDecodeNode(hash[:], enc)
 	//return nilValueNode
 }
 

minigeth/trie/secure_trie.go

+// Copyright 2015 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package trie
+
+import (
+	"fmt"
+
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/log"
+)
+
+// SecureTrie wraps a trie with key hashing. In a secure trie, all
+// access operations hash the key using keccak256. This prevents
+// calling code from creating long chains of nodes that
+// increase the access time.
+//
+// Contrary to a regular trie, a SecureTrie can only be created with
+// New and must have an attached database. The database also stores
+// the preimage of each key.
+//
+// SecureTrie is not safe for concurrent use.
+type SecureTrie struct {
+	trie             Trie
+	hashKeyBuf       [common.HashLength]byte
+	secKeyCache      map[string][]byte
+	secKeyCacheOwner *SecureTrie // Pointer to self, replace the key cache on mismatch
+}
+
+// NewSecure creates a trie with an existing root node from a backing database
+// and optional intermediate in-memory node pool.
+//
+// If root is the zero hash or the sha3 hash of an empty string, the
+// trie is initially empty. Otherwise, New will panic if db is nil
+// and returns MissingNodeError if the root node cannot be found.
+//
+// Accessing the trie loads nodes from the database or node pool on demand.
+// Loaded nodes are kept around until their 'cache generation' expires.
+// A new cache generation is created by each call to Commit.
+// cachelimit sets the number of past cache generations to keep.
+func NewSecure(root common.Hash, db *Database) (*SecureTrie, error) {
+	if db == nil {
+		panic("trie.NewSecure called without a database")
+	}
+	trie, err := New(root, db)
+	if err != nil {
+		return nil, err
+	}
+	return &SecureTrie{trie: *trie}, nil
+}
+
+// Get returns the value for key stored in the trie.
+// The value bytes must not be modified by the caller.
+func (t *SecureTrie) Get(key []byte) []byte {
+	res, err := t.TryGet(key)
+	if err != nil {
+		log.Error(fmt.Sprintf("Unhandled trie error: %v", err))
+	}
+	return res
+}
+
+// TryGet returns the value for key stored in the trie.
+// The value bytes must not be modified by the caller.
+// If a node was not found in the database, a MissingNodeError is returned.
+func (t *SecureTrie) TryGet(key []byte) ([]byte, error) {
+	fmt.Println("TryGet", key)
+	t.trie.db.Fetch(common.BytesToAddress(key))
+
+	return t.trie.TryGet(t.hashKey(key))
+}
+
+// TryGetNode attempts to retrieve a trie node by compact-encoded path. It is not
+// possible to use keybyte-encoding as the path might contain odd nibbles.
+func (t *SecureTrie) TryGetNode(path []byte) ([]byte, int, error) {
+	return t.trie.TryGetNode(path)
+}
+
+// Update associates key with value in the trie. Subsequent calls to
+// Get will return value. If value has length zero, any existing value
+// is deleted from the trie and calls to Get will return nil.
+//
+// The value bytes must not be modified by the caller while they are
+// stored in the trie.
+func (t *SecureTrie) Update(key, value []byte) {
+	if err := t.TryUpdate(key, value); err != nil {
+		log.Error(fmt.Sprintf("Unhandled trie error: %v", err))
+	}
+}
+
+// TryUpdate associates key with value in the trie. Subsequent calls to
+// Get will return value. If value has length zero, any existing value
+// is deleted from the trie and calls to Get will return nil.
+//
+// The value bytes must not be modified by the caller while they are
+// stored in the trie.
+//
+// If a node was not found in the database, a MissingNodeError is returned.
+func (t *SecureTrie) TryUpdate(key, value []byte) error {
+	hk := t.hashKey(key)
+	err := t.trie.TryUpdate(hk, value)
+	if err != nil {
+		return err
+	}
+	t.getSecKeyCache()[string(hk)] = common.CopyBytes(key)
+	return nil
+}
+
+// Delete removes any existing value for key from the trie.
+func (t *SecureTrie) Delete(key []byte) {
+	if err := t.TryDelete(key); err != nil {
+		log.Error(fmt.Sprintf("Unhandled trie error: %v", err))
+	}
+}
+
+// TryDelete removes any existing value for key from the trie.
+// If a node was not found in the database, a MissingNodeError is returned.
+func (t *SecureTrie) TryDelete(key []byte) error {
+	hk := t.hashKey(key)
+	delete(t.getSecKeyCache(), string(hk))
+	return t.trie.TryDelete(hk)
+}
+
+// GetKey returns the sha3 preimage of a hashed key that was
+// previously used to store a value.
+func (t *SecureTrie) GetKey(shaKey []byte) []byte {
+	if key, ok := t.getSecKeyCache()[string(shaKey)]; ok {
+		return key
+	}
+	return t.trie.db.preimage(common.BytesToHash(shaKey))
+}
+
+// Commit writes all nodes and the secure hash pre-images to the trie's database.
+// Nodes are stored with their sha3 hash as the key.
+//
+// Committing flushes nodes from memory. Subsequent Get calls will load nodes
+// from the database.
+func (t *SecureTrie) Commit(onleaf LeafCallback) (root common.Hash, err error) {
+	// Write all the pre-images to the actual disk database
+	if len(t.getSecKeyCache()) > 0 {
+		if t.trie.db.preimages != nil { // Ugly direct check but avoids the below write lock
+			t.trie.db.lock.Lock()
+			for hk, key := range t.secKeyCache {
+				t.trie.db.insertPreimage(common.BytesToHash([]byte(hk)), key)
+			}
+			t.trie.db.lock.Unlock()
+		}
+		t.secKeyCache = make(map[string][]byte)
+	}
+	// Commit the trie to its intermediate node database
+	return t.trie.Commit(onleaf)
+}
+
+// Hash returns the root hash of SecureTrie. It does not write to the
+// database and can be used even if the trie doesn't have one.
+func (t *SecureTrie) Hash() common.Hash {
+	return t.trie.Hash()
+}
+
+// Copy returns a copy of SecureTrie.
+func (t *SecureTrie) Copy() *SecureTrie {
+	cpy := *t
+	return &cpy
+}
+
+// NodeIterator returns an iterator that returns nodes of the underlying trie. Iteration
+// starts at the key after the given start key.
+func (t *SecureTrie) NodeIterator(start []byte) NodeIterator {
+	return t.trie.NodeIterator(start)
+}
+
+// hashKey returns the hash of key as an ephemeral buffer.
+// The caller must not hold onto the return value because it will become
+// invalid on the next call to hashKey or secKey.
+func (t *SecureTrie) hashKey(key []byte) []byte {
+	h := newHasher(false)
+	h.sha.Reset()
+	h.sha.Write(key)
+	h.sha.Read(t.hashKeyBuf[:])
+	returnHasherToPool(h)
+	return t.hashKeyBuf[:]
+}
+
+// getSecKeyCache returns the current secure key cache, creating a new one if
+// ownership changed (i.e. the current secure trie is a copy of another owning
+// the actual cache).
+func (t *SecureTrie) getSecKeyCache() map[string][]byte {
+	if t != t.secKeyCacheOwner {
+		t.secKeyCacheOwner = t
+		t.secKeyCache = make(map[string][]byte)
+	}
+	return t.secKeyCache
+}

minigeth/trie/trie.go

@@ -26,7 +26,6 @@ import (
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/crypto"
 	"github.com/ethereum/go-ethereum/log"
-	"github.com/ethereum/go-ethereum/oracle"
 )
 
 var (
@@ -115,15 +114,11 @@ func (t *Trie) Get(key []byte) []byte {
 // The value bytes must not be modified by the caller.
 // If a node was not found in the database, a MissingNodeError is returned.
 func (t *Trie) TryGet(key []byte) ([]byte, error) {
-	fmt.Println("TryGet", key)
-	/*value, newroot, didResolve, err := t.tryGet(t.root, keybytesToHex(key), 0)
+	value, newroot, didResolve, err := t.tryGet(t.root, keybytesToHex(key), 0)
 	if err == nil && didResolve {
 		t.root = newroot
 	}
-	return value, err*/
-	address := common.BytesToAddress(key)
-	enc := oracle.GetProvedAccountBytes(t.db.BlockNumber, t.db.Root, address)
-	return enc, nil
+	return value, err
 }
 
 func (t *Trie) tryGet(origNode node, key []byte, pos int) (value []byte, newnode node, didResolve bool, err error) {