feature: remove panic in Preimage and pass more complicated test

minigeth/core/state/state_object.go

@@ -28,6 +28,7 @@ import (
 	"github.com/ethereum/go-ethereum/metrics"
 	"github.com/ethereum/go-ethereum/oracle"
 	"github.com/ethereum/go-ethereum/rlp"
+	"github.com/ethereum/go-ethereum/trie"
 )
 
 var emptyCodeHash = crypto.Keccak256(nil)
@@ -248,7 +249,7 @@ func (s *stateObject) GetCommittedState(db Database, key common.Hash) common.Has
 		if metrics.EnabledExpensive {
 			meter = &s.db.StorageReads
 		}
-		oracle.PrefetchStorage(db.BlockNumber, s.address, key)
+		oracle.PrefetchStorage(db.BlockNumber, s.address, key, nil)
 		if enc, err = s.getTrie(db).TryGet(key.Bytes()); err != nil {
 			s.setError(err)
 			return common.Hash{}
@@ -357,7 +358,7 @@ func (s *stateObject) updateTrie(db Database) Trie {
 		if (value == common.Hash{}) {
 			//fmt.Println("delete", s.address, key)
 			// Get absense proof of key in case the deletion needs the sister node.
-			oracle.PrefetchStorage(big.NewInt(db.BlockNumber.Int64()+1), s.address, key)
+			oracle.PrefetchStorage(big.NewInt(db.BlockNumber.Int64()+1), s.address, key, trie.GenPossibleShortNodePreimage)
 			s.setError(tr.TryDelete(key[:]))
 		} else {
 			//fmt.Println("update", s.address, key, value)

minigeth/core/state/statedb.go

@@ -31,6 +31,7 @@ import (
 	"github.com/ethereum/go-ethereum/metrics"
 	"github.com/ethereum/go-ethereum/oracle"
 	"github.com/ethereum/go-ethereum/rlp"
+	"github.com/ethereum/go-ethereum/trie"
 )
 
 // for includes we don't have
@@ -457,7 +458,7 @@ func (s *StateDB) deleteStateObject(obj *stateObject) {
 	// Delete the account from the trie
 	addr := obj.Address()
 	// Get absense proof of account in case the deletion needs the sister node.
-	oracle.PrefetchAccount(big.NewInt(s.db.BlockNumber.Int64()+1), addr)
+	oracle.PrefetchAccount(big.NewInt(s.db.BlockNumber.Int64()+1), addr, trie.GenPossibleShortNodePreimage)
 	if err := s.trie.TryDelete(addr[:]); err != nil {
 		s.setError(fmt.Errorf("deleteStateObject (%x) error: %v", addr[:], err))
 	}
@@ -515,7 +516,7 @@ func (s *StateDB) getDeletedStateObject(addr common.Address) *stateObject {
 		if metrics.EnabledExpensive {
 			defer func(start time.Time) { s.AccountReads += time.Since(start) }(time.Now())
 		}
-		oracle.PrefetchAccount(s.db.BlockNumber, addr)
+		oracle.PrefetchAccount(s.db.BlockNumber, addr, nil)
 		enc, err := s.trie.TryGet(addr.Bytes())
 		if err != nil {
 			s.setError(fmt.Errorf("getDeleteStateObject (%x) error: %v", addr.Bytes(), err))

minigeth/oracle/prefetch.go

@@ -108,7 +108,7 @@ func unhash(addrHash common.Hash) common.Address {
 
 var cached = make(map[string]bool)
 
-func PrefetchStorage(blockNumber *big.Int, addr common.Address, skey common.Hash) {
+func PrefetchStorage(blockNumber *big.Int, addr common.Address, skey common.Hash, postProcess func(map[common.Hash][]byte)) {
 	key := fmt.Sprintf("proof_%d_%s_%s", blockNumber, addr, skey)
 	if cached[key] {
 		return
@@ -117,15 +117,24 @@ func PrefetchStorage(blockNumber *big.Int, addr common.Address, skey common.Hash
 
 	ap := getProofAccount(blockNumber, addr, skey, true)
 	//fmt.Println("PrefetchStorage", blockNumber, addr, skey, len(ap))
+	newPreimages := make(map[common.Hash][]byte)
 	for _, s := range ap {
 		ret, _ := hex.DecodeString(s[2:])
 		hash := crypto.Keccak256Hash(ret)
 		//fmt.Println("   ", i, hash)
-		preimages[hash] = ret
+		newPreimages[hash] = ret
+	}
+
+	if postProcess != nil {
+		postProcess(newPreimages)
+	}
+
+	for hash, val := range newPreimages {
+		preimages[hash] = val
 	}
 }
 
-func PrefetchAccount(blockNumber *big.Int, addr common.Address) {
+func PrefetchAccount(blockNumber *big.Int, addr common.Address, postProcess func(map[common.Hash][]byte)) {
 	key := fmt.Sprintf("proof_%d_%s", blockNumber, addr)
 	if cached[key] {
 		return
@@ -133,10 +142,19 @@ func PrefetchAccount(blockNumber *big.Int, addr common.Address) {
 	cached[key] = true
 
 	ap := getProofAccount(blockNumber, addr, common.Hash{}, false)
+	newPreimages := make(map[common.Hash][]byte)
 	for _, s := range ap {
 		ret, _ := hex.DecodeString(s[2:])
 		hash := crypto.Keccak256Hash(ret)
-		preimages[hash] = ret
+		newPreimages[hash] = ret
+	}
+
+	if postProcess != nil {
+		postProcess(newPreimages)
+	}
+
+	for hash, val := range newPreimages {
+		preimages[hash] = val
 	}
 }
 

minigeth/oracle/preimage.go

@@ -18,7 +18,6 @@ func Preimage(hash common.Hash) []byte {
 	ioutil.WriteFile(key, val, 0644)
 	if !ok {
 		fmt.Println("can't find preimage", hash)
-		panic("preimage missing")
 	}
 	return val
 }

minigeth/trie/database.go

@@ -37,7 +37,7 @@ func NewDatabase(header types.Header) Database {
 	triedb := Database{BlockNumber: header.Number, Root: header.Root}
 	//triedb.preimages = make(map[common.Hash][]byte)
 	fmt.Println("init database")
-	oracle.PrefetchAccount(header.Number, common.Address{})
+	oracle.PrefetchAccount(header.Number, common.Address{}, nil)
 
 	//panic("preseed")
 	return triedb
@@ -53,7 +53,10 @@ func (db *Database) Node(hash common.Hash) ([]byte, error) {
 // found in the memory cache.
 func (db *Database) node(hash common.Hash) node {
 	//fmt.Println("node", hash)
-	return mustDecodeNode(hash[:], oracle.Preimage(hash))
+	if val := oracle.Preimage(hash); val != nil {
+		return mustDecodeNode(hash[:], val)
+	}
+	return nil
 }
 
 // insert inserts a collapsed trie node into the memory database.
@@ -65,8 +68,7 @@ func (db *Database) insert(hash common.Hash, size int, node node) {
 	//fmt.Println("insert", hash, size)
 }
 
-func genPossibleShortNodePreimage(pos int) {
-	preimages := oracle.Preimages()
+func GenPossibleShortNodePreimage(preimages map[common.Hash][]byte) {
 	newPreimages := make(map[common.Hash][]byte)
 
 	for _, val := range preimages {
@@ -76,16 +78,9 @@ func genPossibleShortNodePreimage(pos int) {
 		}
 
 		if node, ok := node.(*shortNode); ok {
-			var begins []int
-			for i := range node.Key {
-				if node.Key[i] == byte(pos) && i+2 < len(node.Key) {
-					begins = append(begins, i+1)
-				}
-			}
-
-			for _, begin := range begins {
+			for i := len(node.Key) - 1; i > 0; i-- {
 				n := shortNode{
-					Key: hexToCompact(node.Key[begin:]),
+					Key: hexToCompact(node.Key[i:]),
 					Val: node.Val,
 				}
 				buf := new(bytes.Buffer)

minigeth/trie/trie.go

@@ -443,16 +443,10 @@ func (t *Trie) delete(n node, prefix, key []byte) (bool, node, error) {
 				// might not be loaded yet, resolve it just for this
 				// check.
 
-				// TODO: Try to cache short nodes already generated
-				genPossibleShortNodePreimage(pos)
-
-				// TODO: We might want to ignore this error and remove the panic in oracle.Preimage
-				//       in case the sister node in deletion is an extension node.
-				// remove this optimisticly? if it's not a shortNode, it doesn't do anything
-				cnode, err := t.resolve(n.Children[pos], prefix)
-				if err != nil {
-					return false, nil, err
-				}
+				// When node is not resolved in next block's absence proof,
+				// it must be an extension node if the state transition is
+				// valid, so we ignore the error here.
+				cnode, _ := t.resolve(n.Children[pos], prefix)
 				if cnode, ok := cnode.(*shortNode); ok {
 					k := append([]byte{byte(pos)}, cnode.Key...)
 					return true, &shortNode{k, cnode.Val, t.newFlag()}, nil