Merge pull request #7910 from ethereum-optimism/cl/rdb-reader-ci

feat(op-service): Add CI for `rethdb-reader`

.circleci/config.yml

@@ -1227,6 +1227,45 @@ jobs:
           name: "Go mod tidy"
           command: make mod-tidy && git diff --exit-code
 
+  op-service-rethdb-tests:
+    docker:
+      - image: <<pipeline.parameters.ci_builder_image>>
+    steps:
+      - checkout
+      - check-changed:
+          patterns: op-service,op-node
+      - restore_cache:
+          name: Restore Go modules cache
+          key: gomod-{{ checksum "go.sum" }}
+      - run:
+          name: Cargo fmt + clippy
+          command: |
+            cargo +nightly fmt -- --check
+            cargo +nightly clippy --all --all-features -- -D warnings
+          working_directory: op-service/rethdb-reader
+      - run:
+          name: Generate testdata db
+          command: cargo test
+          working_directory: op-service/rethdb-reader
+      - run:
+          name: Build dylib
+          command: cargo build --release
+          working_directory: op-service/rethdb-reader
+      - run:
+          name: Update LD_LIBRARY_PATH
+          command: echo 'export LD_LIBRARY_PATH="${LD_LIBRARY_PATH}:/root/project/op-service/rethdb-reader/target/release"' >> $BASH_ENV
+      - run:
+          name: Run op-service RethDB tests
+          command: |
+            gotestsum --format=standard-verbose --junitfile=/tmp/test-results/op-service.xml \
+            -- -parallel=8 -coverpkg=github.com/ethereum-optimism/optimism/... -coverprofile=coverage.out \
+            -run TestRethDB -tags rethdb -v
+          working_directory: op-service/sources
+      # TODO(CLI-148): Fix codecov
+      #- run:
+          #name: upload coverage
+          #command: codecov --verbose --clean --flags bedrock-rethdb-go-tests
+
   bedrock-go-tests:  # just a helper, that depends on all the actual test jobs
     docker:
       - image: <<pipeline.parameters.ci_builder_image>>
@@ -1442,6 +1481,9 @@ workflows:
           name: op-service-tests
           module: op-service
           requires: ["op-stack-go-lint"]
+      - op-service-rethdb-tests:
+          requires:
+            - op-stack-go-lint
       - go-e2e-test:
           name: op-e2e-WS-tests
           module: op-e2e
@@ -1491,6 +1533,7 @@ workflows:
             - op-e2e-WS-tests
             - op-e2e-HTTP-tests
             - op-e2e-ext-geth-tests
+            - op-service-rethdb-tests
       - docker-build:
           name: op-node-docker-build
           docker_name: op-node

op-service/rethdb-reader/.gitignore

@@ -3,3 +3,6 @@ target/
 
 # Bindings
 rdb.h
+
+# Testdata DB
+testdata/db

op-service/rethdb-reader/Cargo.toml

@@ -9,7 +9,18 @@ name = "rethdbreader"
 crate-type = ["cdylib"]
 
 [dependencies]
-reth = { git = "https://github.com/paradigmxyz/reth.git" }
+# reth
+reth-primitives = { git = "https://github.com/paradigmxyz/reth.git", rev = "b58bfe6" }
+reth-provider = { git = "https://github.com/paradigmxyz/reth.git", rev = "b58bfe6" }
+reth-db = { git = "https://github.com/paradigmxyz/reth.git", rev = "b58bfe6" }
+reth-rpc-types = { git = "https://github.com/paradigmxyz/reth.git", rev = "b58bfe6" }
+reth-blockchain-tree = { git = "https://github.com/paradigmxyz/reth.git", rev = "b58bfe6" }
+
+# misc
 serde = "1.0.190"
 serde_json = "1.0.107"
 anyhow = "1.0.75"
+
+[dev-dependencies]
+reth-revm = { git = "https://github.com/paradigmxyz/reth.git", rev = "b58bfe6" }
+alloy-rlp = "0.3.3"

op-service/rethdb-reader/Makefile

+.PHONY: testdata
+testdata:
+	mkdir -p testdata
+	@echo "Fetching block RLP and receipts for block #18,663,292 from ethereum mainnet"
+	cast rpc debug_getRawBlock 0x11CC77C | jq -r | xxd -r -p > testdata/block.rlp
+	cast rpc debug_getRawReceipts 0x11CC77C | jq -r > testdata/receipts.json
+	@echo "Done. Generating testdata DB & testing integrity..."
+	cargo test

op-service/rethdb-reader/README.md

@@ -27,6 +27,14 @@ cargo doc --open
 cargo +nightly fmt -- && cargo +nightly clippy --all --all-features -- -D warnings
 ```
 
+**Generating `testdata`**
+
+The testdata block and `reth` database can be generated by running the `testdata` make directive:
+
+```sh
+ETH_RPC_URL="<your_L1_RPC_URL>" make testdata
+```
+
 **Generating the C header**
 
 To generate the C header, first install `cbindgen` via `cargo install cbindgen --force`. Then, run the generation script:

op-service/rethdb-reader/rustfmt.toml

+reorder_imports = true
+imports_granularity = "Crate"
+use_small_heuristics = "Max"
+comment_width = 100
+wrap_comments = true
+binop_separator = "Back"
+trailing_comma = "Vertical"
+trailing_semicolon = false
+use_field_init_shorthand = true
+format_code_in_doc_comments = true
+doc_comment_code_block_width = 100

op-service/rethdb-reader/src/lib.rs

@@ -8,8 +8,8 @@ mod receipts;
 /// Read the receipts for a blockhash from the RETH database directly.
 ///
 /// # Safety
-/// - All possible nil pointer dereferences are checked, and the function will return a
-///   failing [ReceiptsResult] if any are found.
+/// - All possible nil pointer dereferences are checked, and the function will return a failing
+///   [ReceiptsResult] if any are found.
 #[no_mangle]
 pub unsafe extern "C" fn rdb_read_receipts(
     block_hash: *const u8,

op-service/rethdb-reader/src/receipts.rs

@@ -2,16 +2,14 @@
 //! [reth] database.
 
 use anyhow::{anyhow, Result};
-use reth::{
-    blockchain_tree::noop::NoopBlockchainTree,
-    primitives::{
-        BlockHashOrNumber, Receipt, TransactionKind, TransactionMeta, TransactionSigned, MAINNET,
-        U128, U256, U64,
-    },
-    providers::{providers::BlockchainProvider, BlockReader, ProviderFactory, ReceiptProvider},
-    rpc::types::{Log, TransactionReceipt},
-    utils::db::open_db_read_only,
+use reth_blockchain_tree::noop::NoopBlockchainTree;
+use reth_db::open_db_read_only;
+use reth_primitives::{
+    BlockHashOrNumber, Receipt, TransactionKind, TransactionMeta, TransactionSigned, MAINNET, U128,
+    U256, U64,
 };
+use reth_provider::{providers::BlockchainProvider, BlockReader, ProviderFactory, ReceiptProvider};
+use reth_rpc_types::{Log, TransactionReceipt};
 use std::{ffi::c_char, path::Path};
 
 /// A [ReceiptsResult] is a wrapper around a JSON string containing serialized [TransactionReceipt]s
@@ -30,28 +28,20 @@ pub struct ReceiptsResult {
 impl ReceiptsResult {
     /// Constructs a successful [ReceiptsResult] from a JSON string.
     pub fn success(data: *mut char, data_len: usize) -> Self {
-        Self {
-            data,
-            data_len,
-            error: false,
-        }
+        Self { data, data_len, error: false }
     }
 
     /// Constructs a failing [ReceiptsResult] with a null pointer to the data.
     pub fn fail() -> Self {
-        Self {
-            data: std::ptr::null_mut(),
-            data_len: 0,
-            error: true,
-        }
+        Self { data: std::ptr::null_mut(), data_len: 0, error: true }
     }
 }
 
 /// Read the receipts for a blockhash from the RETH database directly.
 ///
 /// # Safety
-/// - All possible nil pointer dereferences are checked, and the function will return a
-///   failing [ReceiptsResult] if any are found.
+/// - All possible nil pointer dereferences are checked, and the function will return a failing
+///   [ReceiptsResult] if any are found.
 #[inline(always)]
 pub(crate) unsafe fn read_receipts_inner(
     block_hash: *const u8,
@@ -83,9 +73,8 @@ pub(crate) unsafe fn read_receipts_inner(
     let provider = BlockchainProvider::new(factory, NoopBlockchainTree::default())?;
 
     // Fetch the block and the receipts within it
-    let block = provider
-        .block_by_hash(block_hash.into())?
-        .ok_or(anyhow!("Failed to fetch block"))?;
+    let block =
+        provider.block_by_hash(block_hash.into())?.ok_or(anyhow!("Failed to fetch block"))?;
     let receipts = provider
         .receipts_by_block(BlockHashOrNumber::Hash(block_hash.into()))?
         .ok_or(anyhow!("Failed to fetch block receipts"))?;
@@ -165,11 +154,7 @@ fn build_transaction_receipt_with_block_receipts(
         // TODO pre-byzantium receipts have a post-transaction state root
         state_root: None,
         logs_bloom: receipt.bloom_slow(),
-        status_code: if receipt.success {
-            Some(U64::from(1))
-        } else {
-            Some(U64::from(0))
-        },
+        status_code: if receipt.success { Some(U64::from(1)) } else { Some(U64::from(0)) },
 
         // EIP-4844 fields
         blob_gas_price: None,
@@ -209,3 +194,144 @@ fn build_transaction_receipt_with_block_receipts(
 
     Some(res_receipt)
 }
+
+#[cfg(test)]
+mod test {
+    use super::*;
+    use alloy_rlp::Decodable;
+    use reth_db::database::Database;
+    use reth_primitives::{
+        address, b256, bloom, hex, Address, Block, Bytes, ReceiptWithBloom, Receipts,
+        SealedBlockWithSenders, U8,
+    };
+    use reth_provider::{BlockWriter, BundleStateWithReceipts, DatabaseProvider};
+    use reth_revm::revm::db::BundleState;
+    use std::{ffi::CString, fs::File, path::Path};
+
+    #[inline]
+    fn dummy_block_with_receipts() -> Result<(Block, Vec<Receipt>)> {
+        // To generate testdata (block 18,663,292 on Ethereum Mainnet):
+        // 1. BLOCK RLP: `cast rpc debug_getRawBlock 0x11CC77C | jq -r | xxd -r -p >
+        //    testdata/block.rlp`
+        // 2. RECEIPTS RLP: `cast rpc debug_getRawReceipts 0x11CC77C | jq -r >
+        //    testdata/receipts.json`
+        let block_rlp = include_bytes!("../testdata/block.rlp");
+        let block = Block::decode(&mut block_rlp.as_ref())?;
+
+        let receipt_rlp: Vec<Vec<u8>> = serde_json::from_str(include_str!(
+            "../testdata/receipts.json"
+        ))
+        .map(|v: Vec<String>| {
+            v.into_iter().map(|s| hex::decode(s)).collect::<Result<Vec<Vec<u8>>, _>>()
+        })??;
+        let receipts = receipt_rlp
+            .iter()
+            .map(|r| ReceiptWithBloom::decode(&mut r.as_slice()).map(|r| r.receipt))
+            .collect::<Result<Vec<Receipt>, _>>()?;
+
+        Ok((block, receipts))
+    }
+
+    #[inline]
+    fn open_receipts_testdata_db() -> Result<()> {
+        if File::open("testdata/db").is_ok() {
+            return Ok(())
+        }
+
+        // Open a RW handle to the MDBX database
+        let db = reth_db::init_db(Path::new("testdata/db"), None).map_err(|e| anyhow!(e))?;
+        let pr = DatabaseProvider::new_rw(db.tx_mut()?, MAINNET.clone());
+
+        // Grab the dummy block and receipts
+        let (mut block, receipts) = dummy_block_with_receipts()?;
+
+        // Patch: The block's current state root expects the rest of the chain history to be in the
+        // DB; manually override it. Otherwise, the DatabaseProvider will fail to commit the
+        // block.
+        block.header.state_root = reth_primitives::constants::EMPTY_ROOT_HASH;
+
+        // Fetch the block number and tx senders for bundle state creation.
+        let block_number = block.header.number;
+        let senders = block
+            .body
+            .iter()
+            .map(|tx| tx.recover_signer())
+            .collect::<Option<Vec<Address>>>()
+            .ok_or(anyhow!("Failed to recover signers"))?;
+
+        // Commit the bundle state to the database
+        pr.append_blocks_with_bundle_state(
+            vec![SealedBlockWithSenders { block: block.seal_slow(), senders }],
+            BundleStateWithReceipts::new(
+                BundleState::default(),
+                Receipts::from_block_receipt(receipts),
+                block_number,
+            ),
+            None,
+        )?;
+        pr.commit()?;
+
+        Ok(())
+    }
+
+    #[test]
+    fn fetch_receipts() {
+        open_receipts_testdata_db().unwrap();
+
+        unsafe {
+            let mut block_hash =
+                b256!("6a229123d607c2232a8b0bdd36f90745945d05181018e64e60ff2b93ab6b52e5");
+            let receipts_res = super::read_receipts_inner(
+                block_hash.as_mut_ptr(),
+                32,
+                CString::new("testdata/db").unwrap().into_raw() as *const c_char,
+            )
+            .unwrap();
+
+            let receipts_data =
+                std::slice::from_raw_parts(receipts_res.data as *const u8, receipts_res.data_len);
+            let receipt = {
+                let mut receipts: Vec<TransactionReceipt> =
+                    serde_json::from_slice(receipts_data).unwrap();
+                receipts.remove(0)
+            };
+
+            // Check the first receipt in the block for validity
+            assert_eq!(receipt.transaction_type, U8::from(2));
+            assert_eq!(receipt.status_code, Some(U64::from(1)));
+            assert_eq!(receipt.cumulative_gas_used, U256::from(115_316));
+            assert_eq!(receipt.logs_bloom, bloom!("00200000000000000000000080001000000000000000000000000000000000000000000000000000000000000000100002000100080000000000000000000000000000000000000000000008000000200000000400000000000000000000000000000000000000000000000000000000000000000000000000000010000000000000000400000000000001000000000000000100000000080000004000000000000000000000000000000000000002000000000000000000000000000000000000000006000000000000000000000000000000000000001000000000000000000000200000000000000100000000020000000000000000000000000000000010"));
+            assert_eq!(
+                receipt.logs[0].address,
+                address!("c02aaa39b223fe8d0a0e5c4f27ead9083c756cc2")
+            );
+            assert_eq!(
+                receipt.logs[0].topics[0],
+                b256!("ddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef")
+            );
+            assert_eq!(
+                receipt.logs[0].topics[1],
+                b256!("00000000000000000000000000000000003b3cc22af3ae1eac0440bcee416b40")
+            );
+            assert_eq!(
+                receipt.logs[0].data,
+                Bytes::from_static(
+                    hex!("00000000000000000000000000000000000000000000000008a30cd230000000")
+                        .as_slice()
+                )
+            );
+            assert_eq!(receipt.from, address!("41d3ab85aafed2ef9e644cb7d3bbca2fc4d8cac8"));
+            assert_eq!(receipt.to, Some(address!("00000000003b3cc22af3ae1eac0440bcee416b40")));
+            assert_eq!(
+                receipt.transaction_hash,
+                Some(b256!("88b2d153a4e893ba91ac235325c44b1aa0c802fcb42657701e1a73e1c675f7ca"))
+            );
+
+            assert_eq!(receipt.block_number, Some(U256::from(18_663_292)));
+            assert_eq!(receipt.block_hash, Some(block_hash));
+            assert_eq!(receipt.transaction_index, U64::from(0));
+
+            crate::rdb_free_string(receipts_res.data as *mut c_char);
+        }
+    }
+}

op-service/sources/reth_db_test.go

+//go:build rethdb
+
+package sources
+
+import (
+	"math/big"
+	"testing"
+
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/stretchr/testify/require"
+)
+
+func TestRethDBReceiptsLoad(t *testing.T) {
+	t.Parallel()
+
+	// ETH Mainnet block #18,663,292
+	//
+	// https://etherscan.io/tx/0x88b2d153a4e893ba91ac235325c44b1aa0c802fcb42657701e1a73e1c675f7ca
+	//
+	// NOTE: The block hash differs from the live block due to a state root mismatch. In order to generate
+	//       a testdata database with only this block in it, the state root of the block was modified.
+	//       Old State Root: 0xaf81a692d228d56d35c80d65aeba59636b4671403054f6c57446c0e3e4d951c8
+	//       New State Root (Empty MPT): 0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421
+	blockHash := common.HexToHash("0x6a229123d607c2232a8b0bdd36f90745945d05181018e64e60ff2b93ab6b52e5")
+	res, err := FetchRethReceipts("../rethdb-reader/testdata/db", &blockHash)
+	require.NoError(t, err)
+
+	receipt := (*types.Receipt)(res[0])
+	require.Equal(t, receipt.Type, uint8(2))
+	require.Equal(t, receipt.Status, uint64(1))
+	require.Equal(t, receipt.CumulativeGasUsed, uint64(115_316))
+	require.Equal(t, receipt.Bloom, types.BytesToBloom(common.Hex2Bytes("00200000000000000000000080001000000000000000000000000000000000000000000000000000000000000000100002000100080000000000000000000000000000000000000000000008000000200000000400000000000000000000000000000000000000000000000000000000000000000000000000000010000000000000000400000000000001000000000000000100000000080000004000000000000000000000000000000000000002000000000000000000000000000000000000000006000000000000000000000000000000000000001000000000000000000000200000000000000100000000020000000000000000000000000000000010")))
+	require.Equal(t, receipt.Logs[0].Address, common.HexToAddress("c02aaa39b223fe8d0a0e5c4f27ead9083c756cc2"))
+	require.Equal(t, receipt.Logs[0].Topics[0], common.HexToHash("ddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef"))
+	require.Equal(t, receipt.Logs[0].Topics[1], common.HexToHash("00000000000000000000000000000000003b3cc22af3ae1eac0440bcee416b40"))
+	require.Equal(t, receipt.Logs[0].Data, common.Hex2Bytes("00000000000000000000000000000000000000000000000008a30cd230000000"))
+	require.Equal(t, receipt.TxHash, common.HexToHash("0x88b2d153a4e893ba91ac235325c44b1aa0c802fcb42657701e1a73e1c675f7ca"))
+
+	require.Equal(t, receipt.BlockHash, blockHash)
+	require.Equal(t, receipt.BlockNumber, big.NewInt(18_663_292))
+	require.Equal(t, receipt.TransactionIndex, uint(0))
+}