op-e2e: add test option to utilize external eth clients for L2 (#6280)

* op-e2e: add test option to use external eth client

This change is an attempt to allow other Optimism clients like op-erigon
to utilize the op-e2e tests in a minimally invasive way.  Almost all of
the tests are unchanged, with some small structural changes to the setup
to allow for utilization of an in process Geth, or extra-process generic
Ethereum client.

This change establishes a convention for external clients (or shims) to
follow for interoperability with these tests.  The external process shim
must accept a '--config <path>' flag.  This flag specifies a path to a
JSON file which contains necessary execution parameters. These
parameters include the data dir to use for the client, the JWT path, the
genesis JSON path, and the chain ID.

Utilizing the external binary is is generally more resource intensive
than the in process Geth path, but, utilizing parallelism the test suite
there is no noticable difference in execution time between the in
process and an extra-process op-geth.

This commit simply performs the wiring, while the next commit will add
an op-geth consumer and wire into CI.

* op-e2e: add basic external geth runner

This commit adds an example external shim which utilizes the code in the
previous commit.  It also integrates it into the Makefile and into CI
(which unfortunately does not use the Makefile).

To utilize this geth runner locally you can execute:

  make test-external-geth

from the op-e2e directory.  See the README.md included in external_geth
for more details.

---------
Co-authored-by: Jason Yellick <jason@enya.ai>

.circleci/config.yml

@@ -778,6 +778,9 @@ jobs:
       use_http:
         description: If the op-e2e package should use HTTP clients
         type: string
+      use_external:
+        description: The extra-process shim (if any) that should be used
+        type: string
     docker:
       - image: us-docker.pkg.dev/oplabs-tools-artifacts/images/ci-builder:latest
     resource_class: xlarge
@@ -788,6 +791,13 @@ jobs:
       - run:
           name: prep results dir
           command: mkdir -p /tmp/test-results
+      - when:
+          condition: <<parameters.use_external>>
+          steps:
+            - run:
+                name: Build Shim
+                command: make -C <<parameters.use_external>>
+                working_directory: <<parameters.module>>
       - run:
           name: install geth
           command: make install-geth
@@ -807,9 +817,11 @@ jobs:
             # Note: We don't use circle CI test splits because we need to split by test name, not by package. There is an additional
             # constraint that gotestsum does not currently (nor likely will) accept files from different pacakges when building.
             # Note: -parallel must be set to match the number of cores in the resource class
+            export TEST_SUFFIX="<<parameters.use_external>>"
+            export EXTERNAL_L2="$(test -z '<<parameters.use_external>>' || echo '<<parameters.use_external>>/shim')"
             OP_TESTLOG_DISABLE_COLOR=true OP_E2E_DISABLE_PARALLEL=false OP_E2E_USE_HTTP=<<parameters.use_http>>  gotestsum \
-            --format=standard-verbose --junitfile=/tmp/test-results/<<parameters.module>>_http_<<parameters.use_http>>.xml \
-            -- -timeout=20m -parallel=8 ./...
+            --format=standard-verbose --junitfile=/tmp/test-results/<<parameters.module>>_http_<<parameters.use_http>>$TEST_SUFFIX.xml \
+            -- -timeout=20m -parallel=8 --externalL2 "$EXTERNAL_L2" ./...
           working_directory: <<parameters.module>>
       - store_test_results:
           path: /tmp/test-results
@@ -1274,10 +1286,17 @@ workflows:
           name: op-e2e-WS-tests
           module: op-e2e
           use_http: "false"
+          use_external: ""
       - go-e2e-test:
           name: op-e2e-HTTP-tests
           module: op-e2e
           use_http: "true"
+          use_external: ""
+      - go-e2e-test:
+          name: op-e2e-WS-tests-external-geth
+          module: op-e2e
+          use_http: "false"
+          use_external: "external_geth"
       - bedrock-go-tests:
           requires:
             - op-batcher-lint

go.mod

@@ -32,6 +32,7 @@ require (
 	github.com/multiformats/go-multiaddr v0.10.1
 	github.com/multiformats/go-multiaddr-dns v0.3.1
 	github.com/olekukonko/tablewriter v0.0.5
+	github.com/onsi/gomega v1.27.4
 	github.com/pkg/errors v0.9.1
 	github.com/pkg/profile v1.7.0
 	github.com/prometheus/client_golang v1.14.0
@@ -66,23 +67,29 @@ require (
 	github.com/containerd/cgroups v1.1.0 // indirect
 	github.com/coreos/go-systemd/v22 v22.5.0 // indirect
 	github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect
+	github.com/crate-crypto/go-ipa v0.0.0-20220523130400-f11357ae11c7 // indirect
 	github.com/crate-crypto/go-kzg-4844 v0.2.0 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/davidlazar/go-crypto v0.0.0-20200604182044-b73af7476f6c // indirect
 	github.com/deckarep/golang-set/v2 v2.1.0 // indirect
 	github.com/decred/dcrd/crypto/blake256 v1.0.0 // indirect
 	github.com/deepmap/oapi-codegen v1.8.2 // indirect
+	github.com/dlclark/regexp2 v1.7.0 // indirect
 	github.com/docker/docker v20.10.24+incompatible // indirect
 	github.com/docker/go-units v0.5.0 // indirect
+	github.com/dop251/goja v0.0.0-20230122112309-96b1610dd4f7 // indirect
 	github.com/elastic/gosigar v0.14.2 // indirect
 	github.com/ethereum/c-kzg-4844 v0.2.0 // indirect
+	github.com/fatih/color v1.7.0 // indirect
 	github.com/felixge/fgprof v0.9.3 // indirect
 	github.com/fjl/memsize v0.0.1 // indirect
 	github.com/flynn/noise v1.0.0 // indirect
 	github.com/francoispqt/gojay v1.2.13 // indirect
 	github.com/gballet/go-libpcsclite v0.0.0-20191108122812-4678299bea08 // indirect
+	github.com/gballet/go-verkle v0.0.0-20220902153445-097bd83b7732 // indirect
 	github.com/getsentry/sentry-go v0.18.0 // indirect
 	github.com/go-ole/go-ole v1.2.6 // indirect
+	github.com/go-sourcemap/sourcemap v2.1.3+incompatible // indirect
 	github.com/go-stack/stack v1.8.1 // indirect
 	github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 // indirect
 	github.com/godbus/dbus/v5 v5.1.0 // indirect
@@ -110,8 +117,10 @@ require (
 	github.com/jackpal/go-nat-pmp v1.0.2 // indirect
 	github.com/jbenet/go-temp-err-catcher v0.1.0 // indirect
 	github.com/jbenet/goprocess v0.1.4 // indirect
+	github.com/jedisct1/go-minisign v0.0.0-20190909160543-45766022959e // indirect
 	github.com/jinzhu/inflection v1.0.0 // indirect
 	github.com/jinzhu/now v1.1.5 // indirect
+	github.com/karalabe/usb v0.0.2 // indirect
 	github.com/klauspost/compress v1.16.4 // indirect
 	github.com/klauspost/cpuid/v2 v2.2.4 // indirect
 	github.com/koron/go-ssdp v0.0.4 // indirect
@@ -147,6 +156,8 @@ require (
 	github.com/multiformats/go-multihash v0.2.1 // indirect
 	github.com/multiformats/go-multistream v0.4.1 // indirect
 	github.com/multiformats/go-varint v0.0.7 // indirect
+	github.com/naoina/go-stringutil v0.1.0 // indirect
+	github.com/naoina/toml v0.1.2-0.20170918210437-9fafd6967416 // indirect
 	github.com/onsi/ginkgo/v2 v2.9.2 // indirect
 	github.com/opencontainers/runtime-spec v1.0.2 // indirect
 	github.com/opentracing/opentracing-go v1.2.0 // indirect

indexer/e2e_tests/bridge_messages_e2e_test.go

@@ -155,7 +155,7 @@ func TestE2EBridgeL2CrossDomainMessenger(t *testing.T) {
 
 	// (2) Process RelayedMessage on withdrawal finalization
 	require.Nil(t, sentMessage.RelayedMessageEventGUID)
-	_, finalizedReceipt := op_e2e.ProveAndFinalizeWithdrawal(t, *testSuite.OpCfg, testSuite.L1Client, testSuite.OpSys.Nodes["sequencer"], testSuite.OpCfg.Secrets.Alice, sentMsgReceipt)
+	_, finalizedReceipt := op_e2e.ProveAndFinalizeWithdrawal(t, *testSuite.OpCfg, testSuite.L1Client, testSuite.OpSys.EthInstances["sequencer"], testSuite.OpCfg.Secrets.Alice, sentMsgReceipt)
 
 	// wait for processor catchup
 	require.NoError(t, wait.For(context.Background(), 500*time.Millisecond, func() (bool, error) {

indexer/e2e_tests/bridge_transactions_e2e_test.go

@@ -129,7 +129,7 @@ func TestE2EBridgeTransactionsL2ToL1MessagePasserWithdrawal(t *testing.T) {
 	require.Nil(t, withdraw.ProvenL1EventGUID)
 	require.Nil(t, withdraw.FinalizedL1EventGUID)
 
-	withdrawParams, proveReceipt := op_e2e.ProveWithdrawal(t, *testSuite.OpCfg, testSuite.L1Client, testSuite.OpSys.Nodes["sequencer"], testSuite.OpCfg.Secrets.Alice, withdrawReceipt)
+	withdrawParams, proveReceipt := op_e2e.ProveWithdrawal(t, *testSuite.OpCfg, testSuite.L1Client, testSuite.OpSys.EthInstances["sequencer"], testSuite.OpCfg.Secrets.Alice, withdrawReceipt)
 	require.NoError(t, wait.For(context.Background(), 500*time.Millisecond, func() (bool, error) {
 		l1Header := testSuite.Indexer.BridgeProcessor.LatestL1Header
 		return l1Header != nil && l1Header.Number.Uint64() >= proveReceipt.BlockNumber.Uint64(), nil
@@ -189,7 +189,7 @@ func TestE2EBridgeTransactionsL2ToL1MessagePasserFailedWithdrawal(t *testing.T)
 	require.NoError(t, err)
 
 	// Prove&Finalize withdrawal
-	_, finalizeReceipt := op_e2e.ProveAndFinalizeWithdrawal(t, *testSuite.OpCfg, testSuite.L1Client, testSuite.OpSys.Nodes["sequencer"], testSuite.OpCfg.Secrets.Alice, withdrawReceipt)
+	_, finalizeReceipt := op_e2e.ProveAndFinalizeWithdrawal(t, *testSuite.OpCfg, testSuite.L1Client, testSuite.OpSys.EthInstances["sequencer"], testSuite.OpCfg.Secrets.Alice, withdrawReceipt)
 	require.NoError(t, wait.For(context.Background(), 500*time.Millisecond, func() (bool, error) {
 		l1Header := testSuite.Indexer.BridgeProcessor.LatestL1Header
 		return l1Header != nil && l1Header.Number.Uint64() >= finalizeReceipt.BlockNumber.Uint64(), nil

indexer/e2e_tests/bridge_transfers_e2e_test.go

@@ -322,7 +322,7 @@ func TestE2EBridgeTransfersStandardBridgeETHWithdrawal(t *testing.T) {
 	require.Empty(t, aliceWithdrawals.Withdrawals[0].FinalizedL1TransactionHash)
 
 	// wait for processor catchup
-	proveReceipt, finalizeReceipt := op_e2e.ProveAndFinalizeWithdrawal(t, *testSuite.OpCfg, testSuite.L1Client, testSuite.OpSys.Nodes["sequencer"], testSuite.OpCfg.Secrets.Alice, withdrawReceipt)
+	proveReceipt, finalizeReceipt := op_e2e.ProveAndFinalizeWithdrawal(t, *testSuite.OpCfg, testSuite.L1Client, testSuite.OpSys.EthInstances["sequencer"], testSuite.OpCfg.Secrets.Alice, withdrawReceipt)
 	require.NoError(t, wait.For(context.Background(), 500*time.Millisecond, func() (bool, error) {
 		l1Header := testSuite.Indexer.BridgeProcessor.LatestL1Header
 		return l1Header != nil && l1Header.Number.Uint64() >= finalizeReceipt.BlockNumber.Uint64(), nil
@@ -400,7 +400,7 @@ func TestE2EBridgeTransfersL2ToL1MessagePasserReceive(t *testing.T) {
 	require.Empty(t, aliceWithdrawals.Withdrawals[0].FinalizedL1TransactionHash)
 
 	// wait for processor catchup
-	proveReceipt, finalizeReceipt := op_e2e.ProveAndFinalizeWithdrawal(t, *testSuite.OpCfg, testSuite.L1Client, testSuite.OpSys.Nodes["sequencer"], testSuite.OpCfg.Secrets.Alice, l2ToL1WithdrawReceipt)
+	proveReceipt, finalizeReceipt := op_e2e.ProveAndFinalizeWithdrawal(t, *testSuite.OpCfg, testSuite.L1Client, testSuite.OpSys.EthInstances["sequencer"], testSuite.OpCfg.Secrets.Alice, l2ToL1WithdrawReceipt)
 	require.NoError(t, wait.For(context.Background(), 500*time.Millisecond, func() (bool, error) {
 		l1Header := testSuite.Indexer.BridgeProcessor.LatestL1Header
 		return l1Header != nil && l1Header.Number.Uint64() >= finalizeReceipt.BlockNumber.Uint64(), nil

indexer/e2e_tests/etl_e2e_test.go

@@ -79,7 +79,7 @@ func TestE2EETL(t *testing.T) {
 				require.NotNil(t, latestOutput)
 				require.GreaterOrEqual(t, latestOutput.L2BlockNumber.Int.Uint64(), uint64(9))
 
-				l2EthClient, err := node.DialEthClient(testSuite.OpSys.Nodes["sequencer"].HTTPEndpoint())
+				l2EthClient, err := node.DialEthClient(testSuite.OpSys.EthInstances["sequencer"].HTTPEndpoint())
 				require.NoError(t, err)
 
 				submissionInterval := testSuite.OpCfg.DeployConfig.L2OutputOracleSubmissionInterval

indexer/e2e_tests/setup.go

@@ -49,7 +49,8 @@ func createE2ETestSuite(t *testing.T) E2ETestSuite {
 
 	// Rollup System Configuration and Start
 	opCfg := op_e2e.DefaultSystemConfig(t)
-	opSys, err := opCfg.Start()
+	opCfg.DeployConfig.FinalizationPeriodSeconds = 2
+	opSys, err := opCfg.Start(t)
 	require.NoError(t, err)
 
 	// E2E tests can run on the order of magnitude of minutes. Once
@@ -66,8 +67,8 @@ func createE2ETestSuite(t *testing.T) E2ETestSuite {
 			User: dbUser,
 		},
 		RPCs: config.RPCsConfig{
-			L1RPC: opSys.Nodes["l1"].HTTPEndpoint(),
-			L2RPC: opSys.Nodes["sequencer"].HTTPEndpoint(),
+			L1RPC: opSys.EthInstances["l1"].HTTPEndpoint(),
+			L2RPC: opSys.EthInstances["sequencer"].HTTPEndpoint(),
 		},
 		Chain: config.ChainConfig{
 			L1Contracts: config.L1Contracts{

op-e2e/.gitignore

+external_*/shim

op-e2e/Makefile

@@ -28,6 +28,11 @@ clean:
 lint:
 	golangci-lint run -E goimports,sqlclosecheck,bodyclose,asciicheck,misspell,errorlint --timeout 5m -e "errors.As" -e "errors.Is" ./...
 
+test-external-%: pre-test
+	make -C ./external_$*/
+	go test -v --externalL2 ./external_$*/shim
+
+
 .PHONY: \
 	test \
 	lint

op-e2e/bridge_test.go

@@ -24,7 +24,7 @@ func TestERC20BridgeDeposits(t *testing.T) {
 
 	cfg := DefaultSystemConfig(t)
 
-	sys, err := cfg.Start()
+	sys, err := cfg.Start(t)
 	require.Nil(t, err, "Error starting up system")
 	defer sys.Close()
 

op-e2e/config/init.go

@@ -26,6 +26,11 @@ var (
 	L1Deployments *genesis.L1Deployments
 	// DeployConfig represents the deploy config used by the system.
 	DeployConfig *genesis.DeployConfig
+	// ExternalL2Nodes is the shim to use if external ethereum client testing is
+	// enabled
+	ExternalL2Nodes string
+	// EthNodeVerbosity is the level of verbosity to output
+	EthNodeVerbosity int
 )
 
 // Init testing to enable test flags
@@ -53,6 +58,8 @@ func init() {
 	flag.StringVar(&l1AllocsPath, "l1-allocs", defaultL1AllocsPath, "")
 	flag.StringVar(&l1DeploymentsPath, "l1-deployments", defaultL1DeploymentsPath, "")
 	flag.StringVar(&deployConfigPath, "deploy-config", defaultDeployConfigPath, "")
+	flag.StringVar(&ExternalL2Nodes, "externalL2", "", "Enable tests with external L2")
+	flag.IntVar(&EthNodeVerbosity, "ethLogVerbosity", 3, "The level of verbosity to use for the eth node logs")
 	flag.Parse()
 
 	if err := allExist(l1AllocsPath, l1DeploymentsPath, deployConfigPath); err != nil {

op-e2e/external.go

+package op_e2e
+
+import (
+	"encoding/json"
+	"math/big"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"testing"
+	"time"
+
+	"github.com/ethereum-optimism/optimism/op-e2e/config"
+	"github.com/ethereum-optimism/optimism/op-e2e/external"
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/core"
+	"github.com/ethereum/go-ethereum/params"
+	"github.com/onsi/gomega/gexec"
+	"github.com/stretchr/testify/require"
+)
+
+type ExternalRunner struct {
+	Name    string
+	BinPath string
+	Genesis *core.Genesis
+	JWTPath string
+}
+
+type ExternalEthClient struct {
+	Session   *gexec.Session
+	Endpoints external.Endpoints
+}
+
+func (eec *ExternalEthClient) HTTPEndpoint() string {
+	return eec.Endpoints.HTTPEndpoint
+}
+
+func (eec *ExternalEthClient) WSEndpoint() string {
+	return eec.Endpoints.WSEndpoint
+}
+
+func (eec *ExternalEthClient) HTTPAuthEndpoint() string {
+	return eec.Endpoints.HTTPAuthEndpoint
+}
+
+func (eec *ExternalEthClient) WSAuthEndpoint() string {
+	return eec.Endpoints.WSAuthEndpoint
+}
+
+func (eec *ExternalEthClient) Close() {
+	eec.Session.Terminate()
+	select {
+	case <-time.After(5 * time.Second):
+		eec.Session.Kill()
+	case <-eec.Session.Exited:
+	}
+}
+
+func (er *ExternalRunner) Run(t *testing.T) *ExternalEthClient {
+	if er.BinPath == "" {
+		t.Error("no external bin path set")
+	}
+
+	if er.JWTPath == "" {
+		er.JWTPath = writeDefaultJWT(t)
+	}
+
+	if er.Genesis == nil {
+		er.Genesis = &core.Genesis{
+			Alloc: core.GenesisAlloc{
+				common.Address{1}: core.GenesisAccount{Balance: big.NewInt(1)},
+			},
+			Config:     params.OptimismTestConfig,
+			Difficulty: big.NewInt(0),
+		}
+	}
+
+	workDir := t.TempDir()
+
+	config := external.Config{
+		DataDir:            filepath.Join(workDir, "datadir"),
+		JWTPath:            er.JWTPath,
+		ChainID:            er.Genesis.Config.ChainID.Uint64(),
+		GenesisPath:        filepath.Join(workDir, "genesis.json"),
+		EndpointsReadyPath: filepath.Join(workDir, "endpoints.json"),
+		Verbosity:          uint64(config.EthNodeVerbosity),
+	}
+
+	err := os.Mkdir(config.DataDir, 0o700)
+	require.NoError(t, err)
+
+	genesisFile, err := os.Create(config.GenesisPath)
+	require.NoError(t, err)
+	err = json.NewEncoder(genesisFile).Encode(er.Genesis)
+	require.NoError(t, err)
+
+	configPath := filepath.Join(workDir, "config.json")
+	configFile, err := os.Create(configPath)
+	require.NoError(t, err)
+	err = json.NewEncoder(configFile).Encode(config)
+	require.NoError(t, err)
+
+	cmd := exec.Command(er.BinPath, "--config", configPath)
+	cmd.Dir = filepath.Dir(er.BinPath)
+	sess, err := gexec.Start(
+		cmd,
+		gexec.NewPrefixedWriter("[extout:"+er.Name+"]", os.Stdout),
+		gexec.NewPrefixedWriter("[exterr:"+er.Name+"]", os.Stderr),
+	)
+	require.NoError(t, err)
+
+	// 2 minutes may seem like a long timeout, and, it definitely is.  That
+	// being said, when running these tests with high parallelism turned on, the
+	// node startup time can be substantial (remember, this usually is a
+	// multi-step process initializing the database and then starting the
+	// client).
+	require.Eventually(
+		t,
+		func() bool {
+			_, err := os.Stat(config.EndpointsReadyPath)
+			return err == nil
+		},
+		2*time.Minute,
+		10*time.Millisecond,
+		"external runner did not create ready file at %s within timeout",
+		config.EndpointsReadyPath,
+	)
+
+	readyFile, err := os.Open(config.EndpointsReadyPath)
+	require.NoError(t, err)
+	var endpoints external.Endpoints
+	err = json.NewDecoder(readyFile).Decode(&endpoints)
+	require.NoError(t, err)
+
+	return &ExternalEthClient{
+		Session:   sess,
+		Endpoints: endpoints,
+	}
+}

op-e2e/external/config.go

+package external
+
+import (
+	"encoding/json"
+	"os"
+)
+
+type Config struct {
+	DataDir     string `json:"data_dir"`
+	JWTPath     string `json:"jwt_path"`
+	ChainID     uint64 `json:"chain_id"`
+	GasCeil     uint64 `json:"gas_ceil"`
+	GenesisPath string `json:"genesis_path"`
+	Verbosity   uint64 `json:"verbosity"`
+
+	// EndpointsReadyPath is the location to write the endpoint configuration file.
+	// Note, this should be written atomically by writing the JSON, then moving
+	// it to this path to avoid races.  A helper AtomicEncode is provided for
+	// golang clients.
+	EndpointsReadyPath string `json:"endpoints_ready_path"`
+}
+
+// AtomicEncode json encodes val to path+".atomic" then moves the path+".atomic"
+// file to path
+func AtomicEncode(path string, val any) error {
+	atomicPath := path + ".atomic"
+	atomicFile, err := os.Create(atomicPath)
+	if err != nil {
+		return err
+	}
+	if err = json.NewEncoder(atomicFile).Encode(val); err != nil {
+		return err
+	}
+	return os.Rename(atomicPath, path)
+}
+
+type Endpoints struct {
+	HTTPEndpoint     string `json:"http_endpoint"`
+	WSEndpoint       string `json:"ws_endpoint"`
+	HTTPAuthEndpoint string `json:"http_auth_endpoint"`
+	WSAuthEndpoint   string `json:"ws_auth_endpoint"`
+}

op-e2e/external_geth/.gitignore

+op-geth

op-e2e/external_geth/Makefile

+default: shim op-geth
+
+op-geth:
+	go build -o op-geth "github.com/ethereum/go-ethereum/cmd/geth"
+.PHONY: op-geth
+
+shim: main.go
+	go build -o shim .

op-e2e/external_geth/README.md

+# external_geth shim
+
+This shim is an example of how to write an adapter for an external ethereum
+client to allow for its use in the op-e2e tests.
+
+## Invocation
+
+Generally speaking, you can utilize this shim by simply executing:
+
+```
+make test-external-geth
+```
+
+The `Makefile` is structured such that if you duplicate this directory and
+tweak this code, you may simply execute:
+
+```
+make test-external-<your-client>
+```
+
+and the execution should happen as well.
+
+*NOTE:* Attempting to iterate for development requires explicit rebuilding of
+the binary being shimmed.  Most likely to accomplish this, you may want to add
+initialization code to the TestMain of the e2e to build your binary, or use
+some other technique like custom build scripts or IDE integrations which cause
+the binary to be rebuilt before executing the tests.
+
+## Arguments
+
+*--config <path>* The config path is a required argument, it points to a JSON
+file which contains details of the L2 environment to bring up (including the
+`genesis.json` path, the chain ID, the JWT path, and a ready file path).  See
+the data structures in `op-e2e/external/config.go` for more details.
+
+## Operation
+
+This shim will first execute a process to initialize the op-geth database.
+Then, it will start the op-geth process itself.  It watches the output of the
+process and looks for the lines indicating that the HTTP server and Auth HTTP
+server have started up.  It then reads the ports which were allocated (because
+the requested ports were passed in as ephemeral via the CLI arguments).
+
+## Generalization
+
+This shim is included to help document an demonstrate the usage of the
+external ethereum process e2e test execution.  It is configured to execute in
+CI to help ensure that the tests remain compatible with external clients.
+
+To create your own external test client, these files can likely be used as a
+starting point, changing the arguments, log scraping, and other details.  Or,
+depending on the client and your preference, any binary which is capable of
+reading and writing the necessary JSON files should be sufficient (though
+will be required to replicate some of the parsing and other logic encapsulated
+here).

op-e2e/external_geth/main.go

+package main
+
+import (
+	"encoding/json"
+	"flag"
+	"fmt"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strconv"
+	"time"
+
+	"github.com/ethereum-optimism/optimism/op-e2e/external"
+	"github.com/onsi/gomega/gbytes"
+	"github.com/onsi/gomega/gexec"
+)
+
+func main() {
+	var configPath string
+	flag.StringVar(&configPath, "config", "", "Execute based on the config in this file")
+	flag.Parse()
+	if err := run(configPath); err != nil {
+		fmt.Println(err.Error())
+		os.Exit(1)
+	}
+	os.Exit(0)
+}
+
+func run(configPath string) error {
+	if configPath == "" {
+		return fmt.Errorf("must supply a '--config <path>' flag")
+	}
+
+	configFile, err := os.Open(configPath)
+	if err != nil {
+		return fmt.Errorf("could not open config: %w", err)
+	}
+
+	var config external.Config
+	if err := json.NewDecoder(configFile).Decode(&config); err != nil {
+		return fmt.Errorf("could not decode config file: %w", err)
+	}
+
+	binPath, err := filepath.Abs("op-geth")
+	if err != nil {
+		return fmt.Errorf("could not get absolute path of op-geth")
+	}
+	if _, err := os.Stat(binPath); err != nil {
+		return fmt.Errorf("could not locate op-geth in working directory, did you forget to run '--init'?")
+	}
+
+	fmt.Printf("================== op-geth shim initializing chain config ==========================\n")
+	if err := initialize(binPath, config); err != nil {
+		return fmt.Errorf("could not initialize datadir: %s %w", binPath, err)
+	}
+
+	fmt.Printf("==================    op-geth shim executing op-geth     ==========================\n")
+	sess, err := execute(binPath, config)
+	if err != nil {
+		return fmt.Errorf("could not execute geth: %w", err)
+	}
+	defer sess.Close()
+
+	fmt.Printf("==================    op-geth shim encoding ready-file   ==========================\n")
+	if err := external.AtomicEncode(config.EndpointsReadyPath, sess.endpoints); err != nil {
+		return fmt.Errorf("could not encode endpoints")
+	}
+
+	fmt.Printf("==================    op-geth shim awaiting termination  ==========================\n")
+	select {
+	case <-sess.session.Exited:
+		return fmt.Errorf("geth exited")
+	case <-time.After(30 * time.Minute):
+		return fmt.Errorf("exiting after 30 minute timeout")
+	}
+}
+
+func initialize(binPath string, config external.Config) error {
+	cmd := exec.Command(
+		binPath,
+		"--datadir", config.DataDir,
+		"init", config.GenesisPath,
+	)
+	return cmd.Run()
+}
+
+type gethSession struct {
+	session   *gexec.Session
+	endpoints *external.Endpoints
+}
+
+func (es *gethSession) Close() {
+	es.session.Terminate()
+	select {
+	case <-time.After(5 * time.Second):
+		es.session.Kill()
+	case <-es.session.Exited:
+	}
+}
+
+func execute(binPath string, config external.Config) (*gethSession, error) {
+	if config.Verbosity < 2 {
+		return nil, fmt.Errorf("a minimum configured verbosity of 2 is required")
+	}
+	cmd := exec.Command(
+		binPath,
+		"--datadir", config.DataDir,
+		"--http",
+		"--http.addr", "127.0.0.1",
+		"--http.port", "0",
+		"--http.api", "web3,debug,eth,txpool,net,engine",
+		"--ws",
+		"--ws.addr", "127.0.0.1",
+		"--ws.port", "0",
+		"--ws.api", "debug,eth,txpool,net,engine",
+		"--syncmode=full",
+		"--nodiscover",
+		"--port", "0",
+		"--maxpeers", "0",
+		"--networkid", strconv.FormatUint(config.ChainID, 10),
+		"--authrpc.addr", "127.0.0.1",
+		"--authrpc.port", "0",
+		"--authrpc.jwtsecret", config.JWTPath,
+		"--gcmode=archive",
+		"--verbosity", strconv.FormatUint(config.Verbosity, 10),
+	)
+	sess, err := gexec.Start(cmd, os.Stdout, os.Stderr)
+	if err != nil {
+		return nil, fmt.Errorf("could not start op-geth session: %w", err)
+	}
+	matcher := gbytes.Say("HTTP server started\\s*endpoint=127.0.0.1:")
+	var enginePort, httpPort int
+	for enginePort == 0 || httpPort == 0 {
+		match, err := matcher.Match(sess.Err)
+		if err != nil {
+			return nil, fmt.Errorf("could not execute matcher")
+		}
+		if !match {
+			if sess.Err.Closed() {
+				return nil, fmt.Errorf("op-geth exited before announcing http ports")
+			}
+			// Wait for a bit more output, then try again
+			time.Sleep(10 * time.Millisecond)
+			continue
+		}
+		var authString string
+		var port int
+		fmt.Fscanf(sess.Err, "%d %s", &port, &authString)
+		switch authString {
+		case "auth=true":
+			enginePort = port
+		case "auth=false":
+			httpPort = port
+		default:
+			return nil, fmt.Errorf("unexpected auth string %q", authString)
+		}
+	}
+
+	return &gethSession{
+		session: sess,
+		endpoints: &external.Endpoints{
+			HTTPEndpoint:     fmt.Sprintf("http://127.0.0.1:%d/", httpPort),
+			WSEndpoint:       fmt.Sprintf("ws://127.0.0.1:%d/", httpPort),
+			HTTPAuthEndpoint: fmt.Sprintf("http://127.0.0.1:%d/", enginePort),
+			WSAuthEndpoint:   fmt.Sprintf("ws://127.0.0.1:%d/", enginePort),
+		},
+	}, nil
+}

op-e2e/external_geth/main_test.go

+package main
+
+import (
+	"net"
+	"net/url"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"testing"
+	"time"
+
+	e2e "github.com/ethereum-optimism/optimism/op-e2e"
+	"github.com/ethereum-optimism/optimism/op-e2e/config"
+	"github.com/stretchr/testify/require"
+)
+
+func TestShim(t *testing.T) {
+	shimPath, err := filepath.Abs("shim")
+	require.NoError(t, err)
+	cmd := exec.Command("go", "build", "-o", shimPath, ".")
+	cmd.Stdout = os.Stdout
+	cmd.Stderr = os.Stderr
+	err = cmd.Run()
+	require.NoError(t, err)
+	require.FileExists(t, "shim")
+
+	opGethPath, err := filepath.Abs("op-geth")
+	require.NoError(t, err)
+	cmd = exec.Command("go", "build", "-o", opGethPath, "github.com/ethereum/go-ethereum/cmd/geth")
+	cmd.Stdout = os.Stdout
+	cmd.Stderr = os.Stderr
+	err = cmd.Run()
+	require.NoError(t, err)
+	require.FileExists(t, "op-geth")
+
+	config.EthNodeVerbosity = 4
+
+	ec := (&e2e.ExternalRunner{
+		Name:    "TestShim",
+		BinPath: shimPath,
+	}).Run(t)
+	t.Cleanup(ec.Close)
+
+	for _, endpoint := range []string{
+		ec.HTTPEndpoint(),
+		ec.HTTPAuthEndpoint(),
+		ec.WSEndpoint(),
+		ec.WSAuthEndpoint(),
+	} {
+		plainURL, err := url.ParseRequestURI(endpoint)
+		require.NoError(t, err)
+		_, err = net.DialTimeout("tcp", plainURL.Host, time.Second)
+		require.NoError(t, err, "could not connect to HTTP port")
+	}
+}

op-e2e/external_geth/tools.go

+//go:build tools
+
+package main
+
+import _ "github.com/ethereum/go-ethereum/cmd/geth"

op-e2e/faultproof_test.go

@@ -409,7 +409,7 @@ func startFaultDisputeSystem(t *testing.T) (*System, *ethclient.Client) {
 	cfg.SupportL1TimeTravel = true
 	cfg.DeployConfig.L2OutputOracleSubmissionInterval = 1
 	cfg.NonFinalizedProposals = true // Submit output proposals asap
-	sys, err := cfg.Start()
-	require.NoError(t, err, "Error starting up system")
+	sys, err := cfg.Start(t)
+	require.Nil(t, err, "Error starting up system")
 	return sys, sys.Clients["l1"]
 }

op-e2e/helper.go

 package op_e2e
 
 import (
-	"flag"
 	"os"
 	"testing"
 
+	"github.com/ethereum-optimism/optimism/op-e2e/config"
 	"github.com/ethereum/go-ethereum/log"
 )
 
-var enableParallelTesting bool = true
-
-// Init testing to enable test flags
-var _ = func() bool {
-	testing.Init()
-	return true
-}()
-
-var verboseGethNodes bool
-
-func init() {
-	flag.BoolVar(&verboseGethNodes, "gethlogs", true, "Enable logs on geth nodes")
-	flag.Parse()
-	if os.Getenv("OP_E2E_DISABLE_PARALLEL") == "true" {
-		enableParallelTesting = false
-	}
-}
+var enableParallelTesting bool = os.Getenv("OP_E2E_DISABLE_PARALLEL") != "true"
 
 func InitParallel(t *testing.T) {
 	t.Helper()
 	if enableParallelTesting {
 		t.Parallel()
 	}
-	if !verboseGethNodes {
+	if config.EthNodeVerbosity < 0 {
 		log.Root().SetHandler(log.DiscardHandler())
 	}
 }

op-e2e/op_geth_test.go

@@ -48,7 +48,7 @@ func TestMissingGasLimit(t *testing.T) {
 func TestTxGasSameAsBlockGasLimit(t *testing.T) {
 	InitParallel(t)
 	cfg := DefaultSystemConfig(t)
-	sys, err := cfg.Start()
+	sys, err := cfg.Start(t)
 	require.Nil(t, err, "Error starting up system")
 	defer sys.Close()
 

op-e2e/system_adminrpc_test.go

@@ -17,7 +17,7 @@ func TestStopStartSequencer(t *testing.T) {
 	InitParallel(t)
 
 	cfg := DefaultSystemConfig(t)
-	sys, err := cfg.Start()
+	sys, err := cfg.Start(t)
 	require.Nil(t, err, "Error starting up system")
 	defer sys.Close()
 
@@ -83,7 +83,7 @@ func TestPersistSequencerStateWhenChanged(t *testing.T) {
 	delete(cfg.Nodes, "verifier")
 	cfg.Nodes["sequencer"].ConfigPersistence = node.NewConfigPersistence(stateFile)
 
-	sys, err := cfg.Start()
+	sys, err := cfg.Start(t)
 	require.NoError(t, err)
 	defer sys.Close()
 
@@ -118,7 +118,7 @@ func TestLoadSequencerStateOnStarted_Stopped(t *testing.T) {
 	seqCfg := cfg.Nodes["sequencer"]
 	seqCfg.ConfigPersistence = node.NewConfigPersistence(stateFile)
 
-	sys, err := cfg.Start()
+	sys, err := cfg.Start(t)
 	require.NoError(t, err)
 	defer sys.Close()
 
@@ -152,7 +152,7 @@ func TestLoadSequencerStateOnStarted_Started(t *testing.T) {
 	seqCfg.Driver.SequencerStopped = true
 	seqCfg.ConfigPersistence = node.NewConfigPersistence(stateFile)
 
-	sys, err := cfg.Start()
+	sys, err := cfg.Start(t)
 	require.NoError(t, err)
 	defer sys.Close()
 

op-e2e/system_fpp_test.go

@@ -56,7 +56,7 @@ func testVerifyL2OutputRootEmptyBlock(t *testing.T, detached bool) {
 	// But not too small to ensure that our claim and subsequent state change is published
 	cfg.DeployConfig.SequencerWindowSize = 16
 
-	sys, err := cfg.Start()
+	sys, err := cfg.Start(t)
 	require.Nil(t, err, "Error starting up system")
 	defer sys.Close()
 
@@ -154,7 +154,7 @@ func testVerifyL2OutputRoot(t *testing.T, detached bool) {
 	// We don't need a verifier - just the sequencer is enough
 	delete(cfg.Nodes, "verifier")
 
-	sys, err := cfg.Start()
+	sys, err := cfg.Start(t)
 	require.Nil(t, err, "Error starting up system")
 	defer sys.Close()
 
@@ -260,8 +260,8 @@ func testFaultProofProgramScenario(t *testing.T, ctx context.Context, sys *Syste
 	sys.BatchSubmitter.StopIfRunning(context.Background())
 	sys.L2OutputSubmitter.Stop()
 	sys.L2OutputSubmitter = nil
-	for _, node := range sys.Nodes {
-		require.NoError(t, node.Close())
+	for _, node := range sys.EthInstances {
+		node.Close()
 	}
 
 	t.Log("Running fault proof in offline mode")

op-e2e/system_tob_test.go

@@ -41,7 +41,7 @@ func TestGasPriceOracleFeeUpdates(t *testing.T) {
 
 	// Create our system configuration for L1/L2 and start it
 	cfg := DefaultSystemConfig(t)
-	sys, err := cfg.Start()
+	sys, err := cfg.Start(t)
 	require.Nil(t, err, "Error starting up system")
 	defer sys.Close()
 
@@ -124,7 +124,7 @@ func TestL2SequencerRPCDepositTx(t *testing.T) {
 
 	// Create our system configuration for L1/L2 and start it
 	cfg := DefaultSystemConfig(t)
-	sys, err := cfg.Start()
+	sys, err := cfg.Start(t)
 	require.Nil(t, err, "Error starting up system")
 	defer sys.Close()
 
@@ -169,7 +169,7 @@ type TestAccount struct {
 // startConfigWithTestAccounts takes a SystemConfig, generates additional accounts, adds them to the config, so they
 // are funded on startup, starts the system, and imports the keys into the keystore, and obtains transaction opts for
 // each account.
-func startConfigWithTestAccounts(cfg *SystemConfig, accountsToGenerate int) (*System, []*TestAccount, error) {
+func startConfigWithTestAccounts(t *testing.T, cfg *SystemConfig, accountsToGenerate int) (*System, []*TestAccount, error) {
 	// Create our test accounts and add them to the pre-mine cfg.
 	testAccounts := make([]*TestAccount, 0)
 	var err error
@@ -211,7 +211,7 @@ func startConfigWithTestAccounts(cfg *SystemConfig, accountsToGenerate int) (*Sy
 	}
 
 	// Start our system
-	sys, err := cfg.Start()
+	sys, err := cfg.Start(t)
 	if err != nil {
 		return sys, nil, err
 	}
@@ -233,7 +233,7 @@ func TestMixedDepositValidity(t *testing.T) {
 
 	// Create our system configuration, funding all accounts we created for L1/L2, and start it
 	cfg := DefaultSystemConfig(t)
-	sys, testAccounts, err := startConfigWithTestAccounts(&cfg, accountUsedToDeposit)
+	sys, testAccounts, err := startConfigWithTestAccounts(t, &cfg, accountUsedToDeposit)
 	require.Nil(t, err, "Error starting up system")
 	defer sys.Close()
 
@@ -400,7 +400,7 @@ func TestMixedWithdrawalValidity(t *testing.T) {
 			cfg.DeployConfig.L2BlockTime = 2
 			require.LessOrEqual(t, cfg.DeployConfig.FinalizationPeriodSeconds, uint64(6))
 			require.Equal(t, cfg.DeployConfig.FundDevAccounts, true)
-			sys, err := cfg.Start()
+			sys, err := cfg.Start(t)
 			require.NoError(t, err, "error starting up system")
 			defer sys.Close()
 
@@ -544,7 +544,7 @@ func TestMixedWithdrawalValidity(t *testing.T) {
 			cancel()
 			require.Nil(t, err)
 
-			rpcClient, err := rpc.Dial(sys.Nodes["verifier"].WSEndpoint())
+			rpcClient, err := rpc.Dial(sys.EthInstances["verifier"].WSEndpoint())
 			require.Nil(t, err)
 			proofCl := gethclient.New(rpcClient)
 			receiptCl := ethclient.NewClient(rpcClient)
@@ -715,7 +715,7 @@ func TestMixedWithdrawalValidity(t *testing.T) {
 			// TODO: Check L1 balance as well here. We avoided this due to time constraints as it seems L1 fees
 			//  were off slightly.
 			_ = endL1Balance
-			//require.Equal(t, transactor.ExpectedL1Balance, endL1Balance, "Unexpected L1 balance for transactor")
+			// require.Equal(t, transactor.ExpectedL1Balance, endL1Balance, "Unexpected L1 balance for transactor")
 			require.Equal(t, transactor.ExpectedL1Nonce, endL1Nonce, "Unexpected L1 nonce for transactor")
 			require.Equal(t, transactor.ExpectedL2Nonce, endL2SeqNonce, "Unexpected L2 sequencer nonce for transactor")
 			require.Equal(t, transactor.ExpectedL2Balance, endL2SeqBalance, "Unexpected L2 sequencer balance for transactor")

op-e2e/withdrawal_helper.go

@@ -17,7 +17,6 @@ import (
 	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/ethereum/go-ethereum/ethclient"
 	"github.com/ethereum/go-ethereum/ethclient/gethclient"
-	"github.com/ethereum/go-ethereum/node"
 	"github.com/ethereum/go-ethereum/rpc"
 	"github.com/stretchr/testify/require"
 )
@@ -79,13 +78,13 @@ func defaultWithdrawalTxOpts() *WithdrawalTxOpts {
 	}
 }
 
-func ProveAndFinalizeWithdrawal(t *testing.T, cfg SystemConfig, l1Client *ethclient.Client, l2Node *node.Node, ethPrivKey *ecdsa.PrivateKey, l2WithdrawalReceipt *types.Receipt) (*types.Receipt, *types.Receipt) {
+func ProveAndFinalizeWithdrawal(t *testing.T, cfg SystemConfig, l1Client *ethclient.Client, l2Node EthInstance, ethPrivKey *ecdsa.PrivateKey, l2WithdrawalReceipt *types.Receipt) (*types.Receipt, *types.Receipt) {
 	params, proveReceipt := ProveWithdrawal(t, cfg, l1Client, l2Node, ethPrivKey, l2WithdrawalReceipt)
 	finalizeReceipt := FinalizeWithdrawal(t, cfg, l1Client, ethPrivKey, proveReceipt, params)
 	return proveReceipt, finalizeReceipt
 }
 
-func ProveWithdrawal(t *testing.T, cfg SystemConfig, l1Client *ethclient.Client, l2Node *node.Node, ethPrivKey *ecdsa.PrivateKey, l2WithdrawalReceipt *types.Receipt) (withdrawals.ProvenWithdrawalParameters, *types.Receipt) {
+func ProveWithdrawal(t *testing.T, cfg SystemConfig, l1Client *ethclient.Client, l2Node EthInstance, ethPrivKey *ecdsa.PrivateKey, l2WithdrawalReceipt *types.Receipt) (withdrawals.ProvenWithdrawalParameters, *types.Receipt) {
 	// Get l2BlockNumber for proof generation
 	ctx, cancel := context.WithTimeout(context.Background(), 40*time.Duration(cfg.DeployConfig.L1BlockTime)*time.Second)
 	defer cancel()

packages/contracts-bedrock/scripts/differential-testing/differential-testing.go

@@ -277,7 +277,8 @@ func main() {
 		checkErr(err, "Error creating secure trie")
 
 		// Put a "true" bool in the storage slot
-		state.UpdateStorage(common.Address{}, hash.Bytes(), []byte{0x01})
+		err = state.UpdateStorage(common.Address{}, hash.Bytes(), []byte{0x01})
+		checkErr(err, "Error updating storage")
 
 		// Create a secure trie for the world state
 		world, err := trie.NewStateTrie(
@@ -294,7 +295,8 @@ func main() {
 		}
 		writer := new(bytes.Buffer)
 		checkErr(account.EncodeRLP(writer), "Error encoding account")
-		world.UpdateStorage(common.Address{}, predeploys.L2ToL1MessagePasserAddr.Bytes(), writer.Bytes())
+		err = world.UpdateStorage(common.Address{}, predeploys.L2ToL1MessagePasserAddr.Bytes(), writer.Bytes())
+		checkErr(err, "Error updating storage")
 
 		// Get the proof
 		var proof proofList