op-e2e: add test option to utilize external eth clients for L2 (#6280)

* op-e2e: add test option to use external eth client

This change is an attempt to allow other Optimism clients like op-erigon
to utilize the op-e2e tests in a minimally invasive way.  Almost all of
the tests are unchanged, with some small structural changes to the setup
to allow for utilization of an in process Geth, or extra-process generic
Ethereum client.

This change establishes a convention for external clients (or shims) to
follow for interoperability with these tests.  The external process shim
must accept a '--config <path>' flag.  This flag specifies a path to a
JSON file which contains necessary execution parameters. These
parameters include the data dir to use for the client, the JWT path, the
genesis JSON path, and the chain ID.

Utilizing the external binary is is generally more resource intensive
than the in process Geth path, but, utilizing parallelism the test suite
there is no noticable difference in execution time between the in
process and an extra-process op-geth.

This commit simply performs the wiring, while the next commit will add
an op-geth consumer and wire into CI.

* op-e2e: add basic external geth runner

This commit adds an example external shim which utilizes the code in the
previous commit.  It also integrates it into the Makefile and into CI
(which unfortunately does not use the Makefile).

To utilize this geth runner locally you can execute:

  make test-external-geth

from the op-e2e directory.  See the README.md included in external_geth
for more details.

---------
Co-authored-by: Jason Yellick <jason@enya.ai>

.circleci/config.yml

@@ -778,6 +778,9 @@ jobs:
       use_http:
         description: If the op-e2e package should use HTTP clients
         type: string
+      use_external:
+        description: The extra-process shim (if any) that should be used
+        type: string
     docker:
       - image: us-docker.pkg.dev/oplabs-tools-artifacts/images/ci-builder:latest
     resource_class: xlarge
@@ -788,6 +791,13 @@ jobs:
       - run:
           name: prep results dir
           command: mkdir -p /tmp/test-results
+      - when:
+          condition: <<parameters.use_external>>
+          steps:
+            - run:
+                name: Build Shim
+                command: make -C <<parameters.use_external>>
+                working_directory: <<parameters.module>>
       - run:
           name: install geth
           command: make install-geth
@@ -807,9 +817,11 @@ jobs:
             # Note: We don't use circle CI test splits because we need to split by test name, not by package. There is an additional
             # constraint that gotestsum does not currently (nor likely will) accept files from different pacakges when building.
             # Note: -parallel must be set to match the number of cores in the resource class
+            export TEST_SUFFIX="<<parameters.use_external>>"
+            export EXTERNAL_L2="$(test -z '<<parameters.use_external>>' || echo '<<parameters.use_external>>/shim')"
             OP_TESTLOG_DISABLE_COLOR=true OP_E2E_DISABLE_PARALLEL=false OP_E2E_USE_HTTP=<<parameters.use_http>>  gotestsum \
-            --format=standard-verbose --junitfile=/tmp/test-results/<<parameters.module>>_http_<<parameters.use_http>>.xml \
-            -- -timeout=20m -parallel=8 ./...
+            --format=standard-verbose --junitfile=/tmp/test-results/<<parameters.module>>_http_<<parameters.use_http>>$TEST_SUFFIX.xml \
+            -- -timeout=20m -parallel=8 --externalL2 "$EXTERNAL_L2" ./...
           working_directory: <<parameters.module>>
       - store_test_results:
           path: /tmp/test-results
@@ -1274,10 +1286,17 @@ workflows:
           name: op-e2e-WS-tests
           module: op-e2e
           use_http: "false"
+          use_external: ""
       - go-e2e-test:
           name: op-e2e-HTTP-tests
           module: op-e2e
           use_http: "true"
+          use_external: ""
+      - go-e2e-test:
+          name: op-e2e-WS-tests-external-geth
+          module: op-e2e
+          use_http: "false"
+          use_external: "external_geth"
       - bedrock-go-tests:
           requires:
             - op-batcher-lint

go.mod

@@ -32,6 +32,7 @@ require (
 	github.com/multiformats/go-multiaddr v0.10.1
 	github.com/multiformats/go-multiaddr-dns v0.3.1
 	github.com/olekukonko/tablewriter v0.0.5
+	github.com/onsi/gomega v1.27.4
 	github.com/pkg/errors v0.9.1
 	github.com/pkg/profile v1.7.0
 	github.com/prometheus/client_golang v1.14.0
@@ -66,23 +67,29 @@ require (
 	github.com/containerd/cgroups v1.1.0 // indirect
 	github.com/coreos/go-systemd/v22 v22.5.0 // indirect
 	github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect
+	github.com/crate-crypto/go-ipa v0.0.0-20220523130400-f11357ae11c7 // indirect
 	github.com/crate-crypto/go-kzg-4844 v0.2.0 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/davidlazar/go-crypto v0.0.0-20200604182044-b73af7476f6c // indirect
 	github.com/deckarep/golang-set/v2 v2.1.0 // indirect
 	github.com/decred/dcrd/crypto/blake256 v1.0.0 // indirect
 	github.com/deepmap/oapi-codegen v1.8.2 // indirect
+	github.com/dlclark/regexp2 v1.7.0 // indirect
 	github.com/docker/docker v20.10.24+incompatible // indirect
 	github.com/docker/go-units v0.5.0 // indirect
+	github.com/dop251/goja v0.0.0-20230122112309-96b1610dd4f7 // indirect
 	github.com/elastic/gosigar v0.14.2 // indirect
 	github.com/ethereum/c-kzg-4844 v0.2.0 // indirect
+	github.com/fatih/color v1.7.0 // indirect
 	github.com/felixge/fgprof v0.9.3 // indirect
 	github.com/fjl/memsize v0.0.1 // indirect
 	github.com/flynn/noise v1.0.0 // indirect
 	github.com/francoispqt/gojay v1.2.13 // indirect
 	github.com/gballet/go-libpcsclite v0.0.0-20191108122812-4678299bea08 // indirect
+	github.com/gballet/go-verkle v0.0.0-20220902153445-097bd83b7732 // indirect
 	github.com/getsentry/sentry-go v0.18.0 // indirect
 	github.com/go-ole/go-ole v1.2.6 // indirect
+	github.com/go-sourcemap/sourcemap v2.1.3+incompatible // indirect
 	github.com/go-stack/stack v1.8.1 // indirect
 	github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 // indirect
 	github.com/godbus/dbus/v5 v5.1.0 // indirect
@@ -110,8 +117,10 @@ require (
 	github.com/jackpal/go-nat-pmp v1.0.2 // indirect
 	github.com/jbenet/go-temp-err-catcher v0.1.0 // indirect
 	github.com/jbenet/goprocess v0.1.4 // indirect
+	github.com/jedisct1/go-minisign v0.0.0-20190909160543-45766022959e // indirect
 	github.com/jinzhu/inflection v1.0.0 // indirect
 	github.com/jinzhu/now v1.1.5 // indirect
+	github.com/karalabe/usb v0.0.2 // indirect
 	github.com/klauspost/compress v1.16.4 // indirect
 	github.com/klauspost/cpuid/v2 v2.2.4 // indirect
 	github.com/koron/go-ssdp v0.0.4 // indirect
@@ -147,6 +156,8 @@ require (
 	github.com/multiformats/go-multihash v0.2.1 // indirect
 	github.com/multiformats/go-multistream v0.4.1 // indirect
 	github.com/multiformats/go-varint v0.0.7 // indirect
+	github.com/naoina/go-stringutil v0.1.0 // indirect
+	github.com/naoina/toml v0.1.2-0.20170918210437-9fafd6967416 // indirect
 	github.com/onsi/ginkgo/v2 v2.9.2 // indirect
 	github.com/opencontainers/runtime-spec v1.0.2 // indirect
 	github.com/opentracing/opentracing-go v1.2.0 // indirect

go.sum

@@ -107,6 +107,8 @@ github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:ma
 github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
 github.com/cpuguy83/go-md2man/v2 v2.0.2 h1:p1EgwI/C7NhT0JmVkwCD2ZBK8j4aeHQX2pMHHBfMQ6w=
 github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
+github.com/crate-crypto/go-ipa v0.0.0-20220523130400-f11357ae11c7 h1:6IrxszG5G+O7zhtkWxq6+unVvnrm1fqV2Pe+T95DUzw=
+github.com/crate-crypto/go-ipa v0.0.0-20220523130400-f11357ae11c7/go.mod h1:gFnFS95y8HstDP6P9pPwzrxOOC5TRDkwbM+ao15ChAI=
 github.com/crate-crypto/go-kzg-4844 v0.2.0 h1:UVuHOE+5tIWrim4zf/Xaa43+MIsDCPyW76QhUpiMGj4=
 github.com/crate-crypto/go-kzg-4844 v0.2.0/go.mod h1:SBP7ikXEgDnUPONgm33HtuDZEDtWa3L4QtN1ocJSEQ4=
 github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY=
@@ -134,13 +136,19 @@ github.com/dgraph-io/badger v1.6.2 h1:mNw0qs90GVgGGWylh0umH5iag1j6n/PeJtNvL6KY/x
 github.com/dgraph-io/ristretto v0.0.2 h1:a5WaUrDa0qm0YrAAS1tUykT5El3kt62KNZZeMxQn3po=
 github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
 github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw=
+github.com/dlclark/regexp2 v1.4.1-0.20201116162257-a2a8dda75c91/go.mod h1:2pZnwuY/m+8K6iRw6wQdMtk+rH5tNGR1i55kozfMjCc=
 github.com/dlclark/regexp2 v1.7.0 h1:7lJfhqlPssTb1WQx4yvTHN0uElPEv52sbaECrAQxjAo=
+github.com/dlclark/regexp2 v1.7.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8=
 github.com/docker/docker v20.10.24+incompatible h1:Ugvxm7a8+Gz6vqQYQQ2W7GYq5EUPaAiuPgIfVyI3dYE=
 github.com/docker/docker v20.10.24+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
 github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
 github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4=
 github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
+github.com/dop251/goja v0.0.0-20211022113120-dc8c55024d06/go.mod h1:R9ET47fwRVRPZnOGvHxxhuZcbrMCuiqOz3Rlrh4KSnk=
 github.com/dop251/goja v0.0.0-20230122112309-96b1610dd4f7 h1:kgvzE5wLsLa7XKfV85VZl40QXaMCaeFtHpPwJ8fhotY=
+github.com/dop251/goja v0.0.0-20230122112309-96b1610dd4f7/go.mod h1:yRkwfj0CBpOGre+TwBsqPV0IH0Pk73e4PXJOeNDboGs=
+github.com/dop251/goja_nodejs v0.0.0-20210225215109-d91c329300e7/go.mod h1:hn7BA7c8pLvoGndExHudxTDKZ84Pyvv+90pbBjbTz0Y=
+github.com/dop251/goja_nodejs v0.0.0-20211022123610-8dd9abb0616d/go.mod h1:DngW8aVqWbuLRMHItjPUyqdj+HWPvnQe8V8y1nDpIbM=
 github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo=
 github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
 github.com/eknkc/amber v0.0.0-20171010120322-cdade1c07385/go.mod h1:0vRUJqYpeSZifjYj7uP3BG/gKcuzL9xWVV/Y+cK33KM=
@@ -159,6 +167,8 @@ github.com/ethereum-optimism/op-geth v1.101106.1-0.20230724181546-b9c6d36ae9b8/g
 github.com/ethereum/c-kzg-4844 v0.2.0 h1:+cUvymlnoDDQgMInp25Bo3OmLajmmY8mLJ/tLjqd77Q=
 github.com/ethereum/c-kzg-4844 v0.2.0/go.mod h1:WI2Nd82DMZAAZI1wV2neKGost9EKjvbpQR9OqE5Qqa8=
 github.com/fasthttp-contrib/websocket v0.0.0-20160511215533-1f3b11f56072/go.mod h1:duJ4Jxv5lDcvg4QuQr0oowTf7dz4/CR8NtyCooz9HL8=
+github.com/fatih/color v1.7.0 h1:DkWD4oS2D8LGGgTQ6IvwJJXSL5Vp2ffcQg58nFV38Ys=
+github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
 github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M=
 github.com/felixge/fgprof v0.9.3 h1:VvyZxILNuCiUCSXtPtYmmtGvb65nqXh2QFWc0Wpf2/g=
 github.com/felixge/fgprof v0.9.3/go.mod h1:RdbpDgzqYVh/T9fPELJyV7EYJuHB55UTEULNun8eiPw=
@@ -177,6 +187,8 @@ github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbS
 github.com/gavv/httpexpect v2.0.0+incompatible/go.mod h1:x+9tiU1YnrOvnB725RkpoLv1M62hOWzwo5OXotisrKc=
 github.com/gballet/go-libpcsclite v0.0.0-20191108122812-4678299bea08 h1:f6D9Hr8xV8uYKlyuj8XIruxlh9WjVjdh1gIicAS7ays=
 github.com/gballet/go-libpcsclite v0.0.0-20191108122812-4678299bea08/go.mod h1:x7DCsMOv1taUwEWCzT4cmDeAkigA5/QCwUodaVOe8Ww=
+github.com/gballet/go-verkle v0.0.0-20220902153445-097bd83b7732 h1:AB7YjNrzlVHsYz06zCULVV2zYCEft82P86dSmtwxKL0=
+github.com/gballet/go-verkle v0.0.0-20220902153445-097bd83b7732/go.mod h1:o/XfIXWi4/GqbQirfRm5uTbXMG5NpqxkxblnbZ+QM9I=
 github.com/getkin/kin-openapi v0.53.0/go.mod h1:7Yn5whZr5kJi6t+kShccXS8ae1APpYTW6yheSwk8Yi4=
 github.com/getkin/kin-openapi v0.61.0/go.mod h1:7Yn5whZr5kJi6t+kShccXS8ae1APpYTW6yheSwk8Yi4=
 github.com/getsentry/sentry-go v0.12.0/go.mod h1:NSap0JBYWzHND8oMbyi0+XZhUalc1TBdRL1M71JZW2c=
@@ -216,6 +228,7 @@ github.com/go-playground/universal-translator v0.18.0 h1:82dyy6p4OuJq4/CByFNOn/j
 github.com/go-playground/validator/v10 v10.2.0/go.mod h1:uOYAAleCW8F/7oMFd6aG0GOhaH6EGOAJShg8Id5JGkI=
 github.com/go-playground/validator/v10 v10.11.1 h1:prmOlTVv+YjZjmRmNSF3VmspqJIxJWXmqUsHwfTRRkQ=
 github.com/go-sourcemap/sourcemap v2.1.3+incompatible h1:W1iEw64niKVGogNgBN3ePyLFfuisuzeidWPMPWmECqU=
+github.com/go-sourcemap/sourcemap v2.1.3+incompatible/go.mod h1:F8jJfvm2KbVjc5NqelyYJmf/v5J0dwNLS2mL4sNA1Jg=
 github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
 github.com/go-stack/stack v1.8.1 h1:ntEHSVwIt7PNXNpgPmVfMrNhLtgjlmnZha2kOpuRiDw=
 github.com/go-stack/stack v1.8.1/go.mod h1:dcoOX6HbPZSZptuspn9bctJ+N/CnF5gGygcUP3XYfe4=
@@ -422,6 +435,8 @@ github.com/jbenet/go-temp-err-catcher v0.1.0 h1:zpb3ZH6wIE8Shj2sKS+khgRvf7T7RABo
 github.com/jbenet/go-temp-err-catcher v0.1.0/go.mod h1:0kJRvmDZXNMIiJirNPEYfhpPwbGVtZVWC34vc5WLsDk=
 github.com/jbenet/goprocess v0.1.4 h1:DRGOFReOMqqDNXwW70QkacFW0YN9QnwLV0Vqk+3oU0o=
 github.com/jbenet/goprocess v0.1.4/go.mod h1:5yspPrukOVuOLORacaBi858NqyClJPQxYZlqdZVfqY4=
+github.com/jedisct1/go-minisign v0.0.0-20190909160543-45766022959e h1:UvSe12bq+Uj2hWd8aOlwPmoZ+CITRFrdit+sDGfAg8U=
+github.com/jedisct1/go-minisign v0.0.0-20190909160543-45766022959e/go.mod h1:G1CVv03EnqU1wYL2dFwXxW2An0az9JTl/ZsqXQeBlkU=
 github.com/jellevandenhooff/dkim v0.0.0-20150330215556-f50fe3d243e1/go.mod h1:E0B/fFc00Y+Rasa88328GlI/XbtyysCtTHZS8h7IrBU=
 github.com/jessevdk/go-flags v0.0.0-20141203071132-1679536dcc89/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI=
 github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI=
@@ -437,6 +452,8 @@ github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1
 github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
 github.com/julienschmidt/httprouter v1.3.0 h1:U0609e9tgbseu3rBINet9P48AI/D3oJs4dN7jwJOQ1U=
 github.com/k0kubun/colorstring v0.0.0-20150214042306-9440f1994b88/go.mod h1:3w7q1U84EfirKl04SVQ/s7nPm1ZPhiXd34z40TNz36k=
+github.com/karalabe/usb v0.0.2 h1:M6QQBNxF+CQ8OFvxrT90BA0qBOXymndZnk5q235mFc4=
+github.com/karalabe/usb v0.0.2/go.mod h1:Od972xHfMJowv7NGVDiWVxk2zxnWgjLlJzE+F4F7AGU=
 github.com/kataras/golog v0.0.10/go.mod h1:yJ8YKCmyL+nWjERB90Qwn+bdyBZsaQwU3bTVFgkFIp8=
 github.com/kataras/iris/v12 v12.1.8/go.mod h1:LMYy4VlP67TQ3Zgriz8RE2h2kMZV2SgMYbq3UhfoFmE=
 github.com/kataras/neffos v0.0.14/go.mod h1:8lqADm8PnbeFfL7CLXh1WHw53dG27MC3pgi2R1rmoTE=
@@ -478,6 +495,7 @@ github.com/labstack/echo/v4 v4.2.1/go.mod h1:AA49e0DZ8kk5jTOOCKNuPR6oTnBS0dYiM4F
 github.com/labstack/echo/v4 v4.5.0/go.mod h1:czIriw4a0C1dFun+ObrXp7ok03xON0N1awStJ6ArI7Y=
 github.com/labstack/gommon v0.3.0/go.mod h1:MULnywXg0yavhxWKc+lOruYdAhDwPK9wf0OL7NoOu+k=
 github.com/leanovate/gopter v0.2.9 h1:fQjYxZaynp97ozCzfOyOuAGOU4aU/z37zf/tOujFk7c=
+github.com/leanovate/gopter v0.2.9/go.mod h1:U2L/78B+KVFIx2VmW6onHJQzXtFb+p5y3y2Sh+Jxxv8=
 github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII=
 github.com/leodido/go-urn v1.2.1 h1:BqpAaACuzVSgi/VLzGZIobT2z4v53pjosyNd9Yv6n/w=
 github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
@@ -607,6 +625,10 @@ github.com/multiformats/go-multistream v0.4.1/go.mod h1:Mz5eykRVAjJWckE2U78c6xqd
 github.com/multiformats/go-varint v0.0.1/go.mod h1:3Ls8CIEsrijN6+B7PbrXRPxHRPuXSrVKRY101jdMZYE=
 github.com/multiformats/go-varint v0.0.7 h1:sWSGR+f/eu5ABZA2ZpYKBILXTTs9JWpdEM/nEGOHFS8=
 github.com/multiformats/go-varint v0.0.7/go.mod h1:r8PUYw/fD/SjBCiKOoDlGF6QawOELpZAu9eioSos/OU=
+github.com/naoina/go-stringutil v0.1.0 h1:rCUeRUHjBjGTSHl0VC00jUPLz8/F9dDzYI70Hzifhks=
+github.com/naoina/go-stringutil v0.1.0/go.mod h1:XJ2SJL9jCtBh+P9q5btrd/Ylo8XwT/h1USek5+NqSA0=
+github.com/naoina/toml v0.1.2-0.20170918210437-9fafd6967416 h1:shk/vn9oCoOTmwcouEdwIeOtOGA/ELRUw/GwvxwfT+0=
+github.com/naoina/toml v0.1.2-0.20170918210437-9fafd6967416/go.mod h1:NBIhNtsFMo3G2szEBne+bO4gS192HuIYRqfvOWb4i1E=
 github.com/nats-io/jwt v0.3.0/go.mod h1:fRYCDE99xlTsqUzISS1Bi75UBJ6ljOJQOAAu5VglpSg=
 github.com/nats-io/nats.go v1.9.1/go.mod h1:ZjDU1L/7fJ09jvUSRVBR2e7+RnLiiIQyqyzEE/Zbp4w=
 github.com/nats-io/nkeys v0.1.0/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w=
@@ -636,6 +658,7 @@ github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1y
 github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY=
 github.com/onsi/gomega v1.19.0/go.mod h1:LY+I3pBVzYsTBU1AnDwOSxaYi9WoWiqgwooUqq9yPro=
 github.com/onsi/gomega v1.27.4 h1:Z2AnStgsdSayCMDiCU42qIz+HLqEPcgiOCXjAU/w+8E=
+github.com/onsi/gomega v1.27.4/go.mod h1:riYq/GJKh8hhoM01HN6Vmuy93AarCXCBGpvFDK3q3fQ=
 github.com/opencontainers/runtime-spec v1.0.2 h1:UfAcuLBJB9Coz72x1hgl8O5RVzTdNiaglX6v2DM6FI0=
 github.com/opencontainers/runtime-spec v1.0.2/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0=
 github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o=
@@ -851,6 +874,7 @@ golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8U
 golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20190909091759-094676da4a83/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20191227163750-53104e6ec876/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20200602180216-279210d13fed/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
@@ -972,6 +996,7 @@ golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20211020174200-9d6173849985/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220128215802-99c3d69c2c27/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220209214540-3681064d5158/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=

indexer/e2e_tests/bridge_messages_e2e_test.go

@@ -155,7 +155,7 @@ func TestE2EBridgeL2CrossDomainMessenger(t *testing.T) {
 
 	// (2) Process RelayedMessage on withdrawal finalization
 	require.Nil(t, sentMessage.RelayedMessageEventGUID)
-	_, finalizedReceipt := op_e2e.ProveAndFinalizeWithdrawal(t, *testSuite.OpCfg, testSuite.L1Client, testSuite.OpSys.Nodes["sequencer"], testSuite.OpCfg.Secrets.Alice, sentMsgReceipt)
+	_, finalizedReceipt := op_e2e.ProveAndFinalizeWithdrawal(t, *testSuite.OpCfg, testSuite.L1Client, testSuite.OpSys.EthInstances["sequencer"], testSuite.OpCfg.Secrets.Alice, sentMsgReceipt)
 
 	// wait for processor catchup
 	require.NoError(t, wait.For(context.Background(), 500*time.Millisecond, func() (bool, error) {

indexer/e2e_tests/bridge_transactions_e2e_test.go

@@ -129,7 +129,7 @@ func TestE2EBridgeTransactionsL2ToL1MessagePasserWithdrawal(t *testing.T) {
 	require.Nil(t, withdraw.ProvenL1EventGUID)
 	require.Nil(t, withdraw.FinalizedL1EventGUID)
 
-	withdrawParams, proveReceipt := op_e2e.ProveWithdrawal(t, *testSuite.OpCfg, testSuite.L1Client, testSuite.OpSys.Nodes["sequencer"], testSuite.OpCfg.Secrets.Alice, withdrawReceipt)
+	withdrawParams, proveReceipt := op_e2e.ProveWithdrawal(t, *testSuite.OpCfg, testSuite.L1Client, testSuite.OpSys.EthInstances["sequencer"], testSuite.OpCfg.Secrets.Alice, withdrawReceipt)
 	require.NoError(t, wait.For(context.Background(), 500*time.Millisecond, func() (bool, error) {
 		l1Header := testSuite.Indexer.BridgeProcessor.LatestL1Header
 		return l1Header != nil && l1Header.Number.Uint64() >= proveReceipt.BlockNumber.Uint64(), nil
@@ -189,7 +189,7 @@ func TestE2EBridgeTransactionsL2ToL1MessagePasserFailedWithdrawal(t *testing.T)
 	require.NoError(t, err)
 
 	// Prove&Finalize withdrawal
-	_, finalizeReceipt := op_e2e.ProveAndFinalizeWithdrawal(t, *testSuite.OpCfg, testSuite.L1Client, testSuite.OpSys.Nodes["sequencer"], testSuite.OpCfg.Secrets.Alice, withdrawReceipt)
+	_, finalizeReceipt := op_e2e.ProveAndFinalizeWithdrawal(t, *testSuite.OpCfg, testSuite.L1Client, testSuite.OpSys.EthInstances["sequencer"], testSuite.OpCfg.Secrets.Alice, withdrawReceipt)
 	require.NoError(t, wait.For(context.Background(), 500*time.Millisecond, func() (bool, error) {
 		l1Header := testSuite.Indexer.BridgeProcessor.LatestL1Header
 		return l1Header != nil && l1Header.Number.Uint64() >= finalizeReceipt.BlockNumber.Uint64(), nil

indexer/e2e_tests/bridge_transfers_e2e_test.go

@@ -322,7 +322,7 @@ func TestE2EBridgeTransfersStandardBridgeETHWithdrawal(t *testing.T) {
 	require.Empty(t, aliceWithdrawals.Withdrawals[0].FinalizedL1TransactionHash)
 
 	// wait for processor catchup
-	proveReceipt, finalizeReceipt := op_e2e.ProveAndFinalizeWithdrawal(t, *testSuite.OpCfg, testSuite.L1Client, testSuite.OpSys.Nodes["sequencer"], testSuite.OpCfg.Secrets.Alice, withdrawReceipt)
+	proveReceipt, finalizeReceipt := op_e2e.ProveAndFinalizeWithdrawal(t, *testSuite.OpCfg, testSuite.L1Client, testSuite.OpSys.EthInstances["sequencer"], testSuite.OpCfg.Secrets.Alice, withdrawReceipt)
 	require.NoError(t, wait.For(context.Background(), 500*time.Millisecond, func() (bool, error) {
 		l1Header := testSuite.Indexer.BridgeProcessor.LatestL1Header
 		return l1Header != nil && l1Header.Number.Uint64() >= finalizeReceipt.BlockNumber.Uint64(), nil
@@ -400,7 +400,7 @@ func TestE2EBridgeTransfersL2ToL1MessagePasserReceive(t *testing.T) {
 	require.Empty(t, aliceWithdrawals.Withdrawals[0].FinalizedL1TransactionHash)
 
 	// wait for processor catchup
-	proveReceipt, finalizeReceipt := op_e2e.ProveAndFinalizeWithdrawal(t, *testSuite.OpCfg, testSuite.L1Client, testSuite.OpSys.Nodes["sequencer"], testSuite.OpCfg.Secrets.Alice, l2ToL1WithdrawReceipt)
+	proveReceipt, finalizeReceipt := op_e2e.ProveAndFinalizeWithdrawal(t, *testSuite.OpCfg, testSuite.L1Client, testSuite.OpSys.EthInstances["sequencer"], testSuite.OpCfg.Secrets.Alice, l2ToL1WithdrawReceipt)
 	require.NoError(t, wait.For(context.Background(), 500*time.Millisecond, func() (bool, error) {
 		l1Header := testSuite.Indexer.BridgeProcessor.LatestL1Header
 		return l1Header != nil && l1Header.Number.Uint64() >= finalizeReceipt.BlockNumber.Uint64(), nil

indexer/e2e_tests/etl_e2e_test.go

@@ -79,7 +79,7 @@ func TestE2EETL(t *testing.T) {
 				require.NotNil(t, latestOutput)
 				require.GreaterOrEqual(t, latestOutput.L2BlockNumber.Int.Uint64(), uint64(9))
 
-				l2EthClient, err := node.DialEthClient(testSuite.OpSys.Nodes["sequencer"].HTTPEndpoint())
+				l2EthClient, err := node.DialEthClient(testSuite.OpSys.EthInstances["sequencer"].HTTPEndpoint())
 				require.NoError(t, err)
 
 				submissionInterval := testSuite.OpCfg.DeployConfig.L2OutputOracleSubmissionInterval

indexer/e2e_tests/setup.go

@@ -49,7 +49,8 @@ func createE2ETestSuite(t *testing.T) E2ETestSuite {
 
 	// Rollup System Configuration and Start
 	opCfg := op_e2e.DefaultSystemConfig(t)
-	opSys, err := opCfg.Start()
+	opCfg.DeployConfig.FinalizationPeriodSeconds = 2
+	opSys, err := opCfg.Start(t)
 	require.NoError(t, err)
 
 	// E2E tests can run on the order of magnitude of minutes. Once
@@ -66,8 +67,8 @@ func createE2ETestSuite(t *testing.T) E2ETestSuite {
 			User: dbUser,
 		},
 		RPCs: config.RPCsConfig{
-			L1RPC: opSys.Nodes["l1"].HTTPEndpoint(),
-			L2RPC: opSys.Nodes["sequencer"].HTTPEndpoint(),
+			L1RPC: opSys.EthInstances["l1"].HTTPEndpoint(),
+			L2RPC: opSys.EthInstances["sequencer"].HTTPEndpoint(),
 		},
 		Chain: config.ChainConfig{
 			L1Contracts: config.L1Contracts{

op-e2e/.gitignore

+external_*/shim

op-e2e/Makefile

@@ -28,6 +28,11 @@ clean:
 lint:
 	golangci-lint run -E goimports,sqlclosecheck,bodyclose,asciicheck,misspell,errorlint --timeout 5m -e "errors.As" -e "errors.Is" ./...
 
+test-external-%: pre-test
+	make -C ./external_$*/
+	go test -v --externalL2 ./external_$*/shim
+
+
 .PHONY: \
 	test \
 	lint

op-e2e/bridge_test.go

@@ -24,7 +24,7 @@ func TestERC20BridgeDeposits(t *testing.T) {
 
 	cfg := DefaultSystemConfig(t)
 
-	sys, err := cfg.Start()
+	sys, err := cfg.Start(t)
 	require.Nil(t, err, "Error starting up system")
 	defer sys.Close()
 

op-e2e/config/init.go

@@ -26,6 +26,11 @@ var (
 	L1Deployments *genesis.L1Deployments
 	// DeployConfig represents the deploy config used by the system.
 	DeployConfig *genesis.DeployConfig
+	// ExternalL2Nodes is the shim to use if external ethereum client testing is
+	// enabled
+	ExternalL2Nodes string
+	// EthNodeVerbosity is the level of verbosity to output
+	EthNodeVerbosity int
 )
 
 // Init testing to enable test flags
@@ -53,6 +58,8 @@ func init() {
 	flag.StringVar(&l1AllocsPath, "l1-allocs", defaultL1AllocsPath, "")
 	flag.StringVar(&l1DeploymentsPath, "l1-deployments", defaultL1DeploymentsPath, "")
 	flag.StringVar(&deployConfigPath, "deploy-config", defaultDeployConfigPath, "")
+	flag.StringVar(&ExternalL2Nodes, "externalL2", "", "Enable tests with external L2")
+	flag.IntVar(&EthNodeVerbosity, "ethLogVerbosity", 3, "The level of verbosity to use for the eth node logs")
 	flag.Parse()
 
 	if err := allExist(l1AllocsPath, l1DeploymentsPath, deployConfigPath); err != nil {

op-e2e/external.go

+package op_e2e
+
+import (
+	"encoding/json"
+	"math/big"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"testing"
+	"time"
+
+	"github.com/ethereum-optimism/optimism/op-e2e/config"
+	"github.com/ethereum-optimism/optimism/op-e2e/external"
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/core"
+	"github.com/ethereum/go-ethereum/params"
+	"github.com/onsi/gomega/gexec"
+	"github.com/stretchr/testify/require"
+)
+
+type ExternalRunner struct {
+	Name    string
+	BinPath string
+	Genesis *core.Genesis
+	JWTPath string
+}
+
+type ExternalEthClient struct {
+	Session   *gexec.Session
+	Endpoints external.Endpoints
+}
+
+func (eec *ExternalEthClient) HTTPEndpoint() string {
+	return eec.Endpoints.HTTPEndpoint
+}
+
+func (eec *ExternalEthClient) WSEndpoint() string {
+	return eec.Endpoints.WSEndpoint
+}
+
+func (eec *ExternalEthClient) HTTPAuthEndpoint() string {
+	return eec.Endpoints.HTTPAuthEndpoint
+}
+
+func (eec *ExternalEthClient) WSAuthEndpoint() string {
+	return eec.Endpoints.WSAuthEndpoint
+}
+
+func (eec *ExternalEthClient) Close() {
+	eec.Session.Terminate()
+	select {
+	case <-time.After(5 * time.Second):
+		eec.Session.Kill()
+	case <-eec.Session.Exited:
+	}
+}
+
+func (er *ExternalRunner) Run(t *testing.T) *ExternalEthClient {
+	if er.BinPath == "" {
+		t.Error("no external bin path set")
+	}
+
+	if er.JWTPath == "" {
+		er.JWTPath = writeDefaultJWT(t)
+	}
+
+	if er.Genesis == nil {
+		er.Genesis = &core.Genesis{
+			Alloc: core.GenesisAlloc{
+				common.Address{1}: core.GenesisAccount{Balance: big.NewInt(1)},
+			},
+			Config:     params.OptimismTestConfig,
+			Difficulty: big.NewInt(0),
+		}
+	}
+
+	workDir := t.TempDir()
+
+	config := external.Config{
+		DataDir:            filepath.Join(workDir, "datadir"),
+		JWTPath:            er.JWTPath,
+		ChainID:            er.Genesis.Config.ChainID.Uint64(),
+		GenesisPath:        filepath.Join(workDir, "genesis.json"),
+		EndpointsReadyPath: filepath.Join(workDir, "endpoints.json"),
+		Verbosity:          uint64(config.EthNodeVerbosity),
+	}
+
+	err := os.Mkdir(config.DataDir, 0o700)
+	require.NoError(t, err)
+
+	genesisFile, err := os.Create(config.GenesisPath)
+	require.NoError(t, err)
+	err = json.NewEncoder(genesisFile).Encode(er.Genesis)
+	require.NoError(t, err)
+
+	configPath := filepath.Join(workDir, "config.json")
+	configFile, err := os.Create(configPath)
+	require.NoError(t, err)
+	err = json.NewEncoder(configFile).Encode(config)
+	require.NoError(t, err)
+
+	cmd := exec.Command(er.BinPath, "--config", configPath)
+	cmd.Dir = filepath.Dir(er.BinPath)
+	sess, err := gexec.Start(
+		cmd,
+		gexec.NewPrefixedWriter("[extout:"+er.Name+"]", os.Stdout),
+		gexec.NewPrefixedWriter("[exterr:"+er.Name+"]", os.Stderr),
+	)
+	require.NoError(t, err)
+
+	// 2 minutes may seem like a long timeout, and, it definitely is.  That
+	// being said, when running these tests with high parallelism turned on, the
+	// node startup time can be substantial (remember, this usually is a
+	// multi-step process initializing the database and then starting the
+	// client).
+	require.Eventually(
+		t,
+		func() bool {
+			_, err := os.Stat(config.EndpointsReadyPath)
+			return err == nil
+		},
+		2*time.Minute,
+		10*time.Millisecond,
+		"external runner did not create ready file at %s within timeout",
+		config.EndpointsReadyPath,
+	)
+
+	readyFile, err := os.Open(config.EndpointsReadyPath)
+	require.NoError(t, err)
+	var endpoints external.Endpoints
+	err = json.NewDecoder(readyFile).Decode(&endpoints)
+	require.NoError(t, err)
+
+	return &ExternalEthClient{
+		Session:   sess,
+		Endpoints: endpoints,
+	}
+}

op-e2e/external/config.go

+package external
+
+import (
+	"encoding/json"
+	"os"
+)
+
+type Config struct {
+	DataDir     string `json:"data_dir"`
+	JWTPath     string `json:"jwt_path"`
+	ChainID     uint64 `json:"chain_id"`
+	GasCeil     uint64 `json:"gas_ceil"`
+	GenesisPath string `json:"genesis_path"`
+	Verbosity   uint64 `json:"verbosity"`
+
+	// EndpointsReadyPath is the location to write the endpoint configuration file.
+	// Note, this should be written atomically by writing the JSON, then moving
+	// it to this path to avoid races.  A helper AtomicEncode is provided for
+	// golang clients.
+	EndpointsReadyPath string `json:"endpoints_ready_path"`
+}
+
+// AtomicEncode json encodes val to path+".atomic" then moves the path+".atomic"
+// file to path
+func AtomicEncode(path string, val any) error {
+	atomicPath := path + ".atomic"
+	atomicFile, err := os.Create(atomicPath)
+	if err != nil {
+		return err
+	}
+	if err = json.NewEncoder(atomicFile).Encode(val); err != nil {
+		return err
+	}
+	return os.Rename(atomicPath, path)
+}
+
+type Endpoints struct {
+	HTTPEndpoint     string `json:"http_endpoint"`
+	WSEndpoint       string `json:"ws_endpoint"`
+	HTTPAuthEndpoint string `json:"http_auth_endpoint"`
+	WSAuthEndpoint   string `json:"ws_auth_endpoint"`
+}

op-e2e/external_geth/.gitignore

+op-geth

op-e2e/external_geth/Makefile

+default: shim op-geth
+
+op-geth:
+	go build -o op-geth "github.com/ethereum/go-ethereum/cmd/geth"
+.PHONY: op-geth
+
+shim: main.go
+	go build -o shim .

op-e2e/external_geth/README.md

+# external_geth shim
+
+This shim is an example of how to write an adapter for an external ethereum
+client to allow for its use in the op-e2e tests.
+
+## Invocation
+
+Generally speaking, you can utilize this shim by simply executing:
+
+```
+make test-external-geth
+```
+
+The `Makefile` is structured such that if you duplicate this directory and
+tweak this code, you may simply execute:
+
+```
+make test-external-<your-client>
+```
+
+and the execution should happen as well.
+
+*NOTE:* Attempting to iterate for development requires explicit rebuilding of
+the binary being shimmed.  Most likely to accomplish this, you may want to add
+initialization code to the TestMain of the e2e to build your binary, or use
+some other technique like custom build scripts or IDE integrations which cause
+the binary to be rebuilt before executing the tests.
+
+## Arguments
+
+*--config <path>* The config path is a required argument, it points to a JSON
+file which contains details of the L2 environment to bring up (including the
+`genesis.json` path, the chain ID, the JWT path, and a ready file path).  See
+the data structures in `op-e2e/external/config.go` for more details.
+
+## Operation
+
+This shim will first execute a process to initialize the op-geth database.
+Then, it will start the op-geth process itself.  It watches the output of the
+process and looks for the lines indicating that the HTTP server and Auth HTTP
+server have started up.  It then reads the ports which were allocated (because
+the requested ports were passed in as ephemeral via the CLI arguments).
+
+## Generalization
+
+This shim is included to help document an demonstrate the usage of the
+external ethereum process e2e test execution.  It is configured to execute in
+CI to help ensure that the tests remain compatible with external clients.
+
+To create your own external test client, these files can likely be used as a
+starting point, changing the arguments, log scraping, and other details.  Or,
+depending on the client and your preference, any binary which is capable of
+reading and writing the necessary JSON files should be sufficient (though
+will be required to replicate some of the parsing and other logic encapsulated
+here).

op-e2e/external_geth/main.go

+package main
+
+import (
+	"encoding/json"
+	"flag"
+	"fmt"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strconv"
+	"time"
+
+	"github.com/ethereum-optimism/optimism/op-e2e/external"
+	"github.com/onsi/gomega/gbytes"
+	"github.com/onsi/gomega/gexec"
+)
+
+func main() {
+	var configPath string
+	flag.StringVar(&configPath, "config", "", "Execute based on the config in this file")
+	flag.Parse()
+	if err := run(configPath); err != nil {
+		fmt.Println(err.Error())
+		os.Exit(1)
+	}
+	os.Exit(0)
+}
+
+func run(configPath string) error {
+	if configPath == "" {
+		return fmt.Errorf("must supply a '--config <path>' flag")
+	}
+
+	configFile, err := os.Open(configPath)
+	if err != nil {
+		return fmt.Errorf("could not open config: %w", err)
+	}
+
+	var config external.Config
+	if err := json.NewDecoder(configFile).Decode(&config); err != nil {
+		return fmt.Errorf("could not decode config file: %w", err)
+	}
+
+	binPath, err := filepath.Abs("op-geth")
+	if err != nil {
+		return fmt.Errorf("could not get absolute path of op-geth")
+	}
+	if _, err := os.Stat(binPath); err != nil {
+		return fmt.Errorf("could not locate op-geth in working directory, did you forget to run '--init'?")
+	}
+
+	fmt.Printf("================== op-geth shim initializing chain config ==========================\n")
+	if err := initialize(binPath, config); err != nil {
+		return fmt.Errorf("could not initialize datadir: %s %w", binPath, err)
+	}
+
+	fmt.Printf("==================    op-geth shim executing op-geth     ==========================\n")
+	sess, err := execute(binPath, config)
+	if err != nil {
+		return fmt.Errorf("could not execute geth: %w", err)
+	}
+	defer sess.Close()
+
+	fmt.Printf("==================    op-geth shim encoding ready-file   ==========================\n")
+	if err := external.AtomicEncode(config.EndpointsReadyPath, sess.endpoints); err != nil {
+		return fmt.Errorf("could not encode endpoints")
+	}
+
+	fmt.Printf("==================    op-geth shim awaiting termination  ==========================\n")
+	select {
+	case <-sess.session.Exited:
+		return fmt.Errorf("geth exited")
+	case <-time.After(30 * time.Minute):
+		return fmt.Errorf("exiting after 30 minute timeout")
+	}
+}
+
+func initialize(binPath string, config external.Config) error {
+	cmd := exec.Command(
+		binPath,
+		"--datadir", config.DataDir,
+		"init", config.GenesisPath,
+	)
+	return cmd.Run()
+}
+
+type gethSession struct {
+	session   *gexec.Session
+	endpoints *external.Endpoints
+}
+
+func (es *gethSession) Close() {
+	es.session.Terminate()
+	select {
+	case <-time.After(5 * time.Second):
+		es.session.Kill()
+	case <-es.session.Exited:
+	}
+}
+
+func execute(binPath string, config external.Config) (*gethSession, error) {
+	if config.Verbosity < 2 {
+		return nil, fmt.Errorf("a minimum configured verbosity of 2 is required")
+	}
+	cmd := exec.Command(
+		binPath,
+		"--datadir", config.DataDir,
+		"--http",
+		"--http.addr", "127.0.0.1",
+		"--http.port", "0",
+		"--http.api", "web3,debug,eth,txpool,net,engine",
+		"--ws",
+		"--ws.addr", "127.0.0.1",
+		"--ws.port", "0",
+		"--ws.api", "debug,eth,txpool,net,engine",
+		"--syncmode=full",
+		"--nodiscover",
+		"--port", "0",
+		"--maxpeers", "0",
+		"--networkid", strconv.FormatUint(config.ChainID, 10),
+		"--authrpc.addr", "127.0.0.1",
+		"--authrpc.port", "0",
+		"--authrpc.jwtsecret", config.JWTPath,
+		"--gcmode=archive",
+		"--verbosity", strconv.FormatUint(config.Verbosity, 10),
+	)
+	sess, err := gexec.Start(cmd, os.Stdout, os.Stderr)
+	if err != nil {
+		return nil, fmt.Errorf("could not start op-geth session: %w", err)
+	}
+	matcher := gbytes.Say("HTTP server started\\s*endpoint=127.0.0.1:")
+	var enginePort, httpPort int
+	for enginePort == 0 || httpPort == 0 {
+		match, err := matcher.Match(sess.Err)
+		if err != nil {
+			return nil, fmt.Errorf("could not execute matcher")
+		}
+		if !match {
+			if sess.Err.Closed() {
+				return nil, fmt.Errorf("op-geth exited before announcing http ports")
+			}
+			// Wait for a bit more output, then try again
+			time.Sleep(10 * time.Millisecond)
+			continue
+		}
+		var authString string
+		var port int
+		fmt.Fscanf(sess.Err, "%d %s", &port, &authString)
+		switch authString {
+		case "auth=true":
+			enginePort = port
+		case "auth=false":
+			httpPort = port
+		default:
+			return nil, fmt.Errorf("unexpected auth string %q", authString)
+		}
+	}
+
+	return &gethSession{
+		session: sess,
+		endpoints: &external.Endpoints{
+			HTTPEndpoint:     fmt.Sprintf("http://127.0.0.1:%d/", httpPort),
+			WSEndpoint:       fmt.Sprintf("ws://127.0.0.1:%d/", httpPort),
+			HTTPAuthEndpoint: fmt.Sprintf("http://127.0.0.1:%d/", enginePort),
+			WSAuthEndpoint:   fmt.Sprintf("ws://127.0.0.1:%d/", enginePort),
+		},
+	}, nil
+}

op-e2e/external_geth/main_test.go

+package main
+
+import (
+	"net"
+	"net/url"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"testing"
+	"time"
+
+	e2e "github.com/ethereum-optimism/optimism/op-e2e"
+	"github.com/ethereum-optimism/optimism/op-e2e/config"
+	"github.com/stretchr/testify/require"
+)
+
+func TestShim(t *testing.T) {
+	shimPath, err := filepath.Abs("shim")
+	require.NoError(t, err)
+	cmd := exec.Command("go", "build", "-o", shimPath, ".")
+	cmd.Stdout = os.Stdout
+	cmd.Stderr = os.Stderr
+	err = cmd.Run()
+	require.NoError(t, err)
+	require.FileExists(t, "shim")
+
+	opGethPath, err := filepath.Abs("op-geth")
+	require.NoError(t, err)
+	cmd = exec.Command("go", "build", "-o", opGethPath, "github.com/ethereum/go-ethereum/cmd/geth")
+	cmd.Stdout = os.Stdout
+	cmd.Stderr = os.Stderr
+	err = cmd.Run()
+	require.NoError(t, err)
+	require.FileExists(t, "op-geth")
+
+	config.EthNodeVerbosity = 4
+
+	ec := (&e2e.ExternalRunner{
+		Name:    "TestShim",
+		BinPath: shimPath,
+	}).Run(t)
+	t.Cleanup(ec.Close)
+
+	for _, endpoint := range []string{
+		ec.HTTPEndpoint(),
+		ec.HTTPAuthEndpoint(),
+		ec.WSEndpoint(),
+		ec.WSAuthEndpoint(),
+	} {
+		plainURL, err := url.ParseRequestURI(endpoint)
+		require.NoError(t, err)
+		_, err = net.DialTimeout("tcp", plainURL.Host, time.Second)
+		require.NoError(t, err, "could not connect to HTTP port")
+	}
+}

op-e2e/external_geth/tools.go

+//go:build tools
+
+package main
+
+import _ "github.com/ethereum/go-ethereum/cmd/geth"

op-e2e/faultproof_test.go

@@ -409,7 +409,7 @@ func startFaultDisputeSystem(t *testing.T) (*System, *ethclient.Client) {
 	cfg.SupportL1TimeTravel = true
 	cfg.DeployConfig.L2OutputOracleSubmissionInterval = 1
 	cfg.NonFinalizedProposals = true // Submit output proposals asap
-	sys, err := cfg.Start()
-	require.NoError(t, err, "Error starting up system")
+	sys, err := cfg.Start(t)
+	require.Nil(t, err, "Error starting up system")
 	return sys, sys.Clients["l1"]
 }

op-e2e/helper.go

 package op_e2e
 
 import (
-	"flag"
 	"os"
 	"testing"
 
+	"github.com/ethereum-optimism/optimism/op-e2e/config"
 	"github.com/ethereum/go-ethereum/log"
 )
 
-var enableParallelTesting bool = true
-
-// Init testing to enable test flags
-var _ = func() bool {
-	testing.Init()
-	return true
-}()
-
-var verboseGethNodes bool
-
-func init() {
-	flag.BoolVar(&verboseGethNodes, "gethlogs", true, "Enable logs on geth nodes")
-	flag.Parse()
-	if os.Getenv("OP_E2E_DISABLE_PARALLEL") == "true" {
-		enableParallelTesting = false
-	}
-}
+var enableParallelTesting bool = os.Getenv("OP_E2E_DISABLE_PARALLEL") != "true"
 
 func InitParallel(t *testing.T) {
 	t.Helper()
 	if enableParallelTesting {
 		t.Parallel()
 	}
-	if !verboseGethNodes {
+	if config.EthNodeVerbosity < 0 {
 		log.Root().SetHandler(log.DiscardHandler())
 	}
 }

op-e2e/op_geth_test.go

@@ -48,7 +48,7 @@ func TestMissingGasLimit(t *testing.T) {
 func TestTxGasSameAsBlockGasLimit(t *testing.T) {
 	InitParallel(t)
 	cfg := DefaultSystemConfig(t)
-	sys, err := cfg.Start()
+	sys, err := cfg.Start(t)
 	require.Nil(t, err, "Error starting up system")
 	defer sys.Close()
 

op-e2e/setup.go

@@ -139,6 +139,7 @@ func DefaultSystemConfig(t *testing.T) SystemConfig {
 		GethOptions:                map[string][]GethOption{},
 		P2PTopology:                nil, // no P2P connectivity by default
 		NonFinalizedProposals:      false,
+		ExternalL2Nodes:            config.ExternalL2Nodes,
 		BatcherTargetL1TxSizeBytes: 100_000,
 	}
 }
@@ -146,7 +147,7 @@ func DefaultSystemConfig(t *testing.T) SystemConfig {
 func writeDefaultJWT(t *testing.T) string {
 	// Sadly the geth node config cannot load JWT secret from memory, it has to be a file
 	jwtPath := path.Join(t.TempDir(), "jwt_secret")
-	if err := os.WriteFile(jwtPath, []byte(hexutil.Encode(testingJWTSecret[:])), 0600); err != nil {
+	if err := os.WriteFile(jwtPath, []byte(hexutil.Encode(testingJWTSecret[:])), 0o600); err != nil {
 		t.Fatalf("failed to prepare jwt file for geth: %v", err)
 	}
 	return jwtPath
@@ -174,6 +175,8 @@ type SystemConfig struct {
 	ProposerLogger log.Logger
 	BatcherLogger  log.Logger
 
+	ExternalL2Nodes string
+
 	// map of outbound connections to other nodes. Node names prefixed with "~" are unconnected but linked.
 	// A nil map disables P2P completely.
 	// Any node name not in the topology will not have p2p enabled.
@@ -195,6 +198,41 @@ type SystemConfig struct {
 	SupportL1TimeTravel bool
 }
 
+type GethInstance struct {
+	Backend *geth_eth.Ethereum
+	Node    *node.Node
+}
+
+func (gi *GethInstance) HTTPEndpoint() string {
+	return gi.Node.HTTPEndpoint()
+}
+
+func (gi *GethInstance) WSEndpoint() string {
+	return gi.Node.WSEndpoint()
+}
+
+func (gi *GethInstance) WSAuthEndpoint() string {
+	return gi.Node.WSAuthEndpoint()
+}
+
+func (gi *GethInstance) HTTPAuthEndpoint() string {
+	return gi.Node.HTTPAuthEndpoint()
+}
+
+func (gi *GethInstance) Close() {
+	gi.Node.Close()
+}
+
+// EthInstance is either an in process Geth or external process exposing its
+// endpoints over the network
+type EthInstance interface {
+	HTTPEndpoint() string
+	WSEndpoint() string
+	HTTPAuthEndpoint() string
+	WSAuthEndpoint() string
+	Close()
+}
+
 type System struct {
 	cfg SystemConfig
 
@@ -203,9 +241,9 @@ type System struct {
 	L2GenesisCfg *core.Genesis
 
 	// Connections to running nodes
-	Nodes             map[string]*node.Node
-	Backends          map[string]*geth_eth.Ethereum
+	EthInstances      map[string]EthInstance
 	Clients           map[string]*ethclient.Client
+	RawClients        map[string]*rpc.Client
 	RollupNodes       map[string]*rollupNode.OpNode
 	L2OutputSubmitter *l2os.L2OutputSubmitter
 	BatchSubmitter    *bss.BatchSubmitter
@@ -220,7 +258,7 @@ type System struct {
 }
 
 func (sys *System) NodeEndpoint(name string) string {
-	return selectEndpoint(sys.Nodes[name])
+	return selectEndpoint(sys.EthInstances[name])
 }
 
 func (sys *System) Close() {
@@ -236,8 +274,8 @@ func (sys *System) Close() {
 	for _, node := range sys.RollupNodes {
 		node.Close()
 	}
-	for _, node := range sys.Nodes {
-		node.Close()
+	for _, ei := range sys.EthInstances {
+		ei.Close()
 	}
 	sys.Mocknet.Close()
 }
@@ -273,18 +311,18 @@ func (s *SystemConfigOptions) Get(key, role string) (systemConfigHook, bool) {
 	return v, ok
 }
 
-func (cfg SystemConfig) Start(_opts ...SystemConfigOption) (*System, error) {
+func (cfg SystemConfig) Start(t *testing.T, _opts ...SystemConfigOption) (*System, error) {
 	opts, err := NewSystemConfigOptions(_opts)
 	if err != nil {
 		return nil, err
 	}
 
 	sys := &System{
-		cfg:         cfg,
-		Nodes:       make(map[string]*node.Node),
-		Backends:    make(map[string]*geth_eth.Ethereum),
-		Clients:     make(map[string]*ethclient.Client),
-		RollupNodes: make(map[string]*rollupNode.OpNode),
+		cfg:          cfg,
+		EthInstances: make(map[string]EthInstance),
+		Clients:      make(map[string]*ethclient.Client),
+		RawClients:   make(map[string]*rpc.Client),
+		RollupNodes:  make(map[string]*rollupNode.OpNode),
 	}
 	didErrAfterStart := false
 	defer func() {
@@ -292,8 +330,8 @@ func (cfg SystemConfig) Start(_opts ...SystemConfigOption) (*System, error) {
 			for _, node := range sys.RollupNodes {
 				node.Close()
 			}
-			for _, node := range sys.Nodes {
-				node.Close()
+			for _, ei := range sys.EthInstances {
+				ei.Close()
 			}
 		}
 	}()
@@ -388,41 +426,53 @@ func (cfg SystemConfig) Start(_opts ...SystemConfigOption) (*System, error) {
 	if err != nil {
 		return nil, err
 	}
-	sys.Nodes["l1"] = l1Node
-	sys.Backends["l1"] = l1Backend
-
-	for name := range cfg.Nodes {
-		node, backend, err := initL2Geth(name, big.NewInt(int64(cfg.DeployConfig.L2ChainID)), l2Genesis, cfg.JWTFilePath, cfg.GethOptions[name]...)
-		if err != nil {
-			return nil, err
-		}
-		sys.Nodes[name] = node
-		sys.Backends[name] = backend
+	sys.EthInstances["l1"] = &GethInstance{
+		Backend: l1Backend,
+		Node:    l1Node,
 	}
-
-	// Start
 	err = l1Node.Start()
 	if err != nil {
 		didErrAfterStart = true
 		return nil, err
 	}
-	for name, node := range sys.Nodes {
-		if name == "l1" {
-			continue
-		}
-		err = node.Start()
-		if err != nil {
-			didErrAfterStart = true
-			return nil, err
+
+	for name := range cfg.Nodes {
+		var ethClient EthInstance
+		if cfg.ExternalL2Nodes == "" {
+			node, backend, err := initL2Geth(name, big.NewInt(int64(cfg.DeployConfig.L2ChainID)), l2Genesis, cfg.JWTFilePath, cfg.GethOptions[name]...)
+			if err != nil {
+				return nil, err
+			}
+			gethInst := &GethInstance{
+				Backend: backend,
+				Node:    node,
+			}
+			err = gethInst.Node.Start()
+			if err != nil {
+				didErrAfterStart = true
+				return nil, err
+			}
+			ethClient = gethInst
+		} else {
+			if len(cfg.GethOptions[name]) > 0 {
+				t.Errorf("External L2 nodes do not support configuration through GethOptions")
+			}
+			ethClient = (&ExternalRunner{
+				Name:    name,
+				BinPath: cfg.ExternalL2Nodes,
+				Genesis: l2Genesis,
+				JWTPath: cfg.JWTFilePath,
+			}).Run(t)
 		}
+		sys.EthInstances[name] = ethClient
 	}
 
 	// Configure connections to L1 and L2 for rollup nodes.
-	// TODO: refactor testing to use in-process rpc connections instead of websockets.
-
+	// TODO: refactor testing to allow use of in-process rpc connections instead
+	// of only websockets (which are required for external eth client tests).
 	for name, rollupCfg := range cfg.Nodes {
-		configureL1(rollupCfg, l1Node)
-		configureL2(rollupCfg, sys.Nodes[name], cfg.JWTSecret)
+		configureL1(rollupCfg, sys.EthInstances["l1"])
+		configureL2(rollupCfg, sys.EthInstances[name], cfg.JWTSecret)
 
 		rollupCfg.L2Sync = &rollupNode.PreparedL2SyncEndpoint{
 			Client:   nil,
@@ -438,14 +488,18 @@ func (cfg SystemConfig) Start(_opts ...SystemConfigOption) (*System, error) {
 		didErrAfterStart = true
 		return nil, err
 	}
-	l1Client := ethclient.NewClient(rpc.DialInProc(l1Srv))
+	rawL1Client := rpc.DialInProc(l1Srv)
+	l1Client := ethclient.NewClient(rawL1Client)
 	sys.Clients["l1"] = l1Client
-	for name, node := range sys.Nodes {
-		client, err := ethclient.DialContext(ctx, node.WSEndpoint())
+	sys.RawClients["l1"] = rawL1Client
+	for name, ethInst := range sys.EthInstances {
+		rawClient, err := rpc.DialContext(ctx, ethInst.WSEndpoint())
 		if err != nil {
 			didErrAfterStart = true
 			return nil, err
 		}
+		client := ethclient.NewClient(rawClient)
+		sys.RawClients[name] = rawClient
 		sys.Clients[name] = client
 	}
 
@@ -579,11 +633,11 @@ func (cfg SystemConfig) Start(_opts ...SystemConfigOption) (*System, error) {
 
 	// L2Output Submitter
 	sys.L2OutputSubmitter, err = l2os.NewL2OutputSubmitterFromCLIConfig(l2os.CLIConfig{
-		L1EthRpc:          sys.Nodes["l1"].WSEndpoint(),
+		L1EthRpc:          sys.EthInstances["l1"].WSEndpoint(),
 		RollupRpc:         sys.RollupNodes["sequencer"].HTTPEndpoint(),
 		L2OOAddress:       config.L1Deployments.L2OutputOracleProxy.Hex(),
 		PollInterval:      50 * time.Millisecond,
-		TxMgrConfig:       newTxMgrConfig(sys.Nodes["l1"].WSEndpoint(), cfg.Secrets.Proposer),
+		TxMgrConfig:       newTxMgrConfig(sys.EthInstances["l1"].WSEndpoint(), cfg.Secrets.Proposer),
 		AllowNonFinalized: cfg.NonFinalizedProposals,
 		LogConfig: oplog.CLIConfig{
 			Level:  "info",
@@ -600,8 +654,8 @@ func (cfg SystemConfig) Start(_opts ...SystemConfigOption) (*System, error) {
 
 	// Batch Submitter
 	sys.BatchSubmitter, err = bss.NewBatchSubmitterFromCLIConfig(bss.CLIConfig{
-		L1EthRpc:               sys.Nodes["l1"].WSEndpoint(),
-		L2EthRpc:               sys.Nodes["sequencer"].WSEndpoint(),
+		L1EthRpc:               sys.EthInstances["l1"].WSEndpoint(),
+		L2EthRpc:               sys.EthInstances["sequencer"].WSEndpoint(),
 		RollupRpc:              sys.RollupNodes["sequencer"].HTTPEndpoint(),
 		MaxPendingTransactions: 0,
 		MaxChannelDuration:     1,
@@ -613,7 +667,7 @@ func (cfg SystemConfig) Start(_opts ...SystemConfigOption) (*System, error) {
 		},
 		SubSafetyMargin: 4,
 		PollInterval:    50 * time.Millisecond,
-		TxMgrConfig:     newTxMgrConfig(sys.Nodes["l1"].WSEndpoint(), cfg.Secrets.Batcher),
+		TxMgrConfig:     newTxMgrConfig(sys.EthInstances["l1"].WSEndpoint(), cfg.Secrets.Batcher),
 		LogConfig: oplog.CLIConfig{
 			Level:  "info",
 			Format: "text",
@@ -678,7 +732,7 @@ func (sys *System) newMockNetPeer() (host.Host, error) {
 	return sys.Mocknet.AddPeerWithPeerstore(p, eps)
 }
 
-func selectEndpoint(node *node.Node) string {
+func selectEndpoint(node EthInstance) string {
 	useHTTP := os.Getenv("OP_E2E_USE_HTTP") == "true"
 	if useHTTP {
 		log.Info("using HTTP client")
@@ -687,7 +741,7 @@ func selectEndpoint(node *node.Node) string {
 	return node.WSEndpoint()
 }
 
-func configureL1(rollupNodeCfg *rollupNode.Config, l1Node *node.Node) {
+func configureL1(rollupNodeCfg *rollupNode.Config, l1Node EthInstance) {
 	l1EndpointConfig := selectEndpoint(l1Node)
 	rollupNodeCfg.L1 = &rollupNode.L1EndpointConfig{
 		L1NodeAddr:       l1EndpointConfig,
@@ -698,7 +752,13 @@ func configureL1(rollupNodeCfg *rollupNode.Config, l1Node *node.Node) {
 		HttpPollInterval: time.Millisecond * 100,
 	}
 }
-func configureL2(rollupNodeCfg *rollupNode.Config, l2Node *node.Node, jwtSecret [32]byte) {
+
+type WSOrHTTPEndpoint interface {
+	WSAuthEndpoint() string
+	HTTPAuthEndpoint() string
+}
+
+func configureL2(rollupNodeCfg *rollupNode.Config, l2Node WSOrHTTPEndpoint, jwtSecret [32]byte) {
 	useHTTP := os.Getenv("OP_E2E_USE_HTTP") == "true"
 	l2EndpointConfig := l2Node.WSAuthEndpoint()
 	if useHTTP {

op-e2e/system_adminrpc_test.go

@@ -17,7 +17,7 @@ func TestStopStartSequencer(t *testing.T) {
 	InitParallel(t)
 
 	cfg := DefaultSystemConfig(t)
-	sys, err := cfg.Start()
+	sys, err := cfg.Start(t)
 	require.Nil(t, err, "Error starting up system")
 	defer sys.Close()
 
@@ -83,7 +83,7 @@ func TestPersistSequencerStateWhenChanged(t *testing.T) {
 	delete(cfg.Nodes, "verifier")
 	cfg.Nodes["sequencer"].ConfigPersistence = node.NewConfigPersistence(stateFile)
 
-	sys, err := cfg.Start()
+	sys, err := cfg.Start(t)
 	require.NoError(t, err)
 	defer sys.Close()
 
@@ -118,7 +118,7 @@ func TestLoadSequencerStateOnStarted_Stopped(t *testing.T) {
 	seqCfg := cfg.Nodes["sequencer"]
 	seqCfg.ConfigPersistence = node.NewConfigPersistence(stateFile)
 
-	sys, err := cfg.Start()
+	sys, err := cfg.Start(t)
 	require.NoError(t, err)
 	defer sys.Close()
 
@@ -152,7 +152,7 @@ func TestLoadSequencerStateOnStarted_Started(t *testing.T) {
 	seqCfg.Driver.SequencerStopped = true
 	seqCfg.ConfigPersistence = node.NewConfigPersistence(stateFile)
 
-	sys, err := cfg.Start()
+	sys, err := cfg.Start(t)
 	require.NoError(t, err)
 	defer sys.Close()
 

op-e2e/system_fpp_test.go

@@ -56,7 +56,7 @@ func testVerifyL2OutputRootEmptyBlock(t *testing.T, detached bool) {
 	// But not too small to ensure that our claim and subsequent state change is published
 	cfg.DeployConfig.SequencerWindowSize = 16
 
-	sys, err := cfg.Start()
+	sys, err := cfg.Start(t)
 	require.Nil(t, err, "Error starting up system")
 	defer sys.Close()
 
@@ -154,7 +154,7 @@ func testVerifyL2OutputRoot(t *testing.T, detached bool) {
 	// We don't need a verifier - just the sequencer is enough
 	delete(cfg.Nodes, "verifier")
 
-	sys, err := cfg.Start()
+	sys, err := cfg.Start(t)
 	require.Nil(t, err, "Error starting up system")
 	defer sys.Close()
 
@@ -260,8 +260,8 @@ func testFaultProofProgramScenario(t *testing.T, ctx context.Context, sys *Syste
 	sys.BatchSubmitter.StopIfRunning(context.Background())
 	sys.L2OutputSubmitter.Stop()
 	sys.L2OutputSubmitter = nil
-	for _, node := range sys.Nodes {
-		require.NoError(t, node.Close())
+	for _, node := range sys.EthInstances {
+		node.Close()
 	}
 
 	t.Log("Running fault proof in offline mode")

op-e2e/system_test.go

@@ -4,6 +4,9 @@ import (
 	"context"
 	"fmt"
 	"math/big"
+	"os"
+	"path/filepath"
+	"runtime"
 	"testing"
 	"time"
 
@@ -20,12 +23,14 @@ import (
 	"github.com/ethereum/go-ethereum/node"
 	"github.com/ethereum/go-ethereum/params"
 	"github.com/ethereum/go-ethereum/rpc"
+
 	"github.com/libp2p/go-libp2p/core/peer"
 	"github.com/stretchr/testify/require"
 	"golang.org/x/exp/slices"
 
 	"github.com/ethereum-optimism/optimism/op-bindings/bindings"
 	"github.com/ethereum-optimism/optimism/op-bindings/predeploys"
+	"github.com/ethereum-optimism/optimism/op-e2e/config"
 	"github.com/ethereum-optimism/optimism/op-node/client"
 	"github.com/ethereum-optimism/optimism/op-node/metrics"
 	rollupNode "github.com/ethereum-optimism/optimism/op-node/node"
@@ -39,13 +44,44 @@ import (
 	"github.com/ethereum-optimism/optimism/op-service/retry"
 )
 
+func TestMain(m *testing.M) {
+	if config.ExternalL2Nodes != "" {
+		fmt.Println("Running tests with external L2 process adapter at ", config.ExternalL2Nodes)
+		shimPath, err := filepath.Abs(config.ExternalL2Nodes)
+		if err != nil {
+			fmt.Printf("Could not compute abs of externalL2Nodes shim: %s\n", err)
+			os.Exit(2)
+		}
+		// We convert the passed in path to an absolute path, as it simplifies
+		// the path handling logic for the rest of the testing
+		config.ExternalL2Nodes = shimPath
+
+		_, err = os.Stat(config.ExternalL2Nodes)
+		if err != nil {
+			fmt.Printf("Failed to stat externalL2Nodes path: %s\n", err)
+			os.Exit(3)
+		}
+
+		// As these are integration tests which launch many other processes, the
+		// default parallelism makes the tests flaky.  This change aims to
+		// reduce the flakiness of these tests.
+		maxProcs := runtime.NumCPU() / 4
+		if maxProcs == 0 {
+			maxProcs = 1
+		}
+		runtime.GOMAXPROCS(maxProcs)
+	}
+
+	os.Exit(m.Run())
+}
+
 func TestL2OutputSubmitter(t *testing.T) {
 	InitParallel(t)
 
 	cfg := DefaultSystemConfig(t)
 	cfg.NonFinalizedProposals = true // speed up the time till we see output proposals
 
-	sys, err := cfg.Start()
+	sys, err := cfg.Start(t)
 	require.Nil(t, err, "Error starting up system")
 	defer sys.Close()
 
@@ -115,7 +151,7 @@ func TestSystemE2E(t *testing.T) {
 
 	cfg := DefaultSystemConfig(t)
 
-	sys, err := cfg.Start()
+	sys, err := cfg.Start(t)
 	require.Nil(t, err, "Error starting up system")
 	defer sys.Close()
 
@@ -197,7 +233,7 @@ func TestConfirmationDepth(t *testing.T) {
 	cfg.Nodes["sequencer"].Driver.VerifierConfDepth = 0
 	cfg.Nodes["verifier"].Driver.VerifierConfDepth = verConfDepth
 
-	sys, err := cfg.Start()
+	sys, err := cfg.Start(t)
 	require.Nil(t, err, "Error starting up system")
 	defer sys.Close()
 
@@ -237,6 +273,13 @@ func TestPendingGasLimit(t *testing.T) {
 	InitParallel(t)
 
 	cfg := DefaultSystemConfig(t)
+	if cfg.ExternalL2Nodes != "" {
+		// Some eth clients such as Erigon don't currently build blocks until
+		// they receive the engine call which includes the gas limit.  After we
+		// provide a mechanism for external clients to advertise test support we
+		// should enable for those which support it.
+		t.Skip()
+	}
 
 	// configure the L2 gas limit to be high, and the pending gas limits to be lower for resource saving.
 	cfg.DeployConfig.L2GenesisBlockGasLimit = 30_000_000
@@ -255,7 +298,7 @@ func TestPendingGasLimit(t *testing.T) {
 		},
 	}
 
-	sys, err := cfg.Start()
+	sys, err := cfg.Start(t)
 	require.Nil(t, err, "Error starting up system")
 	defer sys.Close()
 
@@ -297,7 +340,7 @@ func TestFinalize(t *testing.T) {
 
 	cfg := DefaultSystemConfig(t)
 
-	sys, err := cfg.Start()
+	sys, err := cfg.Start(t)
 	require.Nil(t, err, "Error starting up system")
 	defer sys.Close()
 
@@ -321,14 +364,14 @@ func TestMintOnRevertedDeposit(t *testing.T) {
 	InitParallel(t)
 	cfg := DefaultSystemConfig(t)
 
-	sys, err := cfg.Start()
+	sys, err := cfg.Start(t)
 	require.Nil(t, err, "Error starting up system")
 	defer sys.Close()
 
 	l1Client := sys.Clients["l1"]
 	l2Verif := sys.Clients["verifier"]
 
-	l1Node := sys.Nodes["l1"]
+	l1Node := sys.EthInstances["l1"].(*GethInstance).Node
 
 	// create signer
 	ks := l1Node.AccountManager().Backends(keystore.KeyStoreType)[0].(*keystore.KeyStore)
@@ -391,7 +434,7 @@ func TestMissingBatchE2E(t *testing.T) {
 	// Specifically set batch submitter balance to stop batches from being included
 	cfg.Premine[cfg.Secrets.Addresses().Batcher] = big.NewInt(0)
 
-	sys, err := cfg.Start()
+	sys, err := cfg.Start(t)
 	require.Nil(t, err, "Error starting up system")
 	defer sys.Close()
 
@@ -428,8 +471,11 @@ func TestMissingBatchE2E(t *testing.T) {
 	ctx2, cancel := context.WithTimeout(context.Background(), time.Second)
 	defer cancel()
 	block, err := l2Seq.BlockByNumber(ctx2, receipt.BlockNumber)
-	require.Nil(t, err, "Get block from sequencer")
-	require.NotEqual(t, block.Hash(), receipt.BlockHash, "L2 Sequencer did not reorg out transaction on it's safe chain")
+	if err != nil {
+		require.Equal(t, "not found", err.Error(), "A not found error indicates the chain must have re-orged back before it")
+	} else {
+		require.NotEqual(t, block.Hash(), receipt.BlockHash, "L2 Sequencer did not reorg out transaction on it's safe chain")
+	}
 }
 
 func L1InfoFromState(ctx context.Context, contract *bindings.L1Block, l2Number *big.Int) (derive.L1BlockInfo, error) {
@@ -517,7 +563,7 @@ func TestSystemMockP2P(t *testing.T) {
 	cfg.Nodes["sequencer"].Tracer = seqTracer
 	cfg.Nodes["verifier"].Tracer = verifTracer
 
-	sys, err := cfg.Start()
+	sys, err := cfg.Start(t)
 	require.Nil(t, err, "Error starting up system")
 	defer sys.Close()
 
@@ -601,13 +647,12 @@ func TestSystemRPCAltSync(t *testing.T) {
 	cfg.Nodes["sequencer"].Tracer = seqTracer
 	cfg.Nodes["verifier"].Tracer = verifTracer
 
-	sys, err := cfg.Start(SystemConfigOption{
+	sys, err := cfg.Start(t, SystemConfigOption{
 		key:  "afterRollupNodeStart",
 		role: "sequencer",
 		action: func(sCfg *SystemConfig, system *System) {
-			rpc, _ := system.Nodes["sequencer"].Attach() // never errors
 			cfg.Nodes["verifier"].L2Sync = &rollupNode.PreparedL2SyncEndpoint{
-				Client: client.NewBaseRPCClient(rpc),
+				Client: client.NewBaseRPCClient(system.RawClients["sequencer"]),
 			}
 		},
 	})
@@ -685,7 +730,7 @@ func TestSystemP2PAltSync(t *testing.T) {
 	// Blocks are now received via the RPC based alt-sync method
 	cfg.Nodes["sequencer"].Tracer = seqTracer
 
-	sys, err := cfg.Start()
+	sys, err := cfg.Start(t)
 	require.Nil(t, err, "Error starting up system")
 	defer sys.Close()
 
@@ -739,7 +784,7 @@ func TestSystemP2PAltSync(t *testing.T) {
 			},
 		},
 	}
-	configureL1(syncNodeCfg, sys.Nodes["l1"])
+	configureL1(syncNodeCfg, sys.EthInstances["l1"])
 	syncerL2Engine, _, err := initL2Geth("syncer", big.NewInt(int64(cfg.DeployConfig.L2ChainID)), sys.L2GenesisCfg, cfg.JWTFilePath)
 	require.NoError(t, err)
 	require.NoError(t, syncerL2Engine.Start())
@@ -842,7 +887,7 @@ func TestSystemDenseTopology(t *testing.T) {
 	cfg.Nodes["verifier2"].Tracer = verifTracer2
 	cfg.Nodes["verifier3"].Tracer = verifTracer3
 
-	sys, err := cfg.Start()
+	sys, err := cfg.Start(t)
 	require.Nil(t, err, "Error starting up system")
 	defer sys.Close()
 
@@ -882,7 +927,7 @@ func TestL1InfoContract(t *testing.T) {
 
 	cfg := DefaultSystemConfig(t)
 
-	sys, err := cfg.Start()
+	sys, err := cfg.Start(t)
 	require.Nil(t, err, "Error starting up system")
 	defer sys.Close()
 
@@ -990,7 +1035,7 @@ func TestWithdrawals(t *testing.T) {
 	cfg := DefaultSystemConfig(t)
 	cfg.DeployConfig.FinalizationPeriodSeconds = 2 // 2s finalization period
 
-	sys, err := cfg.Start()
+	sys, err := cfg.Start(t)
 	require.Nil(t, err, "Error starting up system")
 	defer sys.Close()
 
@@ -1065,7 +1110,7 @@ func TestWithdrawals(t *testing.T) {
 	startBalance, err = l1Client.BalanceAt(ctx, fromAddr, nil)
 	require.Nil(t, err)
 
-	proveReceipt, finalizeReceipt := ProveAndFinalizeWithdrawal(t, cfg, l1Client, sys.Nodes["verifier"], ethPrivKey, receipt)
+	proveReceipt, finalizeReceipt := ProveAndFinalizeWithdrawal(t, cfg, l1Client, sys.EthInstances["verifier"], ethPrivKey, receipt)
 
 	// Verify balance after withdrawal
 	ctx, cancel = context.WithTimeout(context.Background(), 1*time.Second)
@@ -1084,6 +1129,22 @@ func TestWithdrawals(t *testing.T) {
 	require.Equal(t, withdrawAmount, diff)
 }
 
+type stateGetterAdapter struct {
+	ctx      context.Context
+	t        *testing.T
+	client   *ethclient.Client
+	blockNum *big.Int
+}
+
+func (sga *stateGetterAdapter) GetState(addr common.Address, key common.Hash) common.Hash {
+	sga.t.Helper()
+	val, err := sga.client.StorageAt(sga.ctx, addr, key, sga.blockNum)
+	require.NoError(sga.t, err)
+	var res common.Hash
+	copy(res[:], val)
+	return res
+}
+
 // TestFees checks that L1/L2 fees are handled.
 func TestFees(t *testing.T) {
 	InitParallel(t)
@@ -1093,7 +1154,7 @@ func TestFees(t *testing.T) {
 	cfg.DeployConfig.L2GenesisRegolithTimeOffset = nil
 	cfg.DeployConfig.L1GenesisBlockBaseFeePerGas = (*hexutil.Big)(big.NewInt(7))
 
-	sys, err := cfg.Start()
+	sys, err := cfg.Start(t)
 	require.Nil(t, err, "Error starting up system")
 	defer sys.Close()
 
@@ -1101,11 +1162,21 @@ func TestFees(t *testing.T) {
 	l2Verif := sys.Clients["verifier"]
 	l1 := sys.Clients["l1"]
 
-	seqBackend := sys.Backends["sequencer"]
-	seqState, err := seqBackend.BlockChain().State()
-	require.Nil(t, err, "Error getting sequencer state")
+	config := &params.ChainConfig{
+		Optimism: &params.OptimismConfig{
+			EIP1559Elasticity:  cfg.DeployConfig.EIP1559Elasticity,
+			EIP1559Denominator: cfg.DeployConfig.EIP1559Denominator,
+		},
+		BedrockBlock: big.NewInt(0),
+	}
+
+	sga := &stateGetterAdapter{
+		ctx:    context.Background(),
+		t:      t,
+		client: l2Seq,
+	}
 
-	l1CostFn := types.NewL1CostFunc(seqBackend.BlockChain().Config(), seqState)
+	l1CostFn := types.NewL1CostFunc(config, sga)
 
 	// Transactor Account
 	ethPrivKey := cfg.Secrets.Alice
@@ -1234,7 +1305,7 @@ func TestStopStartBatcher(t *testing.T) {
 	InitParallel(t)
 
 	cfg := DefaultSystemConfig(t)
-	sys, err := cfg.Start()
+	sys, err := cfg.Start(t)
 	require.Nil(t, err, "Error starting up system")
 	defer sys.Close()
 
@@ -1317,7 +1388,7 @@ func TestBatcherMultiTx(t *testing.T) {
 	cfg := DefaultSystemConfig(t)
 	cfg.BatcherTargetL1TxSizeBytes = 2 // ensures that batcher txs are as small as possible
 	cfg.DisableBatcher = true
-	sys, err := cfg.Start()
+	sys, err := cfg.Start(t)
 	require.Nil(t, err, "Error starting up system")
 	defer sys.Close()
 
@@ -1366,7 +1437,7 @@ func TestPendingBlockIsLatest(t *testing.T) {
 	InitParallel(t)
 
 	cfg := DefaultSystemConfig(t)
-	sys, err := cfg.Start()
+	sys, err := cfg.Start(t)
 	require.Nil(t, err, "Error starting up system")
 	defer sys.Close()
 	l2Seq := sys.Clients["sequencer"]

op-e2e/system_tob_test.go

@@ -41,7 +41,7 @@ func TestGasPriceOracleFeeUpdates(t *testing.T) {
 
 	// Create our system configuration for L1/L2 and start it
 	cfg := DefaultSystemConfig(t)
-	sys, err := cfg.Start()
+	sys, err := cfg.Start(t)
 	require.Nil(t, err, "Error starting up system")
 	defer sys.Close()
 
@@ -124,7 +124,7 @@ func TestL2SequencerRPCDepositTx(t *testing.T) {
 
 	// Create our system configuration for L1/L2 and start it
 	cfg := DefaultSystemConfig(t)
-	sys, err := cfg.Start()
+	sys, err := cfg.Start(t)
 	require.Nil(t, err, "Error starting up system")
 	defer sys.Close()
 
@@ -169,7 +169,7 @@ type TestAccount struct {
 // startConfigWithTestAccounts takes a SystemConfig, generates additional accounts, adds them to the config, so they
 // are funded on startup, starts the system, and imports the keys into the keystore, and obtains transaction opts for
 // each account.
-func startConfigWithTestAccounts(cfg *SystemConfig, accountsToGenerate int) (*System, []*TestAccount, error) {
+func startConfigWithTestAccounts(t *testing.T, cfg *SystemConfig, accountsToGenerate int) (*System, []*TestAccount, error) {
 	// Create our test accounts and add them to the pre-mine cfg.
 	testAccounts := make([]*TestAccount, 0)
 	var err error
@@ -211,7 +211,7 @@ func startConfigWithTestAccounts(cfg *SystemConfig, accountsToGenerate int) (*Sy
 	}
 
 	// Start our system
-	sys, err := cfg.Start()
+	sys, err := cfg.Start(t)
 	if err != nil {
 		return sys, nil, err
 	}
@@ -233,7 +233,7 @@ func TestMixedDepositValidity(t *testing.T) {
 
 	// Create our system configuration, funding all accounts we created for L1/L2, and start it
 	cfg := DefaultSystemConfig(t)
-	sys, testAccounts, err := startConfigWithTestAccounts(&cfg, accountUsedToDeposit)
+	sys, testAccounts, err := startConfigWithTestAccounts(t, &cfg, accountUsedToDeposit)
 	require.Nil(t, err, "Error starting up system")
 	defer sys.Close()
 
@@ -400,7 +400,7 @@ func TestMixedWithdrawalValidity(t *testing.T) {
 			cfg.DeployConfig.L2BlockTime = 2
 			require.LessOrEqual(t, cfg.DeployConfig.FinalizationPeriodSeconds, uint64(6))
 			require.Equal(t, cfg.DeployConfig.FundDevAccounts, true)
-			sys, err := cfg.Start()
+			sys, err := cfg.Start(t)
 			require.NoError(t, err, "error starting up system")
 			defer sys.Close()
 
@@ -544,7 +544,7 @@ func TestMixedWithdrawalValidity(t *testing.T) {
 			cancel()
 			require.Nil(t, err)
 
-			rpcClient, err := rpc.Dial(sys.Nodes["verifier"].WSEndpoint())
+			rpcClient, err := rpc.Dial(sys.EthInstances["verifier"].WSEndpoint())
 			require.Nil(t, err)
 			proofCl := gethclient.New(rpcClient)
 			receiptCl := ethclient.NewClient(rpcClient)
@@ -715,7 +715,7 @@ func TestMixedWithdrawalValidity(t *testing.T) {
 			// TODO: Check L1 balance as well here. We avoided this due to time constraints as it seems L1 fees
 			//  were off slightly.
 			_ = endL1Balance
-			//require.Equal(t, transactor.ExpectedL1Balance, endL1Balance, "Unexpected L1 balance for transactor")
+			// require.Equal(t, transactor.ExpectedL1Balance, endL1Balance, "Unexpected L1 balance for transactor")
 			require.Equal(t, transactor.ExpectedL1Nonce, endL1Nonce, "Unexpected L1 nonce for transactor")
 			require.Equal(t, transactor.ExpectedL2Nonce, endL2SeqNonce, "Unexpected L2 sequencer nonce for transactor")
 			require.Equal(t, transactor.ExpectedL2Balance, endL2SeqBalance, "Unexpected L2 sequencer balance for transactor")

op-e2e/withdrawal_helper.go

@@ -17,7 +17,6 @@ import (
 	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/ethereum/go-ethereum/ethclient"
 	"github.com/ethereum/go-ethereum/ethclient/gethclient"
-	"github.com/ethereum/go-ethereum/node"
 	"github.com/ethereum/go-ethereum/rpc"
 	"github.com/stretchr/testify/require"
 )
@@ -79,13 +78,13 @@ func defaultWithdrawalTxOpts() *WithdrawalTxOpts {
 	}
 }
 
-func ProveAndFinalizeWithdrawal(t *testing.T, cfg SystemConfig, l1Client *ethclient.Client, l2Node *node.Node, ethPrivKey *ecdsa.PrivateKey, l2WithdrawalReceipt *types.Receipt) (*types.Receipt, *types.Receipt) {
+func ProveAndFinalizeWithdrawal(t *testing.T, cfg SystemConfig, l1Client *ethclient.Client, l2Node EthInstance, ethPrivKey *ecdsa.PrivateKey, l2WithdrawalReceipt *types.Receipt) (*types.Receipt, *types.Receipt) {
 	params, proveReceipt := ProveWithdrawal(t, cfg, l1Client, l2Node, ethPrivKey, l2WithdrawalReceipt)
 	finalizeReceipt := FinalizeWithdrawal(t, cfg, l1Client, ethPrivKey, proveReceipt, params)
 	return proveReceipt, finalizeReceipt
 }
 
-func ProveWithdrawal(t *testing.T, cfg SystemConfig, l1Client *ethclient.Client, l2Node *node.Node, ethPrivKey *ecdsa.PrivateKey, l2WithdrawalReceipt *types.Receipt) (withdrawals.ProvenWithdrawalParameters, *types.Receipt) {
+func ProveWithdrawal(t *testing.T, cfg SystemConfig, l1Client *ethclient.Client, l2Node EthInstance, ethPrivKey *ecdsa.PrivateKey, l2WithdrawalReceipt *types.Receipt) (withdrawals.ProvenWithdrawalParameters, *types.Receipt) {
 	// Get l2BlockNumber for proof generation
 	ctx, cancel := context.WithTimeout(context.Background(), 40*time.Duration(cfg.DeployConfig.L1BlockTime)*time.Second)
 	defer cancel()

packages/contracts-bedrock/scripts/differential-testing/differential-testing.go

@@ -277,7 +277,8 @@ func main() {
 		checkErr(err, "Error creating secure trie")
 
 		// Put a "true" bool in the storage slot
-		state.UpdateStorage(common.Address{}, hash.Bytes(), []byte{0x01})
+		err = state.UpdateStorage(common.Address{}, hash.Bytes(), []byte{0x01})
+		checkErr(err, "Error updating storage")
 
 		// Create a secure trie for the world state
 		world, err := trie.NewStateTrie(
@@ -294,7 +295,8 @@ func main() {
 		}
 		writer := new(bytes.Buffer)
 		checkErr(account.EncodeRLP(writer), "Error encoding account")
-		world.UpdateStorage(common.Address{}, predeploys.L2ToL1MessagePasserAddr.Bytes(), writer.Bytes())
+		err = world.UpdateStorage(common.Address{}, predeploys.L2ToL1MessagePasserAddr.Bytes(), writer.Bytes())
+		checkErr(err, "Error updating storage")
 
 		// Get the proof
 		var proof proofList