ci: Port some CI jobs to self-hosted runners (#12199)

* ci: Port some CI jobs to self-hosted runners

This PR ports the following CI jobs to use self-hosted runners:

- cannon-go-lint-and-test
- contracts-bedrock-build
- go-lint
- go-test-kurtosis
- go-e2e-test
- cannon-prestate

These jobs benefit from running on beefier hardware. Since Go cache data is shared among the executors, it also allowed me to remove a lot of the manual caching logic from our builds while  improving Go compilation times overall.

I'll provide a separate writeup detailing exactly how the executors work.

* use test result directories in the workspace

.circleci/config.yml

@@ -152,8 +152,8 @@ commands:
 
 jobs:
   cannon-go-lint-and-test:
-    docker:
-      - image: <<pipeline.parameters.ci_builder_image>>
+    machine: true
+    resource_class: ethereum-optimism/latitude-1
     parameters:
       skip_slow_tests:
         type: boolean
@@ -165,19 +165,17 @@ jobs:
       mips64:
         type: boolean
         default: false
-    resource_class: xlarge
     steps:
       - checkout
       - check-changed:
           patterns: cannon,packages/contracts-bedrock/src/cannon,op-preimage,go.mod
       - attach_workspace:
           at: "."
-      - restore_cache:
-          name: Restore Go modules cache
-          key: gomod-{{ checksum "go.sum" }}
       - run:
           name: prep Cannon results dir
-          command: mkdir -p /tmp/test-results
+          command: |
+            mkdir -p ./tmp/test-results
+            mkdir -p ./tmp/testlogs
       - run:
           name: build Cannon example binaries
           command: make elf # only compile ELF binaries with Go, we do not have MIPS GCC for creating the debug-dumps.
@@ -195,9 +193,8 @@ jobs:
                 name: Cannon Go 32-bit tests
                 command: |
                   export SKIP_SLOW_TESTS=<<parameters.skip_slow_tests>>
-                  mkdir -p /testlogs
-                  gotestsum --format=testname --junitfile=/tmp/test-results/cannon.xml --jsonfile=/testlogs/log.json \
-                  -- -parallel=8 -coverpkg=github.com/ethereum-optimism/optimism/cannon/... -coverprofile=coverage.out ./...
+                  gotestsum --format=testname --junitfile=../tmp/test-results/cannon.xml --jsonfile=../tmp/testlogs/log.json \
+                  -- -parallel=$(nproc) -coverpkg=github.com/ethereum-optimism/optimism/cannon/... -coverprofile=coverage.out ./...
                 working_directory: cannon
       - when:
           condition: <<parameters.mips64>>
@@ -206,17 +203,16 @@ jobs:
                 name: Cannon Go 64-bit tests
                 command: |
                   export SKIP_SLOW_TESTS=<<parameters.skip_slow_tests>>
-                  mkdir -p /testlogs
-                  gotestsum --format=testname --junitfile=/tmp/test-results/cannon.xml --jsonfile=/testlogs/log.json \
-                  -- --tags=cannon64 -parallel=8 -coverpkg=github.com/ethereum-optimism/optimism/cannon/... -coverprofile=coverage.out ./...
+                  gotestsum --format=testname --junitfile=../tmp/test-results/cannon.xml --jsonfile=../tmp/testlogs/log.json \
+                  -- --tags=cannon64 -parallel=$(nproc) -coverpkg=github.com/ethereum-optimism/optimism/cannon/... -coverprofile=coverage.out ./...
                 working_directory: cannon
       - run:
           name: upload Cannon coverage
           command: codecov --verbose --clean --flags cannon-go-tests
       - store_test_results:
-          path: /tmp/test-results
+          path: ./tmp/test-results
       - store_artifacts:
-          path: /testlogs
+          path: ./tmp/testlogs
           when: always
       - when:
           condition: <<parameters.notify>>
@@ -238,9 +234,8 @@ jobs:
           working_directory: cannon/mipsevm/tests/open_mips_tests
 
   contracts-bedrock-build:
-    docker:
-      - image: <<pipeline.parameters.ci_builder_image>>
-    resource_class: xlarge
+    machine: true
+    resource_class: ethereum-optimism/latitude-1
     parameters:
       skip_pattern:
         description: Glob pattern of tests to skip
@@ -249,16 +244,6 @@ jobs:
     steps:
       - checkout
       - install-contracts-dependencies
-      - restore_cache:
-          name: Restore Go modules cache
-          keys:
-            - gomod-contracts-build-{{ checksum "go.sum" }}
-            - gomod-contracts-build-
-      - restore_cache:
-          name: Restore Go build cache
-          keys:
-            - golang-build-cache-contracts-build-{{ checksum "go.sum" }}
-            - golang-build-cache-contracts-build-
       - run:
           name: Print forge version
           command: forge --version
@@ -276,16 +261,6 @@ jobs:
           name: Generate allocs
           command: |
             make devnet-allocs
-      - save_cache:
-          name: Save Go modules cache
-          key: gomod-contracts-build-{{ checksum "go.sum" }}
-          paths:
-            - "/go/pkg/mod"
-      - save_cache:
-          name: Save Go build cache
-          key: golang-build-cache-contracts-build-{{ checksum "go.sum" }}
-          paths:
-            - "/root/.cache/go-build"
       - persist_to_workspace:
           root: "."
           paths:
@@ -798,40 +773,15 @@ jobs:
             - "/root/.cache/go-build"
 
   go-lint:
-    docker:
-      - image: <<pipeline.parameters.ci_builder_image>>
+    machine: true
+    resource_class: ethereum-optimism/latitude-1
     steps:
       - checkout
-      - restore_cache:
-          name: Restore Go modules cache
-          key: gomod-{{ checksum "go.sum" }}
-      - restore_cache:
-          name: Restore Go build cache
-          keys:
-            - golang-build-cache-lint-{{ checksum "go.sum" }}
-            - golang-build-cache-lint-
-      - restore_cache:
-          name: Restore Go lint cache
-          keys:
-            - golang-lint-cache-{{ checksum "go.sum" }}
-            - golang-lint-cache-
       - run:
           name: run Go linter
           command: |
-            # Identify how many cores it defaults to
-            golangci-lint run --help | grep concurrency
             make lint-go
           working_directory: .
-      - save_cache:
-          name: Save Go build cache
-          key: golang-build-cache-lint-{{ checksum "go.sum" }}
-          paths:
-            - "/root/.cache/go-build"
-      - save_cache:
-          name: Save Go lint cache
-          key: golang-lint-cache-{{ checksum "go.sum" }}
-          paths:
-            - "/root/.cache/golangci-lint"
 
   go-test-kurtosis:
     parameters:
@@ -846,22 +796,9 @@ jobs:
         description: Test directory
         type: string
         default: "./..."
-    machine:
-      image: <<pipeline.parameters.base_image>> # only used to enable codecov.
-    resource_class: xlarge
+    machine: true
+    resource_class: ethereum-optimism/latitude-1
     steps:
-      - run:
-          name: Install components
-          command: |
-            go version
-            go install gotest.tools/gotestsum@v1.11.0
-      - run:
-          name: Install Kurtosis
-          command: |
-            echo "deb [trusted=yes] https://apt.fury.io/kurtosis-tech/ /" | sudo tee /etc/apt/sources.list.d/kurtosis.list
-            sudo apt update
-            sudo apt install kurtosis-cli=1.3.0
-            kurtosis engine start
       - checkout
       - when:
           condition: <<parameters.uses_artifacts>>
@@ -871,24 +808,23 @@ jobs:
           name: prep results dir
           command: |
             # Make sure the workspace is properly owned
-            sudo chown -R $USER:$USER .
-            mkdir -p /tmp/test-results
-            mkdir -p /tmp/testlogs
+            mkdir -p ./tmp/test-results
+            mkdir -p ./tmp/testlogs
       - run:
           name: run tests
           command: |
             ENABLE_KURTOSIS=true gotestsum \
               --format=testname \
-              --junitfile=/tmp/test-results/<<parameters.module>>.xml \
-              --jsonfile=/tmp/testlogs/log.json \
-              -- -parallel=8 \
+              --junitfile=../tmp/test-results/<<parameters.module>>.xml \
+              --jsonfile=../tmp/testlogs/log.json \
+              -- -parallel=$(nproc) \
               -coverpkg=github.com/ethereum-optimism/optimism/... \
               -coverprofile=coverage.out <<parameters.test_directory>>
           working_directory: <<parameters.module>>
       - store_test_results:
-          path: /tmp/test-results
+          path: tmp/test-results
       - store_artifacts:
-          path: /tmp/testlogs
+          path: tmp/testlogs
           when: always
 
   go-test:
@@ -951,10 +887,6 @@ jobs:
       target:
         description: The make target to execute
         type: string
-      parallelism:
-        description: Number of parallel test runs
-        type: integer
-        default: 6
       notify:
         description: Whether to notify on failure
         type: boolean
@@ -963,35 +895,24 @@ jobs:
         description: Slack user or group to mention when notifying of failures
         type: string
         default: ""
-    docker:
-      - image: <<pipeline.parameters.ci_builder_image>>
-    resource_class: xlarge
-    parallelism: <<parameters.parallelism>>
+      resource_class:
+        description: Machine resource class
+        type: string
+        default: ethereum-optimism/latitude-1
+    machine: true
+    resource_class: <<parameters.resource_class>>
     steps:
       - checkout
-      - check-changed:
-          patterns: op-(.+),cannon,contracts-bedrock
-      - run:
-          name: prep results dir
-          command: mkdir -p /tmp/test-results
-      - restore_cache:
-          name: Restore Go modules cache
-          key: gomod-{{ checksum "go.sum" }}
-      - restore_cache:
-          name: Restore Go build cache
-          keys:
-            - golang-build-cache-e2e-{{ checksum "go.sum" }}
-            - golang-build-cache-e2e-
       - attach_workspace:
-          at: /tmp/workspace
+          at: ./tmp/workspace
       - run:
           name: Load devnet-allocs and artifacts
           command: |
             mkdir -p .devnet
-            cp -r /tmp/workspace/.devnet* .
-            cp -r /tmp/workspace/packages/contracts-bedrock/forge-artifacts packages/contracts-bedrock/forge-artifacts
-            cp /tmp/workspace/packages/contracts-bedrock/deploy-config/devnetL1.json packages/contracts-bedrock/deploy-config/devnetL1.json
-            cp -r /tmp/workspace/packages/contracts-bedrock/deployments/devnetL1 packages/contracts-bedrock/deployments/devnetL1
+            cp -r ./tmp/workspace/.devnet* .
+            cp -r ./tmp/workspace/packages/contracts-bedrock/forge-artifacts packages/contracts-bedrock/forge-artifacts
+            cp ./tmp/workspace/packages/contracts-bedrock/deploy-config/devnetL1.json packages/contracts-bedrock/deploy-config/devnetL1.json
+            cp -r ./tmp/workspace/packages/contracts-bedrock/deployments/devnetL1 packages/contracts-bedrock/deployments/devnetL1
       - run:
           name: print go's available MIPS targets
           command: go tool dist list | grep mips
@@ -999,7 +920,8 @@ jobs:
           name: run tests
           no_output_timeout: 20m
           command: |
-            mkdir -p /testlogs
+            mkdir -p ./tmp/testlogs
+            mkdir -p ./tmp/test-results
 
             # The below env var gets overridden when running make test-cannon, but we
             # need to explicitly set it here to prevent Cannon from running when we don't
@@ -1007,21 +929,16 @@ jobs:
             export OP_E2E_CANNON_ENABLED="false"
             # Note: We don't use circle CI test splits because we need to split by test name, not by package. There is an additional
             # constraint that gotestsum does not currently (nor likely will) accept files from different packages when building.
-            JUNIT_FILE=/tmp/test-results/<<parameters.module>>_<<parameters.target>>.xml JSON_LOG_FILE=/testlogs/test.log make <<parameters.target>>
+            JUNIT_FILE=../tmp/test-results/<<parameters.module>>_<<parameters.target>>.xml JSON_LOG_FILE=../tmp/testlogs/test.log make <<parameters.target>>
           working_directory: <<parameters.module>>
       - store_artifacts:
-          path: /testlogs
+          path: ./tmp/testlogs
           when: always
       - store_artifacts:
-          path: /tmp/test-results
+          path: ./tmp/test-results
           when: always
       - store_test_results:
-          path: /tmp/test-results
-      - save_cache:
-          name: Save Go build cache
-          key: golang-build-cache-e2e-{{ checksum "go.sum" }}
-          paths:
-            - "/root/.cache/go-build"
+          path: ./tmp/test-results
       - when:
           condition: "<<parameters.notify>>"
           steps:
@@ -1029,18 +946,10 @@ jobs:
                 mentions: "<<parameters.mentions>>"
 
   cannon-prestate:
-    docker:
-      - image: <<pipeline.parameters.ci_builder_image>>
+    machine: true
+    resource_class: ethereum-optimism/latitude-1
     steps:
       - checkout
-      - restore_cache:
-          name: Restore Go modules cache
-          key: gomod-{{ checksum "go.sum" }}
-      - restore_cache:
-          name: Restore Go build cache
-          keys:
-            - golang-build-cache-cannon-prestate-{{ checksum "go.sum" }}
-            - golang-build-cache-cannon-prestate-
       - run:
           name: Build cannon
           command: make cannon
@@ -1073,11 +982,6 @@ jobs:
             - "op-program/bin/prestate-mt.json"
             - "op-program/bin/meta-mt.json"
             - "op-program/bin/prestate-proof-mt.json"
-      - save_cache:
-          name: Save Go build cache
-          key: golang-build-cache-cannon-prestate-{{ checksum "go.sum" }}
-          paths:
-            - "/root/.cache/go-build"
       - persist_to_workspace:
           root: .
           paths:
@@ -1384,9 +1288,9 @@ workflows:
       and:
         - or:
             # Trigger on new commits
-          - equal: [ webhook, << pipeline.trigger_source >> ]
-            # Trigger on manual triggers if explicitly requested
-          - equal: [ true, << pipeline.parameters.main_dispatch >> ]
+            - equal: [ webhook, << pipeline.trigger_source >> ]
+              # Trigger on manual triggers if explicitly requested
+            - equal: [ true, << pipeline.parameters.main_dispatch >> ]
         - not:
             equal: [ scheduled_pipeline, << pipeline.trigger_source >> ]
     jobs:
@@ -1425,9 +1329,7 @@ workflows:
       - semgrep-scan:
           name: semgrep-scan-local
           scan_command: semgrep scan --timeout=100 --config=./.semgrep --error .
-      - go-lint:
-          requires:
-            - go-mod-download
+      - go-lint
       - fuzz-golang:
           name: fuzz-golang-<<matrix.package_name>>
           requires:
@@ -1488,23 +1390,18 @@ workflows:
           name: op-e2e-HTTP-tests
           module: op-e2e
           target: test-http
-          parallelism: 8
           requires:
-            - go-mod-download
             - contracts-bedrock-build
       - go-e2e-test:
           name: op-e2e-action-tests
           module: op-e2e
           target: test-actions
-          parallelism: 1
           requires:
-            - go-mod-download
             - contracts-bedrock-build
       - go-e2e-test:
           name: op-e2e-fault-proof-tests
           module: op-e2e
           target: test-fault-proofs
-          parallelism: 4
           requires:
             - contracts-bedrock-build
             - cannon-prestate
@@ -1560,9 +1457,7 @@ workflows:
                 - op-supervisor
                 - op-deployer
                 - cannon
-      - cannon-prestate:
-          requires:
-            - go-mod-download
+      - cannon-prestate
       - check-generated-mocks-op-node
       - check-generated-mocks-op-service
       - cannon-go-lint-and-test:
@@ -1712,9 +1607,7 @@ workflows:
             equal: [ scheduled_pipeline, << pipeline.trigger_source >> ]
     jobs:
       - go-mod-download
-      - cannon-prestate:
-          requires:
-            - go-mod-download
+      - cannon-prestate
       - cannon-stf-verify:
           requires:
             - go-mod-download
@@ -1728,7 +1621,6 @@ workflows:
           name: op-e2e-cannon-tests
           module: op-e2e
           target: test-cannon
-          parallelism: 8
           notify: true
           mentions: "@proofs-team"
           requires:

op-chain-ops/deployer/integration_test/apply_test.go

@@ -13,6 +13,8 @@ import (
 	"testing"
 	"time"
 
+	op_e2e "github.com/ethereum-optimism/optimism/op-e2e"
+
 	"github.com/ethereum-optimism/optimism/op-chain-ops/deployer"
 	"github.com/holiman/uint256"
 
@@ -66,6 +68,7 @@ func (d *deployerKey) String() string {
 }
 
 func TestEndToEndApply(t *testing.T) {
+	op_e2e.InitParallel(t)
 	kurtosisutil.Test(t)
 
 	lgr := testlog.Logger(t, slog.LevelDebug)
@@ -363,6 +366,7 @@ func TestApplyExistingOPCM(t *testing.T) {
 }
 
 func TestL2BlockTimeOverride(t *testing.T) {
+	op_e2e.InitParallel(t)
 	kurtosisutil.Test(t)
 
 	lgr := testlog.Logger(t, slog.LevelDebug)

op-e2e/Makefile

+num_cores := $(shell nproc)
+
 # Generally, JUNIT_FILE is set in CI but may be specified to an arbitrary file location to emulate CI locally
 # If JUNIT_FILE is set, JSON_LOG_FILE should also be set
 ifdef JUNIT_FILE
 	go_test = OP_TESTLOG_DISABLE_COLOR=true OP_E2E_DISABLE_PARALLEL=false gotestsum --format=testname --junitfile=$(JUNIT_FILE) --jsonfile=$(JSON_LOG_FILE) -- -failfast
   # Note: -parallel must be set to match the number of cores in the resource class
-	go_test_flags = -timeout=60m -parallel=8
+	go_test_flags = -timeout=60m -parallel=$(num_cores)
 else
 	go_test = go test
 	go_test_flags = -v