ci: Convert all Go tests into a single job (#13046)

* ci: Convert all Go tests into a single job

Takes all the go-e2e and go-test jobs and combines them into one single super-job. This has the following benefits:

1. We can centralize all config for Go tests in one place and simplify our CI pipeline.
2. It leaves parallelism up to the Go runtime, which is more efficient.
4. It makes our test reporting more accurate.

As part of this PR, I also:

1. Created a new resource class just for Go tests. It runs on the large Latitude runner, but limits the number of parallel go test jobs to 3 to prevent resource starvation.
2. Enabled test retries on `gotestsum`. This should reduce the number of flakes we see from the Go tests by retrying them up to 2 times in the event that they fail. Failures are still captured by the test reporting, so we can address them.

* code review updates

.circleci/config.yml

@@ -867,60 +867,8 @@ jobs:
           path: tmp/testlogs
           when: always
 
-  go-test:
-    machine: true
-    resource_class: ethereum-optimism/latitude-1
-    steps:
-      - checkout
-      - attach_workspace:
-          at: "."
-      - run:
-          name: run tests
-          no_output_timeout: 5m
-          command: |
-            mkdir -p ./tmp/test-results && mkdir -p ./tmp/testlogs
-
-            packages=(
-              op-batcher
-              op-chain-ops
-              op-node
-              op-proposer
-              op-challenger
-              op-dispute-mon
-              op-conductor
-              op-program
-              op-service
-              op-supervisor
-              op-deployer
-            )
-            formatted_packages=""
-            for package in "${packages[@]}"; do
-              formatted_packages="$formatted_packages ./$package/..."
-            done
-
-            export ENABLE_ANVIL=true
-            export SEPOLIA_RPC_URL="https://ci-sepolia-l1-archive.optimism.io"
-            export MAINNET_RPC_URL="https://ci-mainnet-l1-archive.optimism.io"
-
-            gotestsum --format=testname \
-              --junitfile=./tmp/test-results/results.xml \
-              --jsonfile=./tmp/testlogs/log.json \
-              -- -coverpkg=github.com/ethereum-optimism/optimism/... \
-              -coverprofile=coverage.out $formatted_packages
-      - store_test_results:
-          path: ./tmp/test-results
-      - store_artifacts:
-          path: ./tmp/testlogs
-          when: always
-
-  go-e2e-test:
+  go-tests:
     parameters:
-      module:
-        description: Go Module Name
-        type: string
-      target:
-        description: The make target to execute
-        type: string
       notify:
         description: Whether to notify on failure
         type: boolean
@@ -932,56 +880,60 @@ jobs:
       resource_class:
         description: Machine resource class
         type: string
-        default: ethereum-optimism/latitude-1
-      skip_slow_tests:
-        description: Indicates that slow tests should be skipped
-        type: boolean
-        default: false
+        default: ethereum-optimism/latitude-1-go-e2e
       no_output_timeout:
         description: Timeout for running tests
         type: string
         default: 5m
+      environment_overrides:
+        description: Environment overrides
+        type: string
+        default: ""
+      packages:
+        description: List of packages to test
+        type: string
     machine: true
     resource_class: <<parameters.resource_class>>
     steps:
       - checkout
       - attach_workspace:
-          at: ./tmp/workspace
-      - run:
-          name: Load devnet-allocs and artifacts
-          command: |
-            mkdir -p .devnet
-            cp -r ./tmp/workspace/.devnet* .
-            cp -r ./tmp/workspace/packages/contracts-bedrock/forge-artifacts packages/contracts-bedrock/forge-artifacts
-            cp ./tmp/workspace/packages/contracts-bedrock/deploy-config/devnetL1.json packages/contracts-bedrock/deploy-config/devnetL1.json
-            cp -r ./tmp/workspace/packages/contracts-bedrock/deployments/devnetL1 packages/contracts-bedrock/deployments/devnetL1
-      - run:
-          name: print go's available MIPS targets
-          command: go tool dist list | grep mips
+          at: "."
       - run:
           name: run tests
           no_output_timeout: <<parameters.no_output_timeout>>
           command: |
-            mkdir -p ./tmp/testlogs
-            mkdir -p ./tmp/test-results
+            mkdir -p ./tmp/test-results && mkdir -p ./tmp/testlogs
+            
+            cd op-e2e && make pre-test && cd ..
+
+            packages=(
+              <<parameters.packages>>
+            )
+            formatted_packages=""
+            for package in "${packages[@]}"; do
+              formatted_packages="$formatted_packages ./$package/..."
+            done
 
-            # The below env var gets overridden when running make test-cannon, but we
-            # need to explicitly set it here to prevent Cannon from running when we don't
-            # want it to.
             export OP_E2E_CANNON_ENABLED="false"
-            export OP_E2E_SKIP_SLOW_TEST=<<parameters.skip_slow_tests>>
-            # Note: We don't use circle CI test splits because we need to split by test name, not by package. There is an additional
-            # constraint that gotestsum does not currently (nor likely will) accept files from different packages when building.
-            JUNIT_FILE=../tmp/test-results/<<parameters.module>>_<<parameters.target>>.xml JSON_LOG_FILE=../tmp/testlogs/test.log make <<parameters.target>>
-          working_directory: <<parameters.module>>
+            export OP_E2E_SKIP_SLOW_TEST=true
+            export OP_E2E_USE_HTTP=true
+            export ENABLE_ANVIL=true
+            export SEPOLIA_RPC_URL="https://ci-sepolia-l1-archive.optimism.io"
+            export MAINNET_RPC_URL="https://ci-mainnet-l1-archive.optimism.io"
+            
+            <<parameters.environment_overrides>>
+
+            gotestsum --format=testname \
+              --junitfile=./tmp/test-results/results.xml \
+              --jsonfile=./tmp/testlogs/log.json \
+              --rerun-fails=2 \
+              --packages="$formatted_packages" \
+              -- -coverprofile=coverage.out
+      - store_test_results:
+          path: ./tmp/test-results
       - store_artifacts:
           path: ./tmp/testlogs
           when: always
-      - store_artifacts:
-          path: ./tmp/test-results
-          when: always
-      - store_test_results:
-          path: ./tmp/test-results
       - when:
           condition: "<<parameters.notify>>"
           steps:
@@ -1389,33 +1341,31 @@ workflows:
           on_changes: op-e2e,packages/contracts-bedrock/src
           uses_artifacts: true
           requires: ["contracts-bedrock-build"]
-      - go-test:
-          name: go-test-all
-          requires:
-            - contracts-bedrock-build
       - go-test-kurtosis:
           name: op-deployer-integration
           module: op-deployer
           test_directory: ./pkg/deployer/integration_test
           uses_artifacts: true
           requires: ["contracts-bedrock-build"]
-      - go-e2e-test:
-          name: op-e2e-HTTP-tests
-          module: op-e2e
-          target: test-http
-          requires:
-            - contracts-bedrock-build
-      - go-e2e-test:
-          name: op-e2e-action-tests
-          module: op-e2e
-          target: test-actions
-          requires:
-            - contracts-bedrock-build
-      - go-e2e-test:
-          name: op-e2e-fault-proof-tests
-          module: op-e2e
-          target: test-fault-proofs
-          skip_slow_tests: true
+      - go-tests:
+          packages: |
+            op-batcher
+            op-chain-ops
+            op-node
+            op-proposer
+            op-challenger
+            op-dispute-mon
+            op-conductor
+            op-program
+            op-service
+            op-supervisor
+            op-deployer
+            op-e2e/system
+            op-e2e/e2eutils
+            op-e2e/opgeth
+            op-e2e/interop
+            op-e2e/actions
+            op-e2e/faultproofs
           requires:
             - contracts-bedrock-build
             - cannon-prestate
@@ -1432,16 +1382,13 @@ workflows:
             - go-mod-download
             - op-deployer-integration
             - op-program-compat
-            - op-e2e-HTTP-tests
-            - op-e2e-fault-proof-tests
-            - op-e2e-action-tests
             # Not needed for the devnet but we want to make sure they build successfully
             - cannon-docker-build
             - op-dispute-mon-docker-build
             - op-program-docker-build
             - op-supervisor-docker-build
             - proofs-tools-docker-build
-            - go-test-all
+            - go-tests
       - docker-build:
           name: <<matrix.docker_name>>-docker-build
           docker_tags: <<pipeline.git.revision>>,<<pipeline.git.branch>>
@@ -1635,17 +1582,16 @@ workflows:
           skip_pattern: test
           context:
             - slack
-      - go-e2e-test:
+      - go-tests:
           name: op-e2e-cannon-tests
-          module: op-e2e
-          target: test-cannon
           notify: true
-          no_output_timeout: 20m
           mentions: "@proofs-team"
+          no_output_timeout: 20m
           resource_class: ethereum-optimism/latitude-fps-1
-          requires:
-            - contracts-bedrock-build
-            - cannon-prestate
+          environment_overrides: |
+            export OP_E2E_CANNON_ENABLED="true"
+          packages: |
+            op-e2e/faultproofs
           context:
             - slack