ci: Phase 1 and 2 CI mitigations (#9304)

Implements items 1 and 2 on the [CI Fixes Plan](https://www.notion.so/oplabs/CI-Fixes-Plan-c7b3fee803c1480eb8e0512bff51e208):

1. Disables `op-e2e-ws-tests`
2. Fails faster on HTTP tests

.circleci/config.yml

@@ -812,10 +812,10 @@ jobs:
       target:
         description: The make target to execute
         type: string
-      cannon_enabled:
-        description: Whether to include cannon tests
-        default: true
-        type: boolean
+      parallelism:
+        description: Number of parallel test runs
+        type: integer
+        default: 6
     docker:
       - image: <<pipeline.parameters.ci_builder_image>>
     resource_class: xlarge
@@ -824,7 +824,7 @@ jobs:
     # InitParallel(t, UseExecutor(1))
     # Any tests assigned to an executor greater than the number available automatically use the last executor.
     # Executor indexes start from 0
-    parallelism: 2
+    parallelism: <<parameters.parallelism>>
     steps:
       - checkout
       - check-changed:
@@ -851,7 +851,10 @@ jobs:
           command: |
             mkdir -p /testlogs
 
-            export OP_E2E_CANNON_ENABLED="<<parameters.cannon_enabled>>"
+            # The below env var gets overridden when running make test-cannon, but we
+            # need to explicitly set it here to prevent Cannon from running when we don't
+            # want it to.
+            export OP_E2E_CANNON_ENABLED="false"
             # Note: We don't use circle CI test splits because we need to split by test name, not by package. There is an additional
             # constraint that gotestsum does not currently (nor likely will) accept files from different pacakges when building.
             JUNIT_FILE=/tmp/test-results/<<parameters.module>>_<<parameters.target>>.xml make <<parameters.target>> 2>&1 | tee /testlogs/test.log
@@ -1501,25 +1504,34 @@ workflows:
           requires:
             - op-stack-go-lint
       - go-e2e-test:
-          name: op-e2e-WS-tests
+          name: op-e2e-HTTP-tests
           module: op-e2e
-          target: test-ws
+          target: test-http
+          parallelism: 4
           requires:
             - op-stack-go-lint
             - devnet-allocs
       - go-e2e-test:
-          name: op-e2e-HTTP-tests
+          name: op-e2e-action-tests
           module: op-e2e
-          target: test-http
-          cannon_enabled: false
+          target: test-actions
+          parallelism: 1
           requires:
             - op-stack-go-lint
             - devnet-allocs
+      - go-e2e-test:
+          name: op-e2e-cannon-tests
+          module: op-e2e
+          target: test-cannon
+          parallelism: 8
+          requires:
+            - op-stack-go-lint
+            - devnet-allocs
+            - cannon-prestate
       - go-e2e-test:
           name: op-e2e-ext-geth-tests
           module: op-e2e
           target: test-external-geth
-          cannon_enabled: false
           requires:
             - op-stack-go-lint
             - devnet-allocs
@@ -1547,8 +1559,9 @@ workflows:
             - op-program-tests
             - op-program-compat
             - op-service-tests
-            - op-e2e-WS-tests
             - op-e2e-HTTP-tests
+            - op-e2e-cannon-tests
+            - op-e2e-action-tests
             - op-e2e-ext-geth-tests
             - op-service-rethdb-tests
       - docker-build: # just to warm up the cache (other jobs run in parallel)

op-e2e/Makefile

 # Generally, JUNIT_FILE is set in CI but may be specified to an arbitrary file location to emulate CI locally
 ifdef JUNIT_FILE
-	go_test = OP_TESTLOG_DISABLE_COLOR=true OP_E2E_DISABLE_PARALLEL=false gotestsum --format=standard-verbose --junitfile=$(JUNIT_FILE) --
+	go_test = OP_TESTLOG_DISABLE_COLOR=true OP_E2E_DISABLE_PARALLEL=false gotestsum --format=standard-verbose --junitfile=$(JUNIT_FILE) -- -failfast
   # Note: -parallel must be set to match the number of cores in the resource class
 	go_test_flags = -timeout=20m -parallel=8
 else
@@ -16,13 +16,20 @@ test-external-%: pre-test
 	$(go_test) $(go_test_flags) --externalL2 ./external_$*/
 
 test-ws: pre-test
-	$(go_test) $(go_test_flags) ./...
+	$(go_test) $(go_test_flags) . ./e2eutils/...
 .PHONY: test-ws
 
+test-actions: pre-test
+	$(go_test) $(go_test_flags) ./actions
+.PHONY: test-actions
+
 test-http: pre-test
-	OP_E2E_USE_HTTP=true $(go_test) $(go_test_flags) ./...
+	OP_E2E_USE_HTTP=true $(go_test) $(go_test_flags) . ./e2eutils/...
 .PHONY: test-http
 
+test-cannon: pre-test
+	OP_E2E_CANNON_ENABLED=true $(go_test) $(go_test_flags) ./faultproofs
+
 cannon-prestate:
 	make -C .. cannon-prestate
 .PHONY: cannon-prestate

op-e2e/faultproofs/output_cannon_test.go

@@ -15,10 +15,8 @@ import (
 	"github.com/stretchr/testify/require"
 )
 
-const outputCannonTestExecutor = 0
-
 func TestOutputCannonGame(t *testing.T) {
-	op_e2e.InitParallel(t, op_e2e.UsesCannon, op_e2e.UseExecutor(outputCannonTestExecutor))
+	op_e2e.InitParallel(t, op_e2e.UsesCannon, op_e2e.UseExecutor(0))
 	ctx := context.Background()
 	sys, l1Client := startFaultDisputeSystem(t)
 	t.Cleanup(sys.Close)
@@ -74,7 +72,7 @@ func TestOutputCannonGame(t *testing.T) {
 
 func TestOutputCannon_ChallengeAllZeroClaim(t *testing.T) {
 	// The dishonest actor always posts claims with all zeros.
-	op_e2e.InitParallel(t, op_e2e.UsesCannon, op_e2e.UseExecutor(outputCannonTestExecutor))
+	op_e2e.InitParallel(t, op_e2e.UsesCannon, op_e2e.UseExecutor(1))
 	ctx := context.Background()
 	sys, l1Client := startFaultDisputeSystem(t)
 	t.Cleanup(sys.Close)
@@ -102,7 +100,6 @@ func TestOutputCannon_ChallengeAllZeroClaim(t *testing.T) {
 }
 
 func TestOutputCannon_PublishCannonRootClaim(t *testing.T) {
-	op_e2e.InitParallel(t, op_e2e.UsesCannon, op_e2e.UseExecutor(outputCannonTestExecutor))
 	tests := []struct {
 		disputeL2BlockNumber uint64
 	}{
@@ -112,7 +109,8 @@ func TestOutputCannon_PublishCannonRootClaim(t *testing.T) {
 	for _, test := range tests {
 		test := test
 		t.Run(fmt.Sprintf("Dispute_%v", test.disputeL2BlockNumber), func(t *testing.T) {
-			op_e2e.InitParallel(t, op_e2e.UsesCannon, op_e2e.UseExecutor(outputCannonTestExecutor))
+			op_e2e.InitParallel(t, op_e2e.UsesCannon, op_e2e.UseExecutor(2))
+
 			ctx := context.Background()
 			sys, _ := startFaultDisputeSystem(t)
 
@@ -130,9 +128,6 @@ func TestOutputCannon_PublishCannonRootClaim(t *testing.T) {
 }
 
 func TestOutputCannonDisputeGame(t *testing.T) {
-	executor := uint64(1) // Different executor to the other tests to help balance things better
-	op_e2e.InitParallel(t, op_e2e.UsesCannon, op_e2e.UseExecutor(executor))
-
 	tests := []struct {
 		name             string
 		defendClaimDepth types.Depth
@@ -144,7 +139,7 @@ func TestOutputCannonDisputeGame(t *testing.T) {
 	for _, test := range tests {
 		test := test
 		t.Run(test.name, func(t *testing.T) {
-			op_e2e.InitParallel(t, op_e2e.UseExecutor(executor))
+			op_e2e.InitParallel(t, op_e2e.UsesCannon, op_e2e.UseExecutor(3))
 
 			ctx := context.Background()
 			sys, l1Client := startFaultDisputeSystem(t)
@@ -181,8 +176,7 @@ func TestOutputCannonDisputeGame(t *testing.T) {
 }
 
 func TestOutputCannonDefendStep(t *testing.T) {
-	executor := uint64(1) // Different executor to the other tests to help balance things better
-	op_e2e.InitParallel(t, op_e2e.UsesCannon, op_e2e.UseExecutor(executor))
+	op_e2e.InitParallel(t, op_e2e.UsesCannon, op_e2e.UseExecutor(4))
 
 	ctx := context.Background()
 	sys, l1Client := startFaultDisputeSystem(t)
@@ -220,7 +214,7 @@ func TestOutputCannonDefendStep(t *testing.T) {
 func TestOutputCannonStepWithLargePreimage(t *testing.T) {
 	// TODO(client-pod#525): Fix preimage insertion and enable this test
 	t.Skip("Preimage not being inserted under correct key")
-	op_e2e.InitParallel(t, op_e2e.UsesCannon, op_e2e.UseExecutor(outputCannonTestExecutor))
+	op_e2e.InitParallel(t, op_e2e.UsesCannon, op_e2e.UseExecutor(0))
 
 	ctx := context.Background()
 	sys, l1Client := startFaultDisputeSystem(t, withLargeBatches())
@@ -269,9 +263,8 @@ func TestOutputCannonStepWithLargePreimage(t *testing.T) {
 }
 
 func TestOutputCannonStepWithPreimage(t *testing.T) {
-	op_e2e.InitParallel(t, op_e2e.UsesCannon, op_e2e.UseExecutor(outputCannonTestExecutor))
 	testPreimageStep := func(t *testing.T, preloadPreimage bool) {
-		op_e2e.InitParallel(t, op_e2e.UsesCannon, op_e2e.UseExecutor(outputCannonTestExecutor))
+		op_e2e.InitParallel(t, op_e2e.UsesCannon, op_e2e.UseExecutor(5))
 
 		ctx := context.Background()
 		sys, l1Client := startFaultDisputeSystem(t)
@@ -310,8 +303,6 @@ func TestOutputCannonStepWithPreimage(t *testing.T) {
 }
 
 func TestOutputCannonProposedOutputRootValid(t *testing.T) {
-	executor := uint64(1) // Different executor to the other tests to help balance things better
-	op_e2e.InitParallel(t, op_e2e.UsesCannon, op_e2e.UseExecutor(executor))
 	// honestStepsFail attempts to perform both an attack and defend step using the correct trace.
 	honestStepsFail := func(ctx context.Context, game *disputegame.OutputCannonGameHelper, correctTrace *disputegame.OutputHonestHelper, parentClaimIdx int64) {
 		// Attack step should fail
@@ -330,6 +321,9 @@ func TestOutputCannonProposedOutputRootValid(t *testing.T) {
 		// performStep is called once the maximum game depth is reached. It should perform a step to counter the
 		// claim at parentClaimIdx. Since the proposed output root is invalid, the step call should always revert.
 		performStep func(ctx context.Context, game *disputegame.OutputCannonGameHelper, correctTrace *disputegame.OutputHonestHelper, parentClaimIdx int64)
+
+		// executor to run the task on
+		executor uint64
 	}{
 		{
 			name: "AttackWithCorrectTrace",
@@ -342,6 +336,7 @@ func TestOutputCannonProposedOutputRootValid(t *testing.T) {
 				return correctTrace.AttackClaim(ctx, claim)
 			},
 			performStep: honestStepsFail,
+			executor:    6,
 		},
 		{
 			name: "DefendWithCorrectTrace",
@@ -360,13 +355,14 @@ func TestOutputCannonProposedOutputRootValid(t *testing.T) {
 				return correctTrace.DefendClaim(ctx, claim)
 			},
 			performStep: honestStepsFail,
+			executor:    7,
 		},
 	}
 
 	for _, test := range tests {
 		test := test
 		t.Run(test.name, func(t *testing.T) {
-			op_e2e.InitParallel(t, op_e2e.UseExecutor(executor))
+			op_e2e.InitParallel(t, op_e2e.UsesCannon, op_e2e.UseExecutor(test.executor))
 
 			ctx := context.Background()
 			sys, l1Client := startFaultDisputeSystem(t)
@@ -400,7 +396,7 @@ func TestOutputCannonProposedOutputRootValid(t *testing.T) {
 }
 
 func TestOutputCannonPoisonedPostState(t *testing.T) {
-	op_e2e.InitParallel(t, op_e2e.UsesCannon, op_e2e.UseExecutor(outputCannonTestExecutor))
+	op_e2e.InitParallel(t, op_e2e.UsesCannon, op_e2e.UseExecutor(1))
 
 	ctx := context.Background()
 	sys, l1Client := startFaultDisputeSystem(t)
@@ -464,7 +460,7 @@ func TestOutputCannonPoisonedPostState(t *testing.T) {
 }
 
 func TestDisputeOutputRootBeyondProposedBlock_ValidOutputRoot(t *testing.T) {
-	op_e2e.InitParallel(t, op_e2e.UsesCannon, op_e2e.UseExecutor(outputCannonTestExecutor))
+	op_e2e.InitParallel(t, op_e2e.UsesCannon, op_e2e.UseExecutor(2))
 
 	ctx := context.Background()
 	sys, l1Client := startFaultDisputeSystem(t)
@@ -514,7 +510,7 @@ func TestDisputeOutputRootBeyondProposedBlock_ValidOutputRoot(t *testing.T) {
 }
 
 func TestDisputeOutputRootBeyondProposedBlock_InvalidOutputRoot(t *testing.T) {
-	op_e2e.InitParallel(t, op_e2e.UsesCannon, op_e2e.UseExecutor(outputCannonTestExecutor))
+	op_e2e.InitParallel(t, op_e2e.UsesCannon, op_e2e.UseExecutor(3))
 
 	ctx := context.Background()
 	sys, l1Client := startFaultDisputeSystem(t)
@@ -565,7 +561,7 @@ func TestDisputeOutputRootBeyondProposedBlock_InvalidOutputRoot(t *testing.T) {
 }
 
 func TestDisputeOutputRoot_ChangeClaimedOutputRoot(t *testing.T) {
-	op_e2e.InitParallel(t, op_e2e.UsesCannon, op_e2e.UseExecutor(outputCannonTestExecutor))
+	op_e2e.InitParallel(t, op_e2e.UsesCannon, op_e2e.UseExecutor(4))
 
 	ctx := context.Background()
 	sys, l1Client := startFaultDisputeSystem(t)

op-e2e/helper.go

 package op_e2e
 
 import (
+	"crypto/md5"
 	"os"
 	"strconv"
 	"testing"
@@ -18,11 +19,17 @@ func InitParallel(t *testing.T, args ...func(t *testing.T, opts *testopts)) {
 		t.Parallel()
 	}
 
-	opts := &testopts{}
+	info := getExecutorInfo(t)
+	tName := t.Name()
+	tHash := md5.Sum([]byte(tName))
+	executor := uint64(tHash[0]) % info.total
+	opts := &testopts{
+		executor: executor,
+	}
 	for _, arg := range args {
 		arg(t, opts)
 	}
-	checkExecutor(t, opts.executor)
+	checkExecutor(t, info, opts.executor)
 }
 
 func UsesCannon(t *testing.T, opts *testopts) {
@@ -43,13 +50,21 @@ func UseExecutor(assignedIdx uint64) func(t *testing.T, opts *testopts) {
 	}
 }
 
-func checkExecutor(t *testing.T, assignedIdx uint64) {
+type executorInfo struct {
+	total      uint64
+	idx        uint64
+	splitInUse bool
+}
+
+func getExecutorInfo(t *testing.T) executorInfo {
+	var info executorInfo
 	envTotal := os.Getenv("CIRCLE_NODE_TOTAL")
 	envIdx := os.Getenv("CIRCLE_NODE_INDEX")
 	if envTotal == "" || envIdx == "" {
 		// Not using test splitting, so ignore assigned executor
-		t.Logf("Running test. Test splitting not in use.")
-		return
+		t.Logf("Test splitting not in use.")
+		info.total = 1
+		return info
 	}
 	total, err := strconv.ParseUint(envTotal, 10, 0)
 	if err != nil {
@@ -59,13 +74,26 @@ func checkExecutor(t *testing.T, assignedIdx uint64) {
 	if err != nil {
 		t.Fatalf("Could not parse CIRCLE_NODE_INDEX env var %v: %v", envIdx, err)
 	}
-	if assignedIdx >= total && idx == total-1 {
-		t.Logf("Running test. Current executor (%v) is the last executor and assigned executor (%v) >= total executors (%v).", idx, assignedIdx, total)
+
+	info.total = total
+	info.idx = idx
+	info.splitInUse = true
+	return info
+}
+
+func checkExecutor(t *testing.T, info executorInfo, assignedIdx uint64) {
+	if !info.splitInUse {
+		t.Logf("Test splitting not in use.")
+		return
+	}
+
+	if assignedIdx >= info.total && info.idx == info.total-1 {
+		t.Logf("Running test. Current executor (%v) is the last executor and assigned executor (%v) >= total executors (%v).", info.idx, assignedIdx, info.total)
 		return
 	}
-	if idx == assignedIdx {
-		t.Logf("Running test. Assigned executor (%v) matches current executor (%v) of total (%v)", assignedIdx, idx, total)
+	if info.idx == assignedIdx {
+		t.Logf("Running test. Assigned executor (%v) matches current executor (%v) of total (%v)", assignedIdx, info.idx, info.total)
 		return
 	}
-	t.Skipf("Skipping test. Assigned executor %v, current executor %v of total %v", assignedIdx, idx, total)
+	t.Skipf("Skipping test. Assigned executor %v, current executor %v of total %v", assignedIdx, info.idx, info.total)
 }

op-e2e/l1_beacon_client_test.go

@@ -13,6 +13,8 @@ import (
 )
 
 func TestGetVersion(t *testing.T) {
+	InitParallel(t)
+
 	l := testlog.Logger(t, log.LvlInfo)
 
 	beaconApi := fakebeacon.NewBeacon(l, t.TempDir(), uint64(0), uint64(0))

op-e2e/op_geth_test.go

@@ -223,7 +223,6 @@ func TestGethOnlyPendingBlockIsLatest(t *testing.T) {
 }
 
 func TestPreregolith(t *testing.T) {
-	InitParallel(t)
 	futureTimestamp := hexutil.Uint64(4)
 	tests := []struct {
 		name         string
@@ -408,7 +407,6 @@ func TestPreregolith(t *testing.T) {
 }
 
 func TestRegolith(t *testing.T) {
-	InitParallel(t)
 	tests := []struct {
 		name             string
 		regolithTime     hexutil.Uint64
@@ -726,7 +724,6 @@ func TestRegolith(t *testing.T) {
 }
 
 func TestPreCanyon(t *testing.T) {
-	InitParallel(t)
 	futureTimestamp := hexutil.Uint64(4)
 
 	tests := []struct {
@@ -793,8 +790,6 @@ func TestPreCanyon(t *testing.T) {
 }
 
 func TestCanyon(t *testing.T) {
-	InitParallel(t)
-
 	tests := []struct {
 		name           string
 		canyonTime     hexutil.Uint64
@@ -869,7 +864,6 @@ func TestCanyon(t *testing.T) {
 }
 
 func TestPreEcotone(t *testing.T) {
-	InitParallel(t)
 	futureTimestamp := hexutil.Uint64(4)
 
 	tests := []struct {
@@ -938,8 +932,6 @@ func TestPreEcotone(t *testing.T) {
 }
 
 func TestEcotone(t *testing.T) {
-	InitParallel(t)
-
 	tests := []struct {
 		name            string
 		ecotoneTime     hexutil.Uint64

op-e2e/system_test.go

@@ -1105,9 +1105,8 @@ func (sga *stateGetterAdapter) GetState(addr common.Address, key common.Hash) co
 
 // TestFees checks that L1/L2 fees are handled.
 func TestFees(t *testing.T) {
-	InitParallel(t)
-
 	t.Run("pre-regolith", func(t *testing.T) {
+		InitParallel(t)
 		cfg := DefaultSystemConfig(t)
 		cfg.DeployConfig.L1GenesisBlockBaseFeePerGas = (*hexutil.Big)(big.NewInt(7))
 
@@ -1118,6 +1117,7 @@ func TestFees(t *testing.T) {
 		testFees(t, cfg)
 	})
 	t.Run("regolith", func(t *testing.T) {
+		InitParallel(t)
 		cfg := DefaultSystemConfig(t)
 		cfg.DeployConfig.L1GenesisBlockBaseFeePerGas = (*hexutil.Big)(big.NewInt(7))
 
@@ -1128,6 +1128,7 @@ func TestFees(t *testing.T) {
 		testFees(t, cfg)
 	})
 	t.Run("ecotone", func(t *testing.T) {
+		InitParallel(t)
 		cfg := DefaultSystemConfig(t)
 		cfg.DeployConfig.L1GenesisBlockBaseFeePerGas = (*hexutil.Big)(big.NewInt(7))
 
@@ -1140,7 +1141,6 @@ func TestFees(t *testing.T) {
 }
 
 func testFees(t *testing.T, cfg SystemConfig) {
-
 	sys, err := cfg.Start(t)
 	require.Nil(t, err, "Error starting up system")
 	defer sys.Close()