Merge pull request #7894 from ethereum-optimism/aj/e2e-groups

op-e2e: Split e2e tests into two executors

.circleci/config.yml

@@ -792,6 +792,12 @@ jobs:
     docker:
       - image: us-docker.pkg.dev/oplabs-tools-artifacts/images/ci-builder:latest
     resource_class: xlarge
+    # Note: Tests are split between runs manually.
+    # Tests default to run on the first executor but can be moved to the second with:
+    # InitParallel(t, UseExecutor(1))
+    # Any tests assigned to an executor greater than the number available automatically use the last executor.
+    # Executor indexes start from 0
+    parallelism: 2
     steps:
       - checkout
       - check-changed:

op-e2e/faultproof_test.go

@@ -18,7 +18,7 @@ import (
 )
 
 func TestMultipleCannonGames(t *testing.T) {
-	InitParallel(t, UsesCannon)
+	InitParallel(t, UsesCannon, UseExecutor(0))
 
 	ctx := context.Background()
 	sys, l1Client := startFaultDisputeSystem(t)
@@ -78,7 +78,7 @@ func TestMultipleCannonGames(t *testing.T) {
 }
 
 func TestMultipleGameTypes(t *testing.T) {
-	InitParallel(t, UsesCannon)
+	InitParallel(t, UsesCannon, UseExecutor(0))
 
 	ctx := context.Background()
 	sys, l1Client := startFaultDisputeSystem(t)
@@ -113,7 +113,7 @@ func TestMultipleGameTypes(t *testing.T) {
 }
 
 func TestChallengerCompleteDisputeGame(t *testing.T) {
-	InitParallel(t)
+	InitParallel(t, UseExecutor(1))
 
 	tests := []struct {
 		name              string
@@ -182,7 +182,7 @@ func TestChallengerCompleteDisputeGame(t *testing.T) {
 	for _, test := range tests {
 		test := test
 		t.Run(test.name, func(t *testing.T) {
-			InitParallel(t)
+			InitParallel(t, UseExecutor(1))
 
 			ctx := context.Background()
 			sys, l1Client := startFaultDisputeSystem(t)
@@ -219,7 +219,7 @@ func TestChallengerCompleteDisputeGame(t *testing.T) {
 }
 
 func TestChallengerCompleteExhaustiveDisputeGame(t *testing.T) {
-	InitParallel(t)
+	InitParallel(t, UseExecutor(1))
 
 	testCase := func(t *testing.T, isRootCorrect bool) {
 		ctx := context.Background()
@@ -267,17 +267,17 @@ func TestChallengerCompleteExhaustiveDisputeGame(t *testing.T) {
 	}
 
 	t.Run("RootCorrect", func(t *testing.T) {
-		InitParallel(t)
+		InitParallel(t, UseExecutor(1))
 		testCase(t, true)
 	})
 	t.Run("RootIncorrect", func(t *testing.T) {
-		InitParallel(t)
+		InitParallel(t, UseExecutor(1))
 		testCase(t, false)
 	})
 }
 
 func TestCannonDisputeGame(t *testing.T) {
-	InitParallel(t, UsesCannon)
+	InitParallel(t, UsesCannon, UseExecutor(1))
 
 	tests := []struct {
 		name             string
@@ -290,7 +290,7 @@ func TestCannonDisputeGame(t *testing.T) {
 	for _, test := range tests {
 		test := test
 		t.Run(test.name, func(t *testing.T) {
-			InitParallel(t)
+			InitParallel(t, UseExecutor(1))
 
 			ctx := context.Background()
 			sys, l1Client := startFaultDisputeSystem(t)
@@ -328,7 +328,7 @@ func TestCannonDisputeGame(t *testing.T) {
 }
 
 func TestCannonDefendStep(t *testing.T) {
-	InitParallel(t, UsesCannon)
+	InitParallel(t, UsesCannon, UseExecutor(1))
 
 	ctx := context.Background()
 	sys, l1Client := startFaultDisputeSystem(t)
@@ -370,7 +370,7 @@ func TestCannonDefendStep(t *testing.T) {
 }
 
 func TestCannonProposedOutputRootInvalid(t *testing.T) {
-	InitParallel(t, UsesCannon)
+	InitParallel(t, UsesCannon, UseExecutor(0))
 	// honestStepsFail attempts to perform both an attack and defend step using the correct trace.
 	honestStepsFail := func(ctx context.Context, game *disputegame.CannonGameHelper, correctTrace *disputegame.HonestHelper, parentClaimIdx int64) {
 		// Attack step should fail
@@ -421,7 +421,7 @@ func TestCannonProposedOutputRootInvalid(t *testing.T) {
 	for _, test := range tests {
 		test := test
 		t.Run(test.name, func(t *testing.T) {
-			InitParallel(t)
+			InitParallel(t, UseExecutor(0))
 
 			ctx := context.Background()
 			sys, l1Client, game, correctTrace := setupDisputeGameForInvalidOutputRoot(t, test.outputRoot)
@@ -448,7 +448,7 @@ func TestCannonProposedOutputRootInvalid(t *testing.T) {
 }
 
 func TestCannonPoisonedPostState(t *testing.T) {
-	InitParallel(t, UsesCannon)
+	InitParallel(t, UsesCannon, UseExecutor(0))
 
 	ctx := context.Background()
 	sys, l1Client := startFaultDisputeSystem(t)
@@ -558,8 +558,7 @@ func setupDisputeGameForInvalidOutputRoot(t *testing.T, outputRoot common.Hash)
 }
 
 func TestCannonChallengeWithCorrectRoot(t *testing.T) {
-	InitParallel(t, UsesCannon)
-
+	InitParallel(t, UsesCannon, UseExecutor(0))
 	ctx := context.Background()
 	sys, l1Client := startFaultDisputeSystem(t)
 	t.Cleanup(sys.Close)

op-e2e/helper.go

@@ -2,23 +2,70 @@ package op_e2e
 
 import (
 	"os"
+	"strconv"
 	"testing"
 )
 
 var enableParallelTesting bool = os.Getenv("OP_E2E_DISABLE_PARALLEL") != "true"
 
-func InitParallel(t *testing.T, opts ...func(t *testing.T)) {
+type testopts struct {
+	executor uint64
+}
+
+func InitParallel(t *testing.T, args ...func(t *testing.T, opts *testopts)) {
 	t.Helper()
 	if enableParallelTesting {
 		t.Parallel()
 	}
-	for _, opt := range opts {
-		opt(t)
+
+	opts := &testopts{}
+	for _, arg := range args {
+		arg(t, opts)
 	}
+	checkExecutor(t, opts.executor)
 }
 
-func UsesCannon(t *testing.T) {
+func UsesCannon(t *testing.T, opts *testopts) {
 	if os.Getenv("OP_E2E_CANNON_ENABLED") == "false" {
 		t.Skip("Skipping cannon test")
 	}
 }
+
+//	UseExecutor allows manually splitting tests between circleci executors
+//
+// Tests default to run on the first executor but can be moved to the second with:
+// InitParallel(t, UseExecutor(1))
+// Any tests assigned to an executor greater than the number available automatically use the last executor.
+// Executor indexes start from 0
+func UseExecutor(assignedIdx uint64) func(t *testing.T, opts *testopts) {
+	return func(t *testing.T, opts *testopts) {
+		opts.executor = assignedIdx
+	}
+}
+
+func checkExecutor(t *testing.T, assignedIdx uint64) {
+	envTotal := os.Getenv("CIRCLE_NODE_TOTAL")
+	envIdx := os.Getenv("CIRCLE_NODE_INDEX")
+	if envTotal == "" || envIdx == "" {
+		// Not using test splitting, so ignore assigned executor
+		t.Logf("Running test. Test splitting not in use.")
+		return
+	}
+	total, err := strconv.ParseUint(envTotal, 10, 0)
+	if err != nil {
+		t.Fatalf("Could not parse CIRCLE_NODE_TOTAL env var %v: %v", envTotal, err)
+	}
+	idx, err := strconv.ParseUint(envIdx, 10, 0)
+	if err != nil {
+		t.Fatalf("Could not parse CIRCLE_NODE_INDEX env var %v: %v", envIdx, err)
+	}
+	if assignedIdx >= total && idx == total-1 {
+		t.Logf("Running test. Current executor (%v) is the last executor and assigned executor (%v) >= total executors (%v).", idx, assignedIdx, total)
+		return
+	}
+	if idx == assignedIdx {
+		t.Logf("Running test. Assigned executor (%v) matches current executor (%v) of total (%v)", assignedIdx, idx, total)
+		return
+	}
+	t.Skipf("Skipping test. Assigned executor %v, current executor %v of total %v", assignedIdx, idx, total)
+}