Merge pull request #3931 from ethereum-optimism/jg/use_circle_parallel

ci: Split go-bedrock tests & use circle CI parallelism for op-e2e

.circleci/config.yml

@@ -262,101 +262,6 @@ jobs:
           name: Upload coverage
           command: codecov --verbose --clean --flags <<parameters.coverage_flag>>
 
-  bedrock-go-tests:
-    docker:
-      - image: ethereumoptimism/ci-builder:latest
-    resource_class: xlarge
-    steps:
-      - checkout
-      - run:
-          name: Check if we should run
-          command: |
-            shopt -s inherit_errexit
-            CHANGED=$(check-changed "op-(batcher|bindings|e2e|node|proposer|chain-ops)" || echo "TRUE")
-            if [[ "$CHANGED" = "FALSE" ]]; then
-              circleci step halt
-            fi
-      - run:
-          name: lint op-bindings
-          command: |
-            golangci-lint run -E goimports,sqlclosecheck,bodyclose,asciicheck,misspell ./...
-          working_directory: op-bindings
-      - run:
-          name: lint op-node
-          command: |
-            golangci-lint run -E goimports,sqlclosecheck,bodyclose,asciicheck,misspell ./...
-          working_directory: op-node
-      - run:
-          name: lint op-proposer
-          command: |
-            golangci-lint run -E goimports,sqlclosecheck,bodyclose,asciicheck,misspell ./...
-          working_directory: op-proposer
-      - run:
-          name: lint op-batcher
-          command: |
-            golangci-lint run -E goimports,sqlclosecheck,bodyclose,asciicheck,misspell ./...
-          working_directory: op-batcher
-      - run:
-          name: lint op-e2e
-          command: |
-            golangci-lint run -E goimports,sqlclosecheck,bodyclose,asciicheck,misspell ./...
-          working_directory: op-e2e
-      - run:
-          name: lint op-service
-          command: |
-            golangci-lint run -E goimports,sqlclosecheck,bodyclose,asciicheck,misspell ./...
-          working_directory: op-service
-      - run:
-          name: lint op-chain-ops
-          command: |
-            golangci-lint run -E goimports,sqlclosecheck,bodyclose,asciicheck,misspell ./...
-          working_directory: op-chain-ops
-      - run:
-          name: prep results dir
-          command: mkdir -p /test-results
-      - run:
-          name: test op-node
-          command: |
-            gotestsum --junitfile /test-results/op-node.xml -- -coverpkg=github.com/ethereum-optimism/optimism/... -coverprofile=coverage.out -covermode=atomic ./...
-          working_directory: op-node
-      - run:
-          name: test op-proposer
-          command: |
-            gotestsum --junitfile /test-results/op-proposer.xml -- -coverpkg=github.com/ethereum-optimism/optimism/... -coverprofile=coverage.out -covermode=atomic ./...
-          working_directory: op-proposer
-      - run:
-          name: test op-batcher
-          command: |
-            gotestsum --junitfile /test-results/op-batcher.xml -- -coverpkg=github.com/ethereum-optimism/optimism/... -coverprofile=coverage.out -covermode=atomic ./...
-          working_directory: op-batcher
-      - run:
-          name: test op-e2e (WS)
-          command: |
-            gotestsum --format standard-verbose --junitfile /test-results/op-e2e.xml -- -coverpkg=github.com/ethereum-optimism/optimism/... -coverprofile=coverage.out -covermode=atomic ./...
-          working_directory: op-e2e
-      - run:
-          name: test op-e2e (HTTP)
-          command: |
-            OP_E2E_USE_HTTP=true gotestsum --junitfile /test-results/op-e2e.xml -- -coverpkg=github.com/ethereum-optimism/optimism/... -coverprofile=coverage.out -covermode=atomic ./...
-          working_directory: op-e2e
-      - run:
-          name: test op-service
-          command: |
-            gotestsum --junitfile /test-results/op-service.xml -- -coverpkg=github.com/ethereum-optimism/optimism/... -coverprofile=coverage.out -covermode=atomic ./...
-          working_directory: op-service
-      - run:
-          name: test op-chain-ops
-          command: |
-            gotestsum --junitfile /test-results/op-chain-ops.xml -- -coverpkg=github.com/ethereum-optimism/optimism/... -coverprofile=coverage.out -covermode=atomic ./...
-          working_directory: op-chain-ops
-      - run:
-          name: upload coverage
-          command: codecov --verbose --clean --flags bedrock-go-tests
-      - store_test_results:
-          path: /test-results
-      - run:
-          command: echo "Done."
-
   bedrock-markdown:
     machine:
       image: ubuntu-2204:2022.07.1
@@ -435,6 +340,74 @@ jobs:
           command: npx depcheck
           working_directory: integration-tests
 
+  go-lint:
+    parameters:
+      module:
+        description: Go Module Name
+        type: string
+    docker:
+      - image: cimg/go:1.19
+    steps:
+      - checkout
+      - run:
+          name: run lint
+          # command: golangci-lint run -E goimports,sqlclosecheck,bodyclose,asciicheck,misspell,errorlint -e "errors.As" -e "errors.Is" ./...
+          command: |
+            golangci-lint run -E goimports,sqlclosecheck,bodyclose,asciicheck,misspell ./...
+          working_directory: <<parameters.module>>
+
+  go-test:
+    parameters:
+      module:
+        description: Go Module Name
+        type: string
+    docker:
+      - image: ethereumoptimism/ci-builder:latest # only used to enable codecov.
+    resource_class: xlarge
+    steps:
+      - checkout
+      - run:
+          name: prep results dir
+          command: mkdir -p /tmp/test-results
+      - run:
+          name: run tests
+          command: |
+            gotestsum --format=standard-verbose --junitfile=/tmp/test-results/<<parameters.module>>.xml \
+            -- -coverpkg=github.com/ethereum-optimism/optimism/... -coverprofile=coverage.out ./...
+          working_directory: <<parameters.module>>
+      - run:
+          name: upload coverage
+          command: codecov --verbose --clean --flags bedrock-go-tests
+      - store_test_results:
+          path: /tmp/test-results
+
+  go-e2e-test:
+    parameters:
+      module:
+        description: Go Module Name
+        type: string
+      use_http:
+        description: If the op-e2e package should use HTTP clients
+        type: string
+    docker:
+      - image: cimg/go:1.19
+    parallelism: 6
+    resource_class: xlarge
+    steps:
+      - checkout
+      - run:
+          name: prep results dir
+          command: mkdir -p /tmp/test-results
+      - run:
+          name: run tests
+          command: |
+            OP_TESTLOG_DISABLE_COLOR=true OP_E2E_DISABLE_PARALLEL=true OP_E2E_USE_HTTP=<<parameters.use_http>>  gotestsum \
+            --format=standard-verbose --junitfile=/tmp/test-results/<<parameters.module>>_http_<<parameters.use_http>>.xml \
+            $(go list ./... | circleci tests split --split-by=timings)
+          working_directory: <<parameters.module>>
+      - store_test_results:
+          path: /tmp/test-results
+
   go-lint-test-build:
     parameters:
       binary_name:
@@ -705,6 +678,13 @@ jobs:
       - run:
           command: "! grep 'pass.*=false' /tmp/hive.log"
 
+  bedrock-go-tests:
+    docker:
+      - image: cimg/go:1.19
+    resource_class: medium
+    steps:
+      - run: echo Done
+
 workflows:
   main:
     jobs:
@@ -800,9 +780,6 @@ workflows:
       - depcheck:
           requires:
             - yarn-monorepo
-      - bedrock-go-tests
-      - fuzz-op-node
-      - bedrock-markdown
       # - devnet:
       #     name: devnet (with deployed contracts)
       #     deploy: true
@@ -841,6 +818,73 @@ workflows:
       - integration-tests
       - semgrep-scan
       - go-mod-tidy
+      - fuzz-op-node
+      - bedrock-markdown
+      - go-lint:
+          name: op-batcher-lint
+          module: op-batcher
+      - go-lint:
+          name: op-bindings-lint
+          module: op-bindings
+      - go-lint:
+          name: op-chain-ops-lint
+          module: op-chain-ops
+      - go-lint:
+          name: op-e2e-lint
+          module: op-e2e
+      - go-lint:
+          name: op-node-lint
+          module: op-node
+      - go-lint:
+          name: op-proposer-lint
+          module: op-proposer
+      - go-lint:
+          name: op-service-lint
+          module: op-service
+      - go-test:
+          name: op-batcher-tests
+          module: op-batcher
+      - go-test:
+          name: op-bindings-tests
+          module: op-bindings
+      - go-test:
+          name: op-chain-ops-tests
+          module: op-chain-ops
+      - go-test:
+          name: op-node-tests
+          module: op-node
+          # TODO: override resource class
+      - go-test:
+          name: op-proposer-tests
+          module: op-proposer
+      - go-test:
+          name: op-service-tests
+          module: op-service
+      - go-e2e-test:
+          name: op-e2e-WS-tests
+          module: op-e2e
+          use_http: "false"
+      - go-e2e-test:
+          name: op-e2e-HTTP-tests
+          module: op-e2e
+          use_http: "true"
+      - bedrock-go-tests:
+          requires:
+            - op-batcher-lint
+            - op-bindings-lint
+            - op-chain-ops-lint
+            - op-e2e-lint
+            - op-node-lint
+            - op-proposer-lint
+            - op-service-lint
+            - op-batcher-tests
+            - op-bindings-tests
+            - op-chain-ops-tests
+            - op-node-tests
+            - op-proposer-tests
+            - op-service-tests
+            - op-e2e-WS-tests
+            - op-e2e-HTTP-tests
       - docker-publish:
           name: op-node-publish-dev
           docker_file: op-node/Dockerfile
@@ -888,4 +932,4 @@ workflows:
           requires:
             - op-node-publish-dev
             - op-batcher-publish-dev
-            - op-proposer-publish-dev
+            - op-proposer-publish-dev
\ No newline at end of file

op-e2e/actions/action.go

@@ -2,10 +2,26 @@ package actions
 
 import (
 	"context"
+	"os"
 
 	"github.com/ethereum-optimism/optimism/op-e2e/e2eutils"
 )
 
+var enableParallelTesting bool = true
+
+func init() {
+	if os.Getenv("OP_E2E_DISABLE_PARALLEL") == "true" {
+		enableParallelTesting = false
+	}
+}
+
+func parallel(t e2eutils.TestingBase) {
+	t.Helper()
+	if enableParallelTesting {
+		t.Parallel()
+	}
+}
+
 // Testing is an interface to Go-like testing,
 // extended with a context getter for the test runner to shut down individual actions without interrupting the test,
 // and a signaling function for when an invalid action is hit.
@@ -53,7 +69,7 @@ type StatefulTesting interface {
 // NewDefaultTesting returns a new testing obj, and enables parallel test execution.
 // Returns an interface, we're likely changing the behavior here as we build more action tests.
 func NewDefaultTesting(tb e2eutils.TestingBase) StatefulTesting {
-	tb.Parallel()
+	parallel(tb)
 	return &defaultTesting{
 		TestingBase: tb,
 		ctx:         context.Background(),

op-e2e/system_test.go

@@ -5,6 +5,7 @@ import (
 	"flag"
 	"fmt"
 	"math/big"
+	"os"
 	"testing"
 	"time"
 
@@ -31,6 +32,8 @@ import (
 	"github.com/ethereum-optimism/optimism/op-node/withdrawals"
 )
 
+var enableParallelTesting bool = true
+
 // Init testing to enable test flags
 var _ = func() bool {
 	testing.Init()
@@ -42,10 +45,20 @@ var verboseGethNodes bool
 func init() {
 	flag.BoolVar(&verboseGethNodes, "gethlogs", true, "Enable logs on geth nodes")
 	flag.Parse()
+	if os.Getenv("OP_E2E_DISABLE_PARALLEL") == "true" {
+		enableParallelTesting = false
+	}
+}
+
+func parallel(t *testing.T) {
+	t.Helper()
+	if enableParallelTesting {
+		t.Parallel()
+	}
 }
 
 func TestL2OutputSubmitter(t *testing.T) {
-	t.Parallel()
+	parallel(t)
 	if !verboseGethNodes {
 		log.Root().SetHandler(log.DiscardHandler())
 	}
@@ -119,7 +132,7 @@ func TestL2OutputSubmitter(t *testing.T) {
 // TestSystemE2E sets up a L1 Geth node, a rollup node, and a L2 geth node and then confirms that L1 deposits are reflected on L2.
 // All nodes are run in process (but are the full nodes, not mocked or stubbed).
 func TestSystemE2E(t *testing.T) {
-	t.Parallel()
+	parallel(t)
 	if !verboseGethNodes {
 		log.Root().SetHandler(log.DiscardHandler())
 	}
@@ -227,7 +240,7 @@ func TestSystemE2E(t *testing.T) {
 
 // TestConfirmationDepth runs the rollup with both sequencer and verifier not immediately processing the tip of the chain.
 func TestConfirmationDepth(t *testing.T) {
-	t.Parallel()
+	parallel(t)
 	if !verboseGethNodes {
 		log.Root().SetHandler(log.DiscardHandler())
 	}
@@ -275,7 +288,7 @@ func TestConfirmationDepth(t *testing.T) {
 
 // TestFinalize tests if L2 finalizes after sufficient time after L1 finalizes
 func TestFinalize(t *testing.T) {
-	t.Parallel()
+	parallel(t)
 	if !verboseGethNodes {
 		log.Root().SetHandler(log.DiscardHandler())
 	}
@@ -303,7 +316,7 @@ func TestFinalize(t *testing.T) {
 }
 
 func TestMintOnRevertedDeposit(t *testing.T) {
-	t.Parallel()
+	parallel(t)
 	if !verboseGethNodes {
 		log.Root().SetHandler(log.DiscardHandler())
 	}
@@ -377,7 +390,7 @@ func TestMintOnRevertedDeposit(t *testing.T) {
 }
 
 func TestMissingBatchE2E(t *testing.T) {
-	t.Parallel()
+	parallel(t)
 	if !verboseGethNodes {
 		log.Root().SetHandler(log.DiscardHandler())
 	}
@@ -500,7 +513,7 @@ func L1InfoFromState(ctx context.Context, contract *bindings.L1Block, l2Number *
 // TestSystemMockP2P sets up a L1 Geth node, a rollup node, and a L2 geth node and then confirms that
 // the nodes can sync L2 blocks before they are confirmed on L1.
 func TestSystemMockP2P(t *testing.T) {
-	t.Parallel()
+	parallel(t)
 	if !verboseGethNodes {
 		log.Root().SetHandler(log.DiscardHandler())
 	}
@@ -569,7 +582,7 @@ func TestSystemMockP2P(t *testing.T) {
 }
 
 func TestL1InfoContract(t *testing.T) {
-	t.Parallel()
+	parallel(t)
 	if !verboseGethNodes {
 		log.Root().SetHandler(log.DiscardHandler())
 	}
@@ -697,7 +710,7 @@ func calcL1GasUsed(data []byte, overhead *big.Int) *big.Int {
 // balance changes on L1 and L2 and has to include gas fees in the balance checks.
 // It does not check that the withdrawal can be executed prior to the end of the finality period.
 func TestWithdrawals(t *testing.T) {
-	t.Parallel()
+	parallel(t)
 	if !verboseGethNodes {
 		log.Root().SetHandler(log.DiscardHandler())
 	}
@@ -870,7 +883,7 @@ func TestWithdrawals(t *testing.T) {
 
 // TestFees checks that L1/L2 fees are handled.
 func TestFees(t *testing.T) {
-	t.Parallel()
+	parallel(t)
 	if !verboseGethNodes {
 		log.Root().SetHandler(log.DiscardHandler())
 	}

op-node/testlog/testlog.go

@@ -18,6 +18,7 @@
 package testlog
 
 import (
+	"os"
 	"runtime"
 	"strconv"
 	"strings"
@@ -26,6 +27,14 @@ import (
 	"github.com/ethereum/go-ethereum/log"
 )
 
+var useColorInTestLog bool = true
+
+func init() {
+	if os.Getenv("OP_TESTLOG_DISABLE_COLOR") == "true" {
+		useColorInTestLog = false
+	}
+}
+
 // Testing interface to log to. Some functions are marked as Helper function to log the call site accurately.
 // Standard Go testing.TB implements this, as well as Hive and other Go-like test frameworks.
 type Testing interface {
@@ -75,7 +84,7 @@ func Logger(t Testing, level log.Lvl) log.Logger {
 		t:  t,
 		l:  log.New(),
 		mu: new(sync.Mutex),
-		h:  &bufHandler{fmt: log.TerminalFormat(true)},
+		h:  &bufHandler{fmt: log.TerminalFormat(useColorInTestLog)},
 	}
 	l.l.SetHandler(log.LvlFilterHandler(level, l.h))
 	return l