Skip to content

N
nebula
Commit 65751ca9 authored by Matthew Slipper's avatar Matthew Slipper Committed by GitHub
Browse files
Options

ci: Phase 1 and 2 CI mitigations (#9304) 

Implements items 1 and 2 on the [CI Fixes Plan](https://www.notion.so/oplabs/CI-Fixes-Plan-c7b3fee803c1480eb8e0512bff51e208):

1. Disables `op-e2e-ws-tests`
2. Fails faster on HTTP tests
parent 7bf4a9b2
Show whitespace changes
Inline Side-by-side
Showing with 97 additions and 59 deletions
.circleci/config.yml
View file @ 65751ca9
......@@ -812,10 +812,10 @@ jobs:
target:
description: The make target to execute
type: string
cannon_enabled:
description: Whether to include cannon tests
default: true
type: boolean
parallelism:
description: Number of parallel test runs
type: integer
default: 6
docker:
- image: <<pipeline.parameters.ci_builder_image>>
resource_class: xlarge
......@@ -824,7 +824,7 @@ jobs:
# InitParallel(t, UseExecutor(1))
# Any tests assigned to an executor greater than the number available automatically use the last executor.
# Executor indexes start from 0
parallelism: 2
parallelism: <<parameters.parallelism>>
steps:
- checkout
- check-changed:
......@@ -851,7 +851,10 @@ jobs:
command: |
mkdir -p /testlogs
export OP_E2E_CANNON_ENABLED="<<parameters.cannon_enabled>>"
# The below env var gets overridden when running make test-cannon, but we
# need to explicitly set it here to prevent Cannon from running when we don't
# want it to.
export OP_E2E_CANNON_ENABLED="false"
# Note: We don't use circle CI test splits because we need to split by test name, not by package. There is an additional
# constraint that gotestsum does not currently (nor likely will) accept files from different pacakges when building.
JUNIT_FILE=/tmp/test-results/<<parameters.module>>_<<parameters.target>>.xml make <<parameters.target>> 2>&1 | tee /testlogs/test.log
......@@ -1501,25 +1504,34 @@ workflows:
requires:
- op-stack-go-lint
- go-e2e-test:
name: op-e2e-WS-tests
name: op-e2e-HTTP-tests
module: op-e2e
target: test-ws
target: test-http
parallelism: 4
requires:
- op-stack-go-lint
- devnet-allocs
- go-e2e-test:
name: op-e2e-HTTP-tests
name: op-e2e-action-tests
module: op-e2e
target: test-http
cannon_enabled: false
target: test-actions
parallelism: 1
requires:
- op-stack-go-lint
- devnet-allocs
- go-e2e-test:
name: op-e2e-cannon-tests
module: op-e2e
target: test-cannon
parallelism: 8
requires:
- op-stack-go-lint
- devnet-allocs
- cannon-prestate
- go-e2e-test:
name: op-e2e-ext-geth-tests
module: op-e2e
target: test-external-geth
cannon_enabled: false
requires:
- op-stack-go-lint
- devnet-allocs
......@@ -1547,8 +1559,9 @@ workflows:
- op-program-tests
- op-program-compat
- op-service-tests
- op-e2e-WS-tests
- op-e2e-HTTP-tests
- op-e2e-cannon-tests
- op-e2e-action-tests
- op-e2e-ext-geth-tests
- op-service-rethdb-tests
- docker-build: # just to warm up the cache (other jobs run in parallel)
......
op-e2e/Makefile
View file @ 65751ca9
# Generally, JUNIT_FILE is set in CI but may be specified to an arbitrary file location to emulate CI locally
ifdef JUNIT_FILE
go_test = OP_TESTLOG_DISABLE_COLOR=true OP_E2E_DISABLE_PARALLEL=false gotestsum --format=standard-verbose --junitfile=$(JUNIT_FILE) --
go_test = OP_TESTLOG_DISABLE_COLOR=true OP_E2E_DISABLE_PARALLEL=false gotestsum --format=standard-verbose --junitfile=$(JUNIT_FILE) -- -failfast
# Note: -parallel must be set to match the number of cores in the resource class
go_test_flags = -timeout=20m -parallel=8
else
......@@ -16,13 +16,20 @@ test-external-%: pre-test
$(go_test) $(go_test_flags) --externalL2 ./external_$*/
test-ws: pre-test
$(go_test) $(go_test_flags) ./...
$(go_test) $(go_test_flags) . ./e2eutils/...
.PHONY: test-ws
test-actions: pre-test
$(go_test) $(go_test_flags) ./actions
.PHONY: test-actions
test-http: pre-test
OP_E2E_USE_HTTP=true $(go_test) $(go_test_flags) ./...
OP_E2E_USE_HTTP=true $(go_test) $(go_test_flags) . ./e2eutils/...
.PHONY: test-http
test-cannon: pre-test
OP_E2E_CANNON_ENABLED=true $(go_test) $(go_test_flags) ./faultproofs
cannon-prestate:
make -C .. cannon-prestate
.PHONY: cannon-prestate
......
op-e2e/faultproofs/output_cannon_test.go
View file @ 65751ca9
......@@ -15,10 +15,8 @@ import (
"github.com/stretchr/testify/require"
)
const outputCannonTestExecutor = 0
func TestOutputCannonGame(t *testing.T) {
op_e2e.InitParallel(t, op_e2e.UsesCannon, op_e2e.UseExecutor(outputCannonTestExecutor))
op_e2e.InitParallel(t, op_e2e.UsesCannon, op_e2e.UseExecutor(0))
ctx := context.Background()
sys, l1Client := startFaultDisputeSystem(t)
t.Cleanup(sys.Close)
......@@ -74,7 +72,7 @@ func TestOutputCannonGame(t *testing.T) {
func TestOutputCannon_ChallengeAllZeroClaim(t *testing.T) {
// The dishonest actor always posts claims with all zeros.
op_e2e.InitParallel(t, op_e2e.UsesCannon, op_e2e.UseExecutor(outputCannonTestExecutor))
op_e2e.InitParallel(t, op_e2e.UsesCannon, op_e2e.UseExecutor(1))
ctx := context.Background()
sys, l1Client := startFaultDisputeSystem(t)
t.Cleanup(sys.Close)
......@@ -102,7 +100,6 @@ func TestOutputCannon_ChallengeAllZeroClaim(t *testing.T) {
}
func TestOutputCannon_PublishCannonRootClaim(t *testing.T) {
op_e2e.InitParallel(t, op_e2e.UsesCannon, op_e2e.UseExecutor(outputCannonTestExecutor))
tests := []struct {
disputeL2BlockNumber uint64
}{
......@@ -112,7 +109,8 @@ func TestOutputCannon_PublishCannonRootClaim(t *testing.T) {
for _, test := range tests {
test := test
t.Run(fmt.Sprintf("Dispute_%v", test.disputeL2BlockNumber), func(t *testing.T) {
op_e2e.InitParallel(t, op_e2e.UsesCannon, op_e2e.UseExecutor(outputCannonTestExecutor))
op_e2e.InitParallel(t, op_e2e.UsesCannon, op_e2e.UseExecutor(2))
ctx := context.Background()
sys, _ := startFaultDisputeSystem(t)
......@@ -130,9 +128,6 @@ func TestOutputCannon_PublishCannonRootClaim(t *testing.T) {
}
func TestOutputCannonDisputeGame(t *testing.T) {
executor := uint64(1) // Different executor to the other tests to help balance things better
op_e2e.InitParallel(t, op_e2e.UsesCannon, op_e2e.UseExecutor(executor))
tests := []struct {
name string
defendClaimDepth types.Depth
......@@ -144,7 +139,7 @@ func TestOutputCannonDisputeGame(t *testing.T) {
for _, test := range tests {
test := test
t.Run(test.name, func(t *testing.T) {
op_e2e.InitParallel(t, op_e2e.UseExecutor(executor))
op_e2e.InitParallel(t, op_e2e.UsesCannon, op_e2e.UseExecutor(3))
ctx := context.Background()
sys, l1Client := startFaultDisputeSystem(t)
......@@ -181,8 +176,7 @@ func TestOutputCannonDisputeGame(t *testing.T) {
}
func TestOutputCannonDefendStep(t *testing.T) {
executor := uint64(1) // Different executor to the other tests to help balance things better
op_e2e.InitParallel(t, op_e2e.UsesCannon, op_e2e.UseExecutor(executor))
op_e2e.InitParallel(t, op_e2e.UsesCannon, op_e2e.UseExecutor(4))
ctx := context.Background()
sys, l1Client := startFaultDisputeSystem(t)
......@@ -220,7 +214,7 @@ func TestOutputCannonDefendStep(t *testing.T) {
func TestOutputCannonStepWithLargePreimage(t *testing.T) {
// TODO(client-pod#525): Fix preimage insertion and enable this test
t.Skip("Preimage not being inserted under correct key")
op_e2e.InitParallel(t, op_e2e.UsesCannon, op_e2e.UseExecutor(outputCannonTestExecutor))
op_e2e.InitParallel(t, op_e2e.UsesCannon, op_e2e.UseExecutor(0))
ctx := context.Background()
sys, l1Client := startFaultDisputeSystem(t, withLargeBatches())
......@@ -269,9 +263,8 @@ func TestOutputCannonStepWithLargePreimage(t *testing.T) {
}
func TestOutputCannonStepWithPreimage(t *testing.T) {
op_e2e.InitParallel(t, op_e2e.UsesCannon, op_e2e.UseExecutor(outputCannonTestExecutor))
testPreimageStep := func(t *testing.T, preloadPreimage bool) {
op_e2e.InitParallel(t, op_e2e.UsesCannon, op_e2e.UseExecutor(outputCannonTestExecutor))
op_e2e.InitParallel(t, op_e2e.UsesCannon, op_e2e.UseExecutor(5))
ctx := context.Background()
sys, l1Client := startFaultDisputeSystem(t)
......@@ -310,8 +303,6 @@ func TestOutputCannonStepWithPreimage(t *testing.T) {
}
func TestOutputCannonProposedOutputRootValid(t *testing.T) {
executor := uint64(1) // Different executor to the other tests to help balance things better
op_e2e.InitParallel(t, op_e2e.UsesCannon, op_e2e.UseExecutor(executor))
// honestStepsFail attempts to perform both an attack and defend step using the correct trace.
honestStepsFail := func(ctx context.Context, game *disputegame.OutputCannonGameHelper, correctTrace *disputegame.OutputHonestHelper, parentClaimIdx int64) {
// Attack step should fail
......@@ -330,6 +321,9 @@ func TestOutputCannonProposedOutputRootValid(t *testing.T) {
// performStep is called once the maximum game depth is reached. It should perform a step to counter the
// claim at parentClaimIdx. Since the proposed output root is invalid, the step call should always revert.
performStep func(ctx context.Context, game *disputegame.OutputCannonGameHelper, correctTrace *disputegame.OutputHonestHelper, parentClaimIdx int64)
// executor to run the task on
executor uint64
}{
{
name: "AttackWithCorrectTrace",
......@@ -342,6 +336,7 @@ func TestOutputCannonProposedOutputRootValid(t *testing.T) {
return correctTrace.AttackClaim(ctx, claim)
},
performStep: honestStepsFail,
executor: 6,
},
{
name: "DefendWithCorrectTrace",
......@@ -360,13 +355,14 @@ func TestOutputCannonProposedOutputRootValid(t *testing.T) {
return correctTrace.DefendClaim(ctx, claim)
},
performStep: honestStepsFail,
executor: 7,
},
}
for _, test := range tests {
test := test
t.Run(test.name, func(t *testing.T) {
op_e2e.InitParallel(t, op_e2e.UseExecutor(executor))
op_e2e.InitParallel(t, op_e2e.UsesCannon, op_e2e.UseExecutor(test.executor))
ctx := context.Background()
sys, l1Client := startFaultDisputeSystem(t)
......@@ -400,7 +396,7 @@ func TestOutputCannonProposedOutputRootValid(t *testing.T) {
}
func TestOutputCannonPoisonedPostState(t *testing.T) {
op_e2e.InitParallel(t, op_e2e.UsesCannon, op_e2e.UseExecutor(outputCannonTestExecutor))
op_e2e.InitParallel(t, op_e2e.UsesCannon, op_e2e.UseExecutor(1))
ctx := context.Background()
sys, l1Client := startFaultDisputeSystem(t)
......@@ -464,7 +460,7 @@ func TestOutputCannonPoisonedPostState(t *testing.T) {
}
func TestDisputeOutputRootBeyondProposedBlock_ValidOutputRoot(t *testing.T) {
op_e2e.InitParallel(t, op_e2e.UsesCannon, op_e2e.UseExecutor(outputCannonTestExecutor))
op_e2e.InitParallel(t, op_e2e.UsesCannon, op_e2e.UseExecutor(2))
ctx := context.Background()
sys, l1Client := startFaultDisputeSystem(t)
......@@ -514,7 +510,7 @@ func TestDisputeOutputRootBeyondProposedBlock_ValidOutputRoot(t *testing.T) {
}
func TestDisputeOutputRootBeyondProposedBlock_InvalidOutputRoot(t *testing.T) {
op_e2e.InitParallel(t, op_e2e.UsesCannon, op_e2e.UseExecutor(outputCannonTestExecutor))
op_e2e.InitParallel(t, op_e2e.UsesCannon, op_e2e.UseExecutor(3))
ctx := context.Background()
sys, l1Client := startFaultDisputeSystem(t)
......@@ -565,7 +561,7 @@ func TestDisputeOutputRootBeyondProposedBlock_InvalidOutputRoot(t *testing.T) {
}
func TestDisputeOutputRoot_ChangeClaimedOutputRoot(t *testing.T) {
op_e2e.InitParallel(t, op_e2e.UsesCannon, op_e2e.UseExecutor(outputCannonTestExecutor))
op_e2e.InitParallel(t, op_e2e.UsesCannon, op_e2e.UseExecutor(4))
ctx := context.Background()
sys, l1Client := startFaultDisputeSystem(t)
......
op-e2e/helper.go
View file @ 65751ca9
package op_e2e
import (
"crypto/md5"
"os"
"strconv"
"testing"
......@@ -18,11 +19,17 @@ func InitParallel(t *testing.T, args ...func(t *testing.T, opts *testopts)) {
t.Parallel()
}
opts := &testopts{}
info := getExecutorInfo(t)
tName := t.Name()
tHash := md5.Sum([]byte(tName))
executor := uint64(tHash[0]) % info.total
opts := &testopts{
executor: executor,
}
for _, arg := range args {
arg(t, opts)
}
checkExecutor(t, opts.executor)
checkExecutor(t, info, opts.executor)
}
func UsesCannon(t *testing.T, opts *testopts) {
......@@ -43,13 +50,21 @@ func UseExecutor(assignedIdx uint64) func(t *testing.T, opts *testopts) {
}
}
func checkExecutor(t *testing.T, assignedIdx uint64) {
type executorInfo struct {
total uint64
idx uint64
splitInUse bool
}
func getExecutorInfo(t *testing.T) executorInfo {
var info executorInfo
envTotal := os.Getenv("CIRCLE_NODE_TOTAL")
envIdx := os.Getenv("CIRCLE_NODE_INDEX")
if envTotal == "" || envIdx == "" {
// Not using test splitting, so ignore assigned executor
t.Logf("Running test. Test splitting not in use.")
return
t.Logf("Test splitting not in use.")
info.total = 1
return info
}
total, err := strconv.ParseUint(envTotal, 10, 0)
if err != nil {
......@@ -59,13 +74,26 @@ func checkExecutor(t *testing.T, assignedIdx uint64) {
if err != nil {
t.Fatalf("Could not parse CIRCLE_NODE_INDEX env var %v: %v", envIdx, err)
}
if assignedIdx >= total && idx == total-1 {
t.Logf("Running test. Current executor (%v) is the last executor and assigned executor (%v) >= total executors (%v).", idx, assignedIdx, total)
info.total = total
info.idx = idx
info.splitInUse = true
return info
}
func checkExecutor(t *testing.T, info executorInfo, assignedIdx uint64) {
if !info.splitInUse {
t.Logf("Test splitting not in use.")
return
}
if assignedIdx >= info.total && info.idx == info.total-1 {
t.Logf("Running test. Current executor (%v) is the last executor and assigned executor (%v) >= total executors (%v).", info.idx, assignedIdx, info.total)
return
}
if idx == assignedIdx {
t.Logf("Running test. Assigned executor (%v) matches current executor (%v) of total (%v)", assignedIdx, idx, total)
if info.idx == assignedIdx {
t.Logf("Running test. Assigned executor (%v) matches current executor (%v) of total (%v)", assignedIdx, info.idx, info.total)
return
}
t.Skipf("Skipping test. Assigned executor %v, current executor %v of total %v", assignedIdx, idx, total)
t.Skipf("Skipping test. Assigned executor %v, current executor %v of total %v", assignedIdx, info.idx, info.total)
}
op-e2e/l1_beacon_client_test.go
View file @ 65751ca9
......@@ -13,6 +13,8 @@ import (
)
func TestGetVersion(t *testing.T) {
InitParallel(t)
l := testlog.Logger(t, log.LvlInfo)
beaconApi := fakebeacon.NewBeacon(l, t.TempDir(), uint64(0), uint64(0))
......
op-e2e/op_geth_test.go
View file @ 65751ca9
......@@ -223,7 +223,6 @@ func TestGethOnlyPendingBlockIsLatest(t *testing.T) {
}
func TestPreregolith(t *testing.T) {
InitParallel(t)
futureTimestamp := hexutil.Uint64(4)
tests := []struct {
name string
......@@ -408,7 +407,6 @@ func TestPreregolith(t *testing.T) {
}
func TestRegolith(t *testing.T) {
InitParallel(t)
tests := []struct {
name string
regolithTime hexutil.Uint64
......@@ -726,7 +724,6 @@ func TestRegolith(t *testing.T) {
}
func TestPreCanyon(t *testing.T) {
InitParallel(t)
futureTimestamp := hexutil.Uint64(4)
tests := []struct {
......@@ -793,8 +790,6 @@ func TestPreCanyon(t *testing.T) {
}
func TestCanyon(t *testing.T) {
InitParallel(t)
tests := []struct {
name string
canyonTime hexutil.Uint64
......@@ -869,7 +864,6 @@ func TestCanyon(t *testing.T) {
}
func TestPreEcotone(t *testing.T) {
InitParallel(t)
futureTimestamp := hexutil.Uint64(4)
tests := []struct {
......@@ -938,8 +932,6 @@ func TestPreEcotone(t *testing.T) {
}
func TestEcotone(t *testing.T) {
InitParallel(t)
tests := []struct {
name string
ecotoneTime hexutil.Uint64
......
op-e2e/system_test.go
View file @ 65751ca9
......@@ -1105,9 +1105,8 @@ func (sga *stateGetterAdapter) GetState(addr common.Address, key common.Hash) co
// TestFees checks that L1/L2 fees are handled.
func TestFees(t *testing.T) {
InitParallel(t)
t.Run("pre-regolith", func(t *testing.T) {
InitParallel(t)
cfg := DefaultSystemConfig(t)
cfg.DeployConfig.L1GenesisBlockBaseFeePerGas = (*hexutil.Big)(big.NewInt(7))
......@@ -1118,6 +1117,7 @@ func TestFees(t *testing.T) {
testFees(t, cfg)
})
t.Run("regolith", func(t *testing.T) {
InitParallel(t)
cfg := DefaultSystemConfig(t)
cfg.DeployConfig.L1GenesisBlockBaseFeePerGas = (*hexutil.Big)(big.NewInt(7))
......@@ -1128,6 +1128,7 @@ func TestFees(t *testing.T) {
testFees(t, cfg)
})
t.Run("ecotone", func(t *testing.T) {
InitParallel(t)
cfg := DefaultSystemConfig(t)
cfg.DeployConfig.L1GenesisBlockBaseFeePerGas = (*hexutil.Big)(big.NewInt(7))
......@@ -1140,7 +1141,6 @@ func TestFees(t *testing.T) {
}
func testFees(t *testing.T, cfg SystemConfig) {
sys, err := cfg.Start(t)
require.Nil(t, err, "Error starting up system")
defer sys.Close()
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment