Merge branch 'develop' into aj/challenger-gzip

.circleci/config.yml

@@ -775,11 +775,8 @@ jobs:
       module:
         description: Go Module Name
         type: string
-      use_http:
-        description: If the op-e2e package should use HTTP clients
-        type: string
-      use_external:
-        description: The extra-process shim (if any) that should be used
+      target:
+        description: The make target to execute
         type: string
     docker:
       - image: us-docker.pkg.dev/oplabs-tools-artifacts/images/ci-builder:latest
@@ -791,13 +788,6 @@ jobs:
       - run:
           name: prep results dir
           command: mkdir -p /tmp/test-results
-      - when:
-          condition: <<parameters.use_external>>
-          steps:
-            - run:
-                name: Build Shim
-                command: make -C <<parameters.use_external>>
-                working_directory: <<parameters.module>>
       - run:
           name: install geth
           command: make install-geth
@@ -807,21 +797,12 @@ jobs:
       - run:
           name: print go's available MIPS targets
           command: go tool dist list | grep mips
-      - run:
-          name: Run all init steps for op-e2e
-          command: make pre-test
-          working_directory: <<parameters.module>>
       - run:
           name: run tests
-          command: |
+          command:
             # Note: We don't use circle CI test splits because we need to split by test name, not by package. There is an additional
             # constraint that gotestsum does not currently (nor likely will) accept files from different pacakges when building.
-            # Note: -parallel must be set to match the number of cores in the resource class
-            export TEST_SUFFIX="<<parameters.use_external>>"
-            export EXTERNAL_L2="$(test -z '<<parameters.use_external>>' || echo '<<parameters.use_external>>/shim')"
-            OP_TESTLOG_DISABLE_COLOR=true OP_E2E_DISABLE_PARALLEL=false OP_E2E_USE_HTTP=<<parameters.use_http>>  gotestsum \
-            --format=standard-verbose --junitfile=/tmp/test-results/<<parameters.module>>_http_<<parameters.use_http>>$TEST_SUFFIX.xml \
-            -- -timeout=20m -parallel=8 --externalL2 "$EXTERNAL_L2" ./...
+            JUNIT_FILE=/tmp/test-results/<<parameters.module>>_<<parameters.target>>.xml make <<parameters.target>>
           working_directory: <<parameters.module>>
       - store_test_results:
           path: /tmp/test-results
@@ -1232,18 +1213,15 @@ workflows:
       - go-e2e-test:
           name: op-e2e-WS-tests
           module: op-e2e
-          use_http: "false"
-          use_external: ""
+          target: test-ws
       - go-e2e-test:
           name: op-e2e-HTTP-tests
           module: op-e2e
-          use_http: "true"
-          use_external: ""
+          target: test-http
       - go-e2e-test:
-          name: op-e2e-WS-tests-external-geth
+          name: op-e2e-ext-geth-tests
           module: op-e2e
-          use_http: "false"
-          use_external: "external_geth"
+          target: test-external-geth
       - bedrock-go-tests:
           requires:
             - op-batcher-lint

op-challenger/Dockerfile

@@ -6,6 +6,8 @@ RUN apk add --no-cache make gcc musl-dev linux-headers git jq bash
 
 # build op-challenger with the shared go.mod & go.sum files
 COPY ./op-challenger /app/op-challenger
+COPY ./op-program /app/op-program
+COPY ./op-preimage /app/op-preimage
 COPY ./op-bindings /app/op-bindings
 COPY ./op-node /app/op-node
 COPY ./op-service /app/op-service
@@ -19,16 +21,25 @@ COPY ./cannon /app/cannon
 COPY ./op-preimage /app/op-preimage
 COPY ./op-chain-ops /app/op-chain-ops
 
-WORKDIR /app/op-challenger
+WORKDIR /app/op-program
 
 RUN go mod download
 
 ARG TARGETOS TARGETARCH
 
+RUN make op-program-host VERSION="$VERSION" GOOS=$TARGETOS GOARCH=$TARGETARCH
+
+WORKDIR /app/op-challenger
+
 RUN make op-challenger VERSION="$VERSION" GOOS=$TARGETOS GOARCH=$TARGETARCH
 
 FROM alpine:3.18
 
+# Make the bundled op-program the default cannon server
+ENV OP_CHALLENGER_CANNON_SERVER /usr/local/bin/op-program
+
 COPY --from=builder /app/op-challenger/bin/op-challenger /usr/local/bin
 
+COPY --from=builder /app/op-program/bin/op-program /usr/local/bin
+
 CMD ["op-challenger"]

op-challenger/game/fault/player.go

@@ -18,9 +18,7 @@ import (
 	"github.com/ethereum/go-ethereum/log"
 )
 
-type Actor interface {
-	Act(ctx context.Context) error
-}
+type actor func(ctx context.Context) error
 
 type GameInfo interface {
 	GetGameStatus(context.Context) (types.GameStatus, error)
@@ -28,7 +26,7 @@ type GameInfo interface {
 }
 
 type GamePlayer struct {
-	agent                   Actor
+	act                     actor
 	agreeWithProposedOutput bool
 	loader                  GameInfo
 	logger                  log.Logger
@@ -53,6 +51,25 @@ func NewGamePlayer(
 
 	loader := NewLoader(contract)
 
+	status, err := loader.GetGameStatus(ctx)
+	if err != nil {
+		return nil, fmt.Errorf("failed to fetch game status: %w", err)
+	}
+	if status != types.GameStatusInProgress {
+		logger.Info("Game already resolved", "status", status)
+		// Game is already complete so skip creating the trace provider, loading game inputs etc.
+		return &GamePlayer{
+			logger:                  logger,
+			loader:                  loader,
+			agreeWithProposedOutput: cfg.AgreeWithProposedOutput,
+			completed:               true,
+			// Act function does nothing because the game is already complete
+			act: func(ctx context.Context) error {
+				return nil
+			},
+		}, nil
+	}
+
 	gameDepth, err := loader.FetchGameDepth(ctx)
 	if err != nil {
 		return nil, fmt.Errorf("failed to fetch the game depth: %w", err)
@@ -88,10 +105,11 @@ func NewGamePlayer(
 	}
 
 	return &GamePlayer{
-		agent:                   NewAgent(loader, int(gameDepth), provider, responder, updater, cfg.AgreeWithProposedOutput, logger),
+		act:                     NewAgent(loader, int(gameDepth), provider, responder, updater, cfg.AgreeWithProposedOutput, logger).Act,
 		agreeWithProposedOutput: cfg.AgreeWithProposedOutput,
 		loader:                  loader,
 		logger:                  logger,
+		completed:               status != types.GameStatusInProgress,
 	}, nil
 }
 
@@ -102,7 +120,7 @@ func (g *GamePlayer) ProgressGame(ctx context.Context) bool {
 		return true
 	}
 	g.logger.Trace("Checking if actions are required")
-	if err := g.agent.Act(ctx); err != nil {
+	if err := g.act(ctx); err != nil {
 		g.logger.Error("Error when acting on game", "err", err)
 	}
 	if status, err := g.loader.GetGameStatus(ctx); err != nil {

op-challenger/game/fault/player_test.go

@@ -157,7 +157,7 @@ func setupProgressGameTest(t *testing.T, agreeWithProposedRoot bool) (*testlog.C
 	logger.SetHandler(handler)
 	gameState := &stubGameState{claimCount: 1}
 	game := &GamePlayer{
-		agent:                   gameState,
+		act:                     gameState.Act,
 		agreeWithProposedOutput: agreeWithProposedRoot,
 		loader:                  gameState,
 		logger:                  logger,

op-e2e/Makefile

-test: pre-test
-	go test -v ./...
+# Generally, JUNIT_FILE is set in CI but may be specified to an arbitrary file location to emulate CI locally
+ifdef JUNIT_FILE
+	go_test = OP_TESTLOG_DISABLE_COLOR=true OP_E2E_DISABLE_PARALLEL=false gotestsum --format=standard-verbose --junitfile=$(JUNIT_FILE) --
+  # Note: -parallel must be set to match the number of cores in the resource class
+	go_test_flags = -timeout=20m -parallel=8
+else
+	go_test = go test
+	go_test_flags = -v
+endif
+
+test: pre-test test-ws
+.PHONY: test
+
+test-external-%: pre-test
+	make -C ./external_$*/
+	$(go_test) $(go_test_flags) --externalL2 ./external_$*/shim
+
+test-ws: pre-test
+	$(go_test) $(go_test_flags) ./...
+.PHONY: test-ws
+
+test-http: pre-test
+	OP_E2E_USE_HTTP=true $(go_test) $(go_test_flags) ./...
+.PHONY: test-ws
 
 cannon-prestate:
 	make -C .. cannon-prestate
+.PHONY: cannon-prestate
 
 # We depend on the absolute pre-state generated by cannon to deploy the dispute game contracts.
 devnet-allocs: pre-test-cannon
 	make -C .. devnet-allocs
+.PHONY: devnet-allocs
 
 pre-test: pre-test-cannon pre-test-allocs
-
+.PHONY: pre-test
 
 pre-test-cannon:
 	@if [ ! -e ../op-program/bin ]; then \
 		make cannon-prestate; \
 	fi
+.PHONY: pre-test-cannon
 
-pre-test-allocs: 
+pre-test-allocs:
 	@if [ ! -e ../.devnet ]; then \
 		make devnet-allocs; \
 	fi
+.PHONY: pre-test-allocs
 
 clean:
 	rm -r ../.devnet
 	rm -r ../op-program/bin
+.PHONY: clean
 
 lint:
 	golangci-lint run -E goimports,sqlclosecheck,bodyclose,asciicheck,misspell,errorlint --timeout 5m -e "errors.As" -e "errors.Is" ./...
-
-test-external-%: pre-test
-	make -C ./external_$*/
-	go test -v --externalL2 ./external_$*/shim
-
-
-.PHONY: \
-	test \
-	lint
+.PHONY: lint

op-node/flags/p2p_flags.go

@@ -152,6 +152,13 @@ var (
 		Value:    "",
 		EnvVars:  p2pEnv("STATIC"),
 	}
+	NetRestrict = &cli.StringFlag{
+		Name:     "p2p.netrestrict",
+		Usage:    "Comma-separated list of CIDR masks. P2P will only try to connect on these networks",
+		Required: false,
+		Value:    "",
+		EnvVars:  p2pEnv("NETRESTRICT"),
+	}
 	HostMux = &cli.StringFlag{
 		Name:     "p2p.mux",
 		Usage:    "Comma-separated list of multiplexing protocols in order of preference. At least 1 required. Options: 'yamux','mplex'.",
@@ -322,6 +329,7 @@ var p2pFlags = []cli.Flag{
 	AdvertiseUDPPort,
 	Bootnodes,
 	StaticPeers,
+	NetRestrict,
 	HostMux,
 	HostSecurity,
 	PeersLo,

op-node/p2p/cli/load_config.go

@@ -23,6 +23,7 @@ import (
 	"github.com/urfave/cli/v2"
 
 	"github.com/ethereum/go-ethereum/p2p/enode"
+	"github.com/ethereum/go-ethereum/p2p/netutil"
 )
 
 func NewConfig(ctx *cli.Context, rollupCfg *rollup.Config) (*p2p.Config, error) {
@@ -193,6 +194,13 @@ func loadDiscoveryOpts(conf *p2p.Config, ctx *cli.Context) error {
 		conf.Bootnodes = p2p.DefaultBootnodes
 	}
 
+	netRestrict, err := netutil.ParseNetlist(ctx.String(flags.NetRestrict.Name))
+	if err != nil {
+		return fmt.Errorf("failed to parse net list: %w", err)
+	}
+
+	conf.NetRestrict = netRestrict
+
 	return nil
 }
 

op-node/p2p/config.go

@@ -11,6 +11,7 @@ import (
 	"github.com/ethereum/go-ethereum/log"
 	"github.com/ethereum/go-ethereum/p2p/discover"
 	"github.com/ethereum/go-ethereum/p2p/enode"
+	"github.com/ethereum/go-ethereum/p2p/netutil"
 	ds "github.com/ipfs/go-datastore"
 	"github.com/libp2p/go-libp2p"
 	pubsub "github.com/libp2p/go-libp2p-pubsub"
@@ -84,6 +85,7 @@ type Config struct {
 	AdvertiseUDPPort uint16
 	Bootnodes        []*enode.Node
 	DiscoveryDB      *enode.DB
+	NetRestrict      *netutil.Netlist
 
 	StaticPeers []core.Multiaddr
 

op-node/p2p/discovery.go

@@ -97,7 +97,7 @@ func (conf *Config) Discovery(log log.Logger, rollupCfg *rollup.Config, tcpPort
 
 	cfg := discover.Config{
 		PrivateKey:   priv,
-		NetRestrict:  nil,
+		NetRestrict:  conf.NetRestrict,
 		Bootnodes:    conf.Bootnodes,
 		Unhandled:    nil, // Not used in dv5
 		Log:          log,

specs/rollup-node-p2p.md

@@ -147,7 +147,7 @@ TODO: the connection gater does currently not gate by IP address on the dial Acc
 
 #### Transport security
 
-[Libp2p-noise][libp2p-noise], `XX` handshake, with the the `secp256k1` P2P identity, as popularized in Eth2.
+[Libp2p-noise][libp2p-noise], `XX` handshake, with the `secp256k1` P2P identity, as popularized in Eth2.
 The TLS option is available as well, but `noise` should be prioritized in negotiation.
 
 #### Protocol negotiation