Merge pull request #7623 from danyalprout/migrate-v2-apis

Migrate from V1 to V2 APIs

op-program/client/l2/engineapi/l2_engine_api.go

@@ -6,6 +6,7 @@ import (
 	"encoding/binary"
 	"errors"
 	"fmt"
+	"math/big"
 	"time"
 
 	"github.com/ethereum-optimism/optimism/op-service/eth"
@@ -177,6 +178,82 @@ func (ea *L2EngineAPI) endBlock() (*types.Block, error) {
 }
 
 func (ea *L2EngineAPI) GetPayloadV1(ctx context.Context, payloadId eth.PayloadID) (*eth.ExecutionPayload, error) {
+	return ea.getPayload(ctx, payloadId)
+}
+
+func (ea *L2EngineAPI) GetPayloadV2(ctx context.Context, payloadId eth.PayloadID) (*eth.ExecutionPayloadEnvelope, error) {
+	payload, err := ea.getPayload(ctx, payloadId)
+	return &eth.ExecutionPayloadEnvelope{ExecutionPayload: payload}, err
+}
+
+func (ea *L2EngineAPI) config() *params.ChainConfig {
+	return ea.backend.Config()
+}
+
+func (ea *L2EngineAPI) ForkchoiceUpdatedV1(ctx context.Context, state *eth.ForkchoiceState, attr *eth.PayloadAttributes) (*eth.ForkchoiceUpdatedResult, error) {
+	if attr != nil {
+		if attr.Withdrawals != nil {
+			return STATUS_INVALID, engine.InvalidParams.With(errors.New("withdrawals not supported in V1"))
+		}
+		if ea.config().IsShanghai(ea.config().LondonBlock, uint64(attr.Timestamp)) {
+			return STATUS_INVALID, engine.InvalidParams.With(errors.New("forkChoiceUpdateV1 called post-shanghai"))
+		}
+	}
+
+	return ea.forkchoiceUpdated(ctx, state, attr)
+}
+
+func (ea *L2EngineAPI) ForkchoiceUpdatedV2(ctx context.Context, state *eth.ForkchoiceState, attr *eth.PayloadAttributes) (*eth.ForkchoiceUpdatedResult, error) {
+	if attr != nil {
+		if err := ea.verifyPayloadAttributes(attr); err != nil {
+			return STATUS_INVALID, engine.InvalidParams.With(err)
+		}
+	}
+
+	return ea.forkchoiceUpdated(ctx, state, attr)
+}
+
+func (ea *L2EngineAPI) verifyPayloadAttributes(attr *eth.PayloadAttributes) error {
+	c := ea.config()
+
+	// Verify withdrawals attribute for Shanghai.
+	if err := checkAttribute(c.IsShanghai, attr.Withdrawals != nil, c.LondonBlock, uint64(attr.Timestamp)); err != nil {
+		return fmt.Errorf("invalid withdrawals: %w", err)
+	}
+	return nil
+}
+
+func checkAttribute(active func(*big.Int, uint64) bool, exists bool, block *big.Int, time uint64) error {
+	if active(block, time) && !exists {
+		return errors.New("fork active, missing expected attribute")
+	}
+	if !active(block, time) && exists {
+		return errors.New("fork inactive, unexpected attribute set")
+	}
+	return nil
+}
+
+func (ea *L2EngineAPI) NewPayloadV1(ctx context.Context, payload *eth.ExecutionPayload) (*eth.PayloadStatusV1, error) {
+	if payload.Withdrawals != nil {
+		return &eth.PayloadStatusV1{Status: eth.ExecutionInvalid}, engine.InvalidParams.With(errors.New("withdrawals not supported in V1"))
+	}
+
+	return ea.newPayload(ctx, payload)
+}
+
+func (ea *L2EngineAPI) NewPayloadV2(ctx context.Context, payload *eth.ExecutionPayload) (*eth.PayloadStatusV1, error) {
+	if ea.config().IsShanghai(new(big.Int).SetUint64(uint64(payload.BlockNumber)), uint64(payload.Timestamp)) {
+		if payload.Withdrawals == nil {
+			return &eth.PayloadStatusV1{Status: eth.ExecutionInvalid}, engine.InvalidParams.With(errors.New("nil withdrawals post-shanghai"))
+		}
+	} else if payload.Withdrawals != nil {
+		return &eth.PayloadStatusV1{Status: eth.ExecutionInvalid}, engine.InvalidParams.With(errors.New("non-nil withdrawals pre-shanghai"))
+	}
+
+	return ea.newPayload(ctx, payload)
+}
+
+func (ea *L2EngineAPI) getPayload(ctx context.Context, payloadId eth.PayloadID) (*eth.ExecutionPayload, error) {
 	ea.log.Trace("L2Engine API request received", "method", "GetPayload", "id", payloadId)
 	if ea.payloadID != payloadId {
 		ea.log.Warn("unexpected payload ID requested for block building", "expected", ea.payloadID, "got", payloadId)
@@ -190,7 +267,7 @@ func (ea *L2EngineAPI) GetPayloadV1(ctx context.Context, payloadId eth.PayloadID
 	return eth.BlockAsPayload(bl)
 }
 
-func (ea *L2EngineAPI) ForkchoiceUpdatedV1(ctx context.Context, state *eth.ForkchoiceState, attr *eth.PayloadAttributes) (*eth.ForkchoiceUpdatedResult, error) {
+func (ea *L2EngineAPI) forkchoiceUpdated(ctx context.Context, state *eth.ForkchoiceState, attr *eth.PayloadAttributes) (*eth.ForkchoiceUpdatedResult, error) {
 	ea.log.Trace("L2Engine API request received", "method", "ForkchoiceUpdated", "head", state.HeadBlockHash, "finalized", state.FinalizedBlockHash, "safe", state.SafeBlockHash)
 	if state.HeadBlockHash == (common.Hash{}) {
 		ea.log.Warn("Forkchoice requested update to zero hash")
@@ -273,7 +350,7 @@ func (ea *L2EngineAPI) ForkchoiceUpdatedV1(ctx context.Context, state *eth.Forkc
 	return valid(nil), nil
 }
 
-func (ea *L2EngineAPI) NewPayloadV1(ctx context.Context, payload *eth.ExecutionPayload) (*eth.PayloadStatusV1, error) {
+func (ea *L2EngineAPI) newPayload(ctx context.Context, payload *eth.ExecutionPayload) (*eth.PayloadStatusV1, error) {
 	ea.log.Trace("L2Engine API request received", "method", "ExecutePayload", "number", payload.BlockNumber, "hash", payload.BlockHash)
 	txs := make([][]byte, len(payload.Transactions))
 	for i, tx := range payload.Transactions {

op-service/eth/types.go

@@ -125,6 +125,10 @@ type Data = hexutil.Bytes
 
 type PayloadID = engine.PayloadID
 
+type ExecutionPayloadEnvelope struct {
+	ExecutionPayload *ExecutionPayload `json:"executionPayload"`
+}
+
 type ExecutionPayload struct {
 	ParentHash    common.Hash     `json:"parentHash"`
 	FeeRecipient  common.Address  `json:"feeRecipient"`
@@ -139,6 +143,7 @@ type ExecutionPayload struct {
 	ExtraData     BytesMax32      `json:"extraData"`
 	BaseFeePerGas Uint256Quantity `json:"baseFeePerGas"`
 	BlockHash     common.Hash     `json:"blockHash"`
+	Withdrawals   *[]Withdrawal   `json:"withdrawals,omitempty"`
 	// Array of transaction objects, each object is a byte list (DATA) representing
 	// TransactionType || TransactionPayload or LegacyTransaction as defined in EIP-2718
 	Transactions []Data `json:"transactions"`
@@ -228,6 +233,8 @@ type PayloadAttributes struct {
 	PrevRandao Bytes32 `json:"prevRandao"`
 	// suggested value for the coinbase field of the new payload
 	SuggestedFeeRecipient common.Address `json:"suggestedFeeRecipient"`
+	// Withdrawals to include into the block -- should be nil or empty depending on Shanghai enablement
+	Withdrawals *[]Withdrawal `json:"withdrawals,omitempty"`
 	// Transactions to force into the block (always at the start of the transactions list).
 	Transactions []Data `json:"transactions,omitempty"`
 	// NoTxPool to disable adding any transactions from the transaction-pool.
@@ -293,3 +300,11 @@ type SystemConfig struct {
 	GasLimit uint64 `json:"gasLimit"`
 	// More fields can be added for future SystemConfig versions.
 }
+
+// Withdrawal represents a validator withdrawal from the consensus layer.
+type Withdrawal struct {
+	Index     uint64         `json:"index"`          // monotonically increasing identifier issued by consensus layer
+	Validator uint64         `json:"validatorIndex"` // index of validator associated with withdrawal
+	Address   common.Address `json:"address"`        // target address for withdrawn ether
+	Amount    uint64         `json:"amount"`         // value of withdrawal in Gwei
+}

op-service/sources/engine_client.go

@@ -56,7 +56,7 @@ func (s *EngineClient) ForkchoiceUpdate(ctx context.Context, fc *eth.ForkchoiceS
 	fcCtx, cancel := context.WithTimeout(ctx, time.Second*5)
 	defer cancel()
 	var result eth.ForkchoiceUpdatedResult
-	err := s.client.CallContext(fcCtx, &result, "engine_forkchoiceUpdatedV1", fc, attributes)
+	err := s.client.CallContext(fcCtx, &result, "engine_forkchoiceUpdatedV2", fc, attributes)
 	if err == nil {
 		e.Trace("Shared forkchoice-updated signal")
 		if attributes != nil { // block building is optional, we only get a payload ID if we are building a block
@@ -91,7 +91,7 @@ func (s *EngineClient) NewPayload(ctx context.Context, payload *eth.ExecutionPay
 	execCtx, cancel := context.WithTimeout(ctx, time.Second*5)
 	defer cancel()
 	var result eth.PayloadStatusV1
-	err := s.client.CallContext(execCtx, &result, "engine_newPayloadV1", payload)
+	err := s.client.CallContext(execCtx, &result, "engine_newPayloadV2", payload)
 	e.Trace("Received payload execution result", "status", result.Status, "latestValidHash", result.LatestValidHash, "message", result.ValidationError)
 	if err != nil {
 		e.Error("Payload execution failed", "err", err)
@@ -107,8 +107,8 @@ func (s *EngineClient) NewPayload(ctx context.Context, payload *eth.ExecutionPay
 func (s *EngineClient) GetPayload(ctx context.Context, payloadId eth.PayloadID) (*eth.ExecutionPayload, error) {
 	e := s.log.New("payload_id", payloadId)
 	e.Trace("getting payload")
-	var result eth.ExecutionPayload
-	err := s.client.CallContext(ctx, &result, "engine_getPayloadV1", payloadId)
+	var result eth.ExecutionPayloadEnvelope
+	err := s.client.CallContext(ctx, &result, "engine_getPayloadV2", payloadId)
 	if err != nil {
 		e.Warn("Failed to get payload", "payload_id", payloadId, "err", err)
 		if rpcErr, ok := err.(rpc.Error); ok {
@@ -126,7 +126,7 @@ func (s *EngineClient) GetPayload(ctx context.Context, payloadId eth.PayloadID)
 		return nil, err
 	}
 	e.Trace("Received payload")
-	return &result, nil
+	return result.ExecutionPayload, nil
 }
 
 func (s *EngineClient) SignalSuperchainV1(ctx context.Context, recommended, required params.ProtocolVersion) (params.ProtocolVersion, error) {

specs/derivation.md

@@ -679,14 +679,17 @@ To interact with the engine, the [execution engine API][exec-engine] is used, wi
 
 [exec-engine]: exec-engine.md
 
-- [`engine_forkchoiceUpdatedV1`] — updates the forkchoice (i.e. the chain head) to `headBlockHash` if different, and
+- [`engine_forkchoiceUpdatedV2`] — updates the forkchoice (i.e. the chain head) to `headBlockHash` if different, and
   instructs the engine to start building an execution payload if the payload attributes parameter is not `null`.
-- [`engine_getPayloadV1`] — retrieves a previously requested execution payload build.
-- [`engine_newPayloadV1`] — executes an execution payload to create a block.
+- [`engine_getPayloadV2`] — retrieves a previously requested execution payload build.
+- [`engine_newPayloadV2`] — executes an execution payload to create a block.
 
-[`engine_forkchoiceUpdatedV1`]: exec-engine.md#engine_forkchoiceupdatedv1
-[`engine_getPayloadV1`]: exec-engine.md#engine_getpayloadv1
-[`engine_newPayloadV1`]: exec-engine.md#engine_newpayloadv1
+The current version of `op-node` uses the `v2` RPC methods from the engine API, whereas prior versions used the `v1`
+equivalents. The `v2` methods are backwards compatible with `v1` payloads but support Shanghai.
+
+[`engine_forkchoiceUpdatedV2`]: exec-engine.md#engine_forkchoiceupdatedv2
+[`engine_getPayloadV2`]: exec-engine.md#engine_getpayloadv2
+[`engine_newPayloadV2`]: exec-engine.md#engine_newpayloadv2
 
 The execution payload is an object of type [`ExecutionPayloadV1`][eth-payload].
 
@@ -703,7 +706,7 @@ This synchronization may happen when:
 - A successful consolidation of unsafe L2 blocks: updating the "safe" L2 block.
 - The first thing after a derivation pipeline reset, to ensure a consistent execution engine forkchoice state.
 
-The new forkchoice state is applied with `engine_forkchoiceUpdatedV1`.
+The new forkchoice state is applied with `engine_forkchoiceUpdatedV2`.
 On forkchoice-state validity errors the derivation pipeline must be reset to recover to consistent state.
 
 #### L1-consolidation: payload attributes matching
@@ -749,11 +752,11 @@ Engine][exec-engine-comm] section.
 
 The payload attributes are then processed with a sequence of:
 
-- `engine_forkchoiceUpdatedV1` with current forkchoice state of the stage, and the attributes to start block building.
+- `engine_forkchoiceUpdatedV2` with current forkchoice state of the stage, and the attributes to start block building.
   - Non-deterministic sources, like the tx-pool, must be disabled to reconstruct the expected block.
 - `engine_getPayload` to retrieve the payload, by the payload-ID in the result of the previous step.
 - `engine_newPayload` to import the new payload into the execution engine.
-- `engine_forkchoiceUpdatedV1` to make the new payload canonical,
+- `engine_forkchoiceUpdatedV2` to make the new payload canonical,
    now with a change of both `safe` and `unsafe` fields to refer to the payload, and no payload attributes.
 
 Engine API Error handling:
@@ -782,8 +785,8 @@ To process unsafe payloads, the payload must:
 
 The payload is then processed with a sequence of:
 
-- `engine_newPayloadV1`: process the payload. It does not become canonical yet.
-- `engine_forkchoiceUpdatedV1`: make the payload the canonical unsafe L2 head, and keep the safe/finalized L2 heads.
+- `engine_newPayloadV2`: process the payload. It does not become canonical yet.
+- `engine_forkchoiceUpdatedV2`: make the payload the canonical unsafe L2 head, and keep the safe/finalized L2 heads.
 
 Engine API Error handling:
 

specs/exec-engine.md

@@ -12,10 +12,10 @@
   - [Base fees (Base Fee Vault)](#base-fees-base-fee-vault)
   - [L1-Cost fees (L1 Fee Vault)](#l1-cost-fees-l1-fee-vault)
 - [Engine API](#engine-api)
-  - [`engine_forkchoiceUpdatedV1`](#engine_forkchoiceupdatedv1)
+  - [`engine_forkchoiceUpdatedV2`](#engine_forkchoiceupdatedv2)
     - [Extended PayloadAttributesV1](#extended-payloadattributesv1)
-  - [`engine_newPayloadV1`](#engine_newpayloadv1)
-  - [`engine_getPayloadV1`](#engine_getpayloadv1)
+  - [`engine_newPayloadV2`](#engine_newpayloadv2)
+  - [`engine_getPayloadV2`](#engine_getpayloadv2)
   - [`engine_signalSuperchainV1`](#engine_signalsuperchainv1)
 - [Networking](#networking)
 - [Sync](#sync)
@@ -127,7 +127,7 @@ can be accessed in two interchangeable ways:
 There may be subtle tweaks, beta starts in a few weeks*
 -->
 
-### `engine_forkchoiceUpdatedV1`
+### `engine_forkchoiceUpdatedV2`
 
 This updates which L2 blocks the engine considers to be canonical (`forkchoiceState` argument),
 and optionally initiates block production (`payloadAttributes` argument).
@@ -140,7 +140,7 @@ Within the rollup, the types of forkchoice updates translate as:
 - `finalizedBlockHash`: irreversible block hash, matches lower boundary of the dispute period.
 
 To support rollup functionality, one backwards-compatible change is introduced
-to [`engine_forkchoiceUpdatedV1`][engine_forkchoiceUpdatedV1]: the extended `PayloadAttributesV1`
+to [`engine_forkchoiceUpdatedV2`][engine_forkchoiceUpdatedV2]: the extended `PayloadAttributesV1`
 
 #### Extended PayloadAttributesV1
 
@@ -181,7 +181,7 @@ The `noTxPool` is optional as well, and extends the `transactions` meaning:
 If the `transactions` field is present, the engine must execute the transactions in order and return `STATUS_INVALID`
 if there is an error processing the transactions. It must return `STATUS_VALID` if all of the transactions could
 be executed without error. **Note**: The state transition rules have been modified such that deposits will never fail
-so if `engine_forkchoiceUpdatedV1` returns `STATUS_INVALID` it is because a batched transaction is invalid.
+so if `engine_forkchoiceUpdatedV2` returns `STATUS_INVALID` it is because a batched transaction is invalid.
 
 The `gasLimit` is optional w.r.t. compatibility with L1, but required when used as rollup.
 This field overrides the gas limit used during block-building.
@@ -189,15 +189,15 @@ If not specified as rollup, a `STATUS_INVALID` is returned.
 
 [rollup-driver]: rollup-node.md
 
-### `engine_newPayloadV1`
+### `engine_newPayloadV2`
 
-No modifications to [`engine_newPayloadV1`][engine_newPayloadV1].
+No modifications to [`engine_newPayloadV2`][engine_newPayloadV2].
 Applies a L2 block to the engine state.
 
-### `engine_getPayloadV1`
+### `engine_getPayloadV2`
 
-No modifications to [`engine_getPayloadV1`][engine_getPayloadV1].
-Retrieves a payload by ID, prepared by `engine_forkchoiceUpdatedV1` when called with `payloadAttributes`.
+No modifications to [`engine_getPayloadV2`][engine_getPayloadV2].
+Retrieves a payload by ID, prepared by `engine_forkchoiceUpdatedV2` when called with `payloadAttributes`.
 
 ### `engine_signalSuperchainV1`
 
@@ -274,8 +274,8 @@ as the engine implementation can sync state faster through methods like [snap-sy
 ### Happy-path sync
 
 1. The rollup node informs the engine of the L2 chain head, unconditionally (part of regular node operation):
-   - [`engine_newPayloadV1`][engine_newPayloadV1] is called with latest L2 block received from P2P.
-   - [`engine_forkchoiceUpdatedV1`][engine_forkchoiceUpdatedV1] is called with the current
+   - [`engine_newPayloadV2`][engine_newPayloadV2] is called with latest L2 block received from P2P.
+   - [`engine_forkchoiceUpdatedV2`][engine_forkchoiceUpdatedV2] is called with the current
      `unsafe`/`safe`/`finalized` L2 block hashes.
 2. The engine requests headers from peers, in reverse till the parent hash matches the local chain
 3. The engine catches up:
@@ -291,7 +291,7 @@ the operation within the engine is the exact same as with L1 (although with an E
 2. The rollup node maintains latest head from engine (poll `eth_getBlockByNumber` and/or maintain a header subscription)
 3. The rollup node activates sync if the engine is out of sync but not syncing through P2P (`eth_syncing`)
 4. The rollup node inserts blocks, derived from L1, one by one, potentially adapting to L1 reorg(s),
-   as outlined in the [rollup node spec] (`engine_forkchoiceUpdatedV1`, `engine_newPayloadV1`)
+   as outlined in the [rollup node spec] (`engine_forkchoiceUpdatedV2`, `engine_newPayloadV2`)
 
 [rollup node spec]: rollup-node.md
 
@@ -303,8 +303,8 @@ the operation within the engine is the exact same as with L1 (although with an E
 [l1-api-spec]: https://github.com/ethereum/execution-apis/blob/769c53c94c4e487337ad0edea9ee0dce49c79bfa/src/engine/specification.md
 [PayloadAttributesV1]: https://github.com/ethereum/execution-apis/blob/769c53c94c4e487337ad0edea9ee0dce49c79bfa/src/engine/specification.md#PayloadAttributesV1
 [ExecutionPayloadV1]: https://github.com/ethereum/execution-apis/blob/769c53c94c4e487337ad0edea9ee0dce49c79bfa/src/engine/specification.md#ExecutionPayloadV1
-[engine_forkchoiceUpdatedV1]: https://github.com/ethereum/execution-apis/blob/769c53c94c4e487337ad0edea9ee0dce49c79bfa/src/engine/specification.md#engine_forkchoiceupdatedv1
-[engine_newPayloadV1]: https://github.com/ethereum/execution-apis/blob/769c53c94c4e487337ad0edea9ee0dce49c79bfa/src/engine/specification.md#engine_newPayloadV1
-[engine_getPayloadV1]: https://github.com/ethereum/execution-apis/blob/769c53c94c4e487337ad0edea9ee0dce49c79bfa/src/engine/specification.md#engine_getPayloadV1
+[engine_forkchoiceUpdatedV2]: https://github.com/ethereum/execution-apis/blob/584905270d8ad665718058060267061ecfd79ca5/src/engine/shanghai.md#engine_forkchoiceupdatedv2
+[engine_newPayloadV2]: https://github.com/ethereum/execution-apis/blob/584905270d8ad665718058060267061ecfd79ca5/src/engine/shanghai.md#engine_newpayloadv2
+[engine_getPayloadV2]: https://github.com/ethereum/execution-apis/blob/584905270d8ad665718058060267061ecfd79ca5/src/engine/shanghai.md#engine_getpayloadv2
 [HEX value encoding]: https://eth.wiki/json-rpc/API#hex-value-encoding
 [JSON-RPC-API]: https://github.com/ethereum/execution-apis

specs/overview.md

@@ -185,13 +185,13 @@ For each iteration of the block derivation loop described above, the rollup driv
 object and send it to the execution engine. The execution engine will then convert the payload attributes object into a
 block, and add it to the chain. The basic sequence the rollup driver is as follows:
 
-1. Call `engine_forkchoiceUpdatedV1` with the payload attributes object. We'll skip over the details of the fork choice
+1. Call `engine_forkchoiceUpdatedV2` with the payload attributes object. We'll skip over the details of the fork choice
 state parameter for now - just know that one of its fields is the L2 chain's `headBlockHash`, and that it is set to the
 block hash of the tip of the L2 chain. The Engine API returns a payload ID.
-2. Call `engine_getPayloadV1` with the payload ID returned in step 1. The engine API returns a payload object that
+2. Call `engine_getPayloadV2` with the payload ID returned in step 1. The engine API returns a payload object that
 includes a block hash as one of its fields.
-3. Call `engine_newPayloadV1` with the payload returned in step 2.
-4. Call `engine_forkchoiceUpdatedV1` with the fork choice parameter's `headBlockHash` set to the block hash returned in
+3. Call `engine_newPayloadV2` with the payload returned in step 2.
+4. Call `engine_forkchoiceUpdatedV2` with the fork choice parameter's `headBlockHash` set to the block hash returned in
 step 2. The tip of the L2 chain is now the block created in step 1.
 
 The swimlane diagram below visualizes the process:

specs/rollup-node-p2p.md

@@ -301,7 +301,7 @@ A node may apply the block to their local engine ahead of L1 availability, if it
 
 - The application of the block is reversible, in case of a conflict with delayed L1 information
 - The subsequent forkchoice-update ensures this block is recognized as "unsafe"
-  (see [`engine_forkchoiceUpdatedV1`](./exec-engine.md#engine_forkchoiceupdatedv1))
+  (see [`engine_forkchoiceUpdatedV2`](./exec-engine.md#engine_forkchoiceupdatedv2))
 
 ##### Block topic scoring parameters