Commit 2a13621b authored by mergify[bot]'s avatar mergify[bot] Committed by GitHub

Merge branch 'develop' into finalize-while-syncing

parents 9773c66e 712b95e8
......@@ -61,7 +61,7 @@ jobs:
yarn-monorepo:
docker:
- image: ethereumoptimism/ci-builder:latest
resource_class: xlarge
resource_class: large
steps:
- checkout
- check-changed:
......@@ -142,7 +142,7 @@ jobs:
default: "oplabs-tools-artifacts/images"
machine:
image: ubuntu-2204:2022.07.1
resource_class: xlarge
resource_class: medium
steps:
- checkout
- run:
......@@ -207,7 +207,7 @@ jobs:
default: "linux/amd64"
machine:
image: ubuntu-2204:2022.07.1
resource_class: xlarge
resource_class: medium
steps:
- gcp-oidc-authenticate
# Below is CircleCI recommended way of specifying nameservers on an Ubuntu box:
......@@ -261,7 +261,7 @@ jobs:
default: "linux/amd64"
machine:
image: ubuntu-2204:2022.07.1
resource_class: xlarge
resource_class: medium
steps:
- gcp-cli/install
- gcp-oidc-authenticate
......@@ -379,7 +379,7 @@ jobs:
contracts-bedrock-slither:
docker:
- image: ethereumoptimism/ci-builder:latest
resource_class: xlarge
resource_class: large
steps:
- checkout
- attach_workspace: { at: "." }
......@@ -600,7 +600,7 @@ jobs:
- run:
name: run lint
command: |
golangci-lint run -E goimports,sqlclosecheck,bodyclose,asciicheck,misspell,errorlint --timeout 2m -e "errors.As" -e "errors.Is" ./...
golangci-lint run -E goimports,sqlclosecheck,bodyclose,asciicheck,misspell,errorlint --timeout 5m -e "errors.As" -e "errors.Is" ./...
working_directory: <<parameters.module>>
go-test:
......@@ -900,7 +900,7 @@ jobs:
docker:
- image: returntocorp/semgrep
resource_class: xlarge
resource_class: medium
steps:
- checkout
- unless:
......@@ -942,7 +942,7 @@ jobs:
machine:
image: ubuntu-2204:2022.07.1
docker_layer_caching: true
resource_class: xlarge
resource_class: large
steps:
- attach_workspace:
at: /tmp/docker_images
......
......@@ -82,7 +82,7 @@ func (s *channelManager) TxFailed(id txID) {
}
s.metr.RecordBatchTxFailed()
if s.closed && len(s.confirmedTransactions) == 0 && len(s.pendingTransactions) == 0 {
if s.closed && len(s.confirmedTransactions) == 0 && len(s.pendingTransactions) == 0 && s.pendingChannel != nil {
s.log.Info("Channel has no submitted transactions, clearing for shutdown", "chID", s.pendingChannel.ID())
s.clearPendingChannel()
}
......
......@@ -191,7 +191,7 @@ func (l *BatchSubmitter) loadBlocksIntoState(ctx context.Context) {
if err != nil {
l.log.Warn("Error calculating L2 block range", "err", err)
return
} else if start.Number == end.Number {
} else if start.Number >= end.Number {
return
}
......
......@@ -64,6 +64,8 @@ type Metrics struct {
ChannelClosedReason prometheus.Gauge
ChannelNumFrames prometheus.Gauge
ChannelComprRatio prometheus.Histogram
ChannelInputBytesTotal prometheus.Counter
ChannelOutputBytesTotal prometheus.Counter
BatcherTxEvs opmetrics.EventVec
}
......@@ -144,6 +146,16 @@ func NewMetrics(procName string) *Metrics {
Help: "Compression ratios of closed channel.",
Buckets: append([]float64{0.1, 0.2}, prometheus.LinearBuckets(0.3, 0.05, 14)...),
}),
ChannelInputBytesTotal: factory.NewCounter(prometheus.CounterOpts{
Namespace: ns,
Name: "input_bytes_total",
Help: "Total number of bytes to a channel.",
}),
ChannelOutputBytesTotal: factory.NewCounter(prometheus.CounterOpts{
Namespace: ns,
Name: "output_bytes_total",
Help: "Total number of compressed output bytes from a channel.",
}),
BatcherTxEvs: opmetrics.NewEventVec(factory, ns, "", "batcher_tx", "BatcherTx", []string{"stage"}),
}
......@@ -219,6 +231,8 @@ func (m *Metrics) RecordChannelClosed(id derive.ChannelID, numPendingBlocks int,
m.ChannelNumFrames.Set(float64(numFrames))
m.ChannelInputBytes.WithLabelValues(StageClosed).Set(float64(inputBytes))
m.ChannelOutputBytes.Set(float64(outputComprBytes))
m.ChannelInputBytesTotal.Add(float64(inputBytes))
m.ChannelOutputBytesTotal.Add(float64(outputComprBytes))
var comprRatio float64
if inputBytes > 0 {
......
......@@ -139,12 +139,11 @@ func TestLargeL1Gaps(gt *testing.T) {
signer := types.LatestSigner(sd.L2Cfg.Config)
cl := sequencerEngine.EthClient()
aliceNonce := uint64(0) // manual nonce, avoid pending-tx nonce management, that causes flakes
aliceTx := func() {
n, err := cl.PendingNonceAt(t.Ctx(), dp.Addresses.Alice)
require.NoError(t, err)
tx := types.MustSignNewTx(dp.Secrets.Alice, signer, &types.DynamicFeeTx{
ChainID: sd.L2Cfg.Config.ChainID,
Nonce: n,
Nonce: aliceNonce,
GasTipCap: big.NewInt(2 * params.GWei),
GasFeeCap: new(big.Int).Add(miner.l1Chain.CurrentBlock().BaseFee, big.NewInt(2*params.GWei)),
Gas: params.TxGas,
......@@ -152,6 +151,7 @@ func TestLargeL1Gaps(gt *testing.T) {
Value: e2eutils.Ether(2),
})
require.NoError(gt, cl.SendTransaction(t.Ctx(), tx))
aliceNonce += 1
}
makeL2BlockWithAliceTx := func() {
aliceTx()
......
......@@ -191,7 +191,7 @@ func TestL2EngineAPIFail(gt *testing.T) {
}
func TestEngineAPITests(t *testing.T) {
test.RunEngineAPITests(t, func() engineapi.EngineBackend {
test.RunEngineAPITests(t, func(t *testing.T) engineapi.EngineBackend {
jwtPath := e2eutils.WriteDefaultJWT(t)
dp := e2eutils.MakeDeployParams(t, defaultRollupTestParams)
sd := e2eutils.Setup(t, dp, defaultAlloc)
......
......@@ -589,8 +589,10 @@ func TestSystemMockP2P(t *testing.T) {
}
cfg := DefaultSystemConfig(t)
// Disable batcher, so we don't sync from L1
// Disable batcher, so we don't sync from L1 & set a large sequence window so we only have unsafe blocks
cfg.DisableBatcher = true
cfg.DeployConfig.SequencerWindowSize = 100_000
cfg.DeployConfig.MaxSequencerDrift = 100_000
// disable at the start, so we don't miss any gossiped blocks.
cfg.Nodes["sequencer"].Driver.SequencerStopped = true
......
......@@ -22,6 +22,7 @@ type OracleBackedL2Chain struct {
oracle Oracle
chainCfg *params.ChainConfig
engine consensus.Engine
oracleHead *types.Header
head *types.Header
safe *types.Header
finalized *types.Header
......@@ -47,6 +48,7 @@ func NewOracleBackedL2Chain(logger log.Logger, oracle Oracle, chainCfg *params.C
head: head.Header(),
safe: head.Header(),
finalized: head.Header(),
oracleHead: head.Header(),
blocks: make(map[common.Hash]*types.Block),
db: NewOracleBackedDB(oracle),
}, nil
......@@ -82,11 +84,7 @@ func (o *OracleBackedL2Chain) CurrentFinalBlock() *types.Header {
}
func (o *OracleBackedL2Chain) GetHeaderByHash(hash common.Hash) *types.Header {
block := o.GetBlockByHash(hash)
if block == nil {
return nil
}
return block.Header()
return o.GetBlockByHash(hash).Header()
}
func (o *OracleBackedL2Chain) GetBlockByHash(hash common.Hash) *types.Block {
......@@ -96,15 +94,18 @@ func (o *OracleBackedL2Chain) GetBlockByHash(hash common.Hash) *types.Block {
return block
}
// Retrieve from the oracle
block = o.oracle.BlockByHash(hash)
if block == nil {
return nil
}
return block
return o.oracle.BlockByHash(hash)
}
func (o *OracleBackedL2Chain) GetBlock(hash common.Hash, number uint64) *types.Block {
block := o.GetBlockByHash(hash)
var block *types.Block
if o.oracleHead.Number.Uint64() < number {
// For blocks above the chain head, only consider newly built blocks
// Avoids requesting an unknown block from the oracle which would panic.
block = o.blocks[hash]
} else {
block = o.GetBlockByHash(hash)
}
if block == nil {
return nil
}
......@@ -116,9 +117,6 @@ func (o *OracleBackedL2Chain) GetBlock(hash common.Hash, number uint64) *types.B
func (o *OracleBackedL2Chain) GetHeader(hash common.Hash, u uint64) *types.Header {
block := o.GetBlock(hash, u)
if block == nil {
return nil
}
return block.Header()
}
......
......@@ -42,17 +42,6 @@ func TestGetBlocks(t *testing.T) {
}
}
func TestUnknownBlock(t *testing.T) {
_, chain := setupOracleBackedChain(t, 1)
hash := common.HexToHash("0x556677881122")
blockNumber := uint64(1)
require.Nil(t, chain.GetBlockByHash(hash))
require.Nil(t, chain.GetHeaderByHash(hash))
require.Nil(t, chain.GetBlock(hash, blockNumber))
require.Nil(t, chain.GetHeader(hash, blockNumber))
require.False(t, chain.HasBlockAndState(hash, blockNumber))
}
func TestCanonicalHashNotFoundPastChainHead(t *testing.T) {
blocks, chain := setupOracleBackedChainWithLowerHead(t, 5, 3)
......@@ -69,7 +58,7 @@ func TestCanonicalHashNotFoundPastChainHead(t *testing.T) {
func TestAppendToChain(t *testing.T) {
blocks, chain := setupOracleBackedChainWithLowerHead(t, 4, 3)
newBlock := blocks[4]
require.Nil(t, chain.GetBlockByHash(newBlock.Hash()), "block unknown before being added")
require.Nil(t, chain.GetBlock(newBlock.Hash(), newBlock.NumberU64()), "block unknown before being added")
require.NoError(t, chain.InsertBlockWithoutSetHead(newBlock))
require.Equal(t, blocks[3].Header(), chain.CurrentHeader(), "should not update chain head yet")
......@@ -113,9 +102,7 @@ func TestUpdateStateDatabaseWhenImportingBlock(t *testing.T) {
require.NotEqual(t, blocks[1].Root(), newBlock.Root(), "block should have modified world state")
require.Panics(t, func() {
_, _ = chain.StateAt(newBlock.Root())
}, "state from non-imported block should not be available")
require.False(t, chain.HasBlockAndState(newBlock.Root(), newBlock.NumberU64()), "state from non-imported block should not be available")
err = chain.InsertBlockWithoutSetHead(newBlock)
require.NoError(t, err)
......@@ -181,7 +168,7 @@ func setupOracle(t *testing.T, blockCount int, headBlockNumber int) (*params.Cha
genesisBlock := l2Genesis.MustCommit(db)
blocks, _ := core.GenerateChain(chainCfg, genesisBlock, consensus, db, blockCount, func(i int, gen *core.BlockGen) {})
blocks = append([]*types.Block{genesisBlock}, blocks...)
oracle := newStubBlockOracle(blocks[:headBlockNumber+1], db)
oracle := newStubBlockOracle(t, blocks[:headBlockNumber+1], db)
return chainCfg, blocks, oracle
}
......@@ -213,23 +200,27 @@ type stubBlockOracle struct {
kvStateOracle
}
func newStubBlockOracle(chain []*types.Block, db ethdb.Database) *stubBlockOracle {
func newStubBlockOracle(t *testing.T, chain []*types.Block, db ethdb.Database) *stubBlockOracle {
blocks := make(map[common.Hash]*types.Block, len(chain))
for _, block := range chain {
blocks[block.Hash()] = block
}
return &stubBlockOracle{
blocks: blocks,
kvStateOracle: kvStateOracle{source: db},
kvStateOracle: kvStateOracle{t: t, source: db},
}
}
func (o stubBlockOracle) BlockByHash(blockHash common.Hash) *types.Block {
return o.blocks[blockHash]
block, ok := o.blocks[blockHash]
if !ok {
o.t.Fatalf("requested unknown block %s", blockHash)
}
return block
}
func TestEngineAPITests(t *testing.T) {
test.RunEngineAPITests(t, func() engineapi.EngineBackend {
test.RunEngineAPITests(t, func(t *testing.T) engineapi.EngineBackend {
_, chain := setupOracleBackedChain(t, 0)
return chain
})
......
......@@ -301,7 +301,7 @@ func (ea *L2EngineAPI) NewPayloadV1(ctx context.Context, payload *eth.ExecutionP
}
// If we already have the block locally, ignore the entire execution and just
// return a fake success.
if block := ea.backend.GetBlockByHash(payload.BlockHash); block != nil {
if block := ea.backend.GetBlock(payload.BlockHash, uint64(payload.BlockNumber)); block != nil {
ea.log.Warn("Ignoring already known beacon payload", "number", payload.BlockNumber, "hash", payload.BlockHash, "age", common.PrettyAge(time.Unix(int64(block.Time()), 0)))
hash := block.Hash()
return &eth.PayloadStatusV1{Status: eth.ExecutionValid, LatestValidHash: &hash}, nil
......
......@@ -18,7 +18,7 @@ import (
var gasLimit = eth.Uint64Quantity(30_000_000)
var feeRecipient = common.Address{}
func RunEngineAPITests(t *testing.T, createBackend func() engineapi.EngineBackend) {
func RunEngineAPITests(t *testing.T, createBackend func(t *testing.T) engineapi.EngineBackend) {
t.Run("CreateBlock", func(t *testing.T) {
api := newTestHelper(t, createBackend)
......@@ -292,10 +292,10 @@ type testHelper struct {
assert *require.Assertions
}
func newTestHelper(t *testing.T, createBackend func() engineapi.EngineBackend) *testHelper {
func newTestHelper(t *testing.T, createBackend func(t *testing.T) engineapi.EngineBackend) *testHelper {
logger := testlog.Logger(t, log.LvlDebug)
ctx := context.Background()
backend := createBackend()
backend := createBackend(t)
api := engineapi.NewL2EngineAPI(logger, backend)
test := &testHelper{
t: t,
......
......@@ -4,6 +4,7 @@ import (
"context"
"fmt"
"github.com/ethereum-optimism/optimism/op-node/eth"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/common/hexutil"
"github.com/ethereum/go-ethereum/core/rawdb"
......@@ -24,19 +25,25 @@ type CallContext interface {
type FetchingL2Oracle struct {
ctx context.Context
logger log.Logger
head eth.BlockInfo
blockSource BlockSource
callContext CallContext
}
func NewFetchingL2Oracle(ctx context.Context, logger log.Logger, l2Url string) (*FetchingL2Oracle, error) {
func NewFetchingL2Oracle(ctx context.Context, logger log.Logger, l2Url string, l2Head common.Hash) (*FetchingL2Oracle, error) {
rpcClient, err := rpc.Dial(l2Url)
if err != nil {
return nil, err
}
ethClient := ethclient.NewClient(rpcClient)
head, err := ethClient.HeaderByHash(ctx, l2Head)
if err != nil {
return nil, fmt.Errorf("retrieve l2 head %v: %w", l2Head, err)
}
return &FetchingL2Oracle{
ctx: ctx,
logger: logger,
head: eth.HeaderBlockInfo(head),
blockSource: ethClient,
callContext: rpcClient,
}, nil
......@@ -78,5 +85,8 @@ func (o *FetchingL2Oracle) BlockByHash(blockHash common.Hash) *types.Block {
if err != nil {
panic(fmt.Errorf("fetch block %s: %w", blockHash.Hex(), err))
}
if block.NumberU64() > o.head.NumberU64() {
panic(fmt.Errorf("fetched block %v number %d above head block number %d", blockHash, block.NumberU64(), o.head.NumberU64()))
}
return block
}
......@@ -5,12 +5,14 @@ import (
"encoding/json"
"errors"
"fmt"
"math/big"
"math/rand"
"reflect"
"testing"
"github.com/ethereum-optimism/optimism/op-node/testutils"
cll2 "github.com/ethereum-optimism/optimism/op-program/client/l2"
"github.com/ethereum/go-ethereum/trie"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/common/hexutil"
......@@ -23,35 +25,7 @@ import (
// Require the fetching oracle to implement StateOracle
var _ cll2.StateOracle = (*FetchingL2Oracle)(nil)
type callContextRequest struct {
ctx context.Context
method string
args []interface{}
}
type stubCallContext struct {
nextResult any
nextErr error
requests []callContextRequest
}
func (c *stubCallContext) CallContext(ctx context.Context, result any, method string, args ...interface{}) error {
if result != nil && reflect.TypeOf(result).Kind() != reflect.Ptr {
return fmt.Errorf("call result parameter must be pointer or nil interface: %v", result)
}
c.requests = append(c.requests, callContextRequest{ctx: ctx, method: method, args: args})
if c.nextErr != nil {
return c.nextErr
}
res, err := json.Marshal(c.nextResult)
if err != nil {
return fmt.Errorf("json marshal: %w", err)
}
err = json.Unmarshal(res, result)
if err != nil {
return fmt.Errorf("json unmarshal: %w", err)
}
return nil
}
const headBlockNumber = 1000
func TestNodeByHash(t *testing.T) {
rng := rand.New(rand.NewSource(1234))
......@@ -152,31 +126,12 @@ func TestCodeByHash(t *testing.T) {
})
}
type blockRequest struct {
ctx context.Context
blockHash common.Hash
}
type stubBlockSource struct {
requests []blockRequest
nextErr error
nextResult *types.Block
}
func (s *stubBlockSource) BlockByHash(ctx context.Context, blockHash common.Hash) (*types.Block, error) {
s.requests = append(s.requests, blockRequest{
ctx: ctx,
blockHash: blockHash,
})
return s.nextResult, s.nextErr
}
func TestBlockByHash(t *testing.T) {
rng := rand.New(rand.NewSource(1234))
hash := testutils.RandomHash(rng)
t.Run("Success", func(t *testing.T) {
block, _ := testutils.RandomBlock(rng, 1)
block := blockWithNumber(rng, headBlockNumber-1)
stub := &stubBlockSource{nextResult: block}
fetcher := newFetcher(stub, nil)
......@@ -194,7 +149,7 @@ func TestBlockByHash(t *testing.T) {
})
t.Run("RequestArgs", func(t *testing.T) {
stub := &stubBlockSource{}
stub := &stubBlockSource{nextResult: blockWithNumber(rng, 1)}
fetcher := newFetcher(stub, nil)
fetcher.BlockByHash(hash)
......@@ -203,11 +158,86 @@ func TestBlockByHash(t *testing.T) {
req := stub.requests[0]
require.Equal(t, hash, req.blockHash)
})
t.Run("PanicWhenBlockAboveHeadRequested", func(t *testing.T) {
// Block that the source can provide but is above the head block number
block := blockWithNumber(rng, headBlockNumber+1)
stub := &stubBlockSource{nextResult: block}
fetcher := newFetcher(stub, nil)
require.Panics(t, func() {
fetcher.BlockByHash(block.Hash())
})
})
}
func blockWithNumber(rng *rand.Rand, num int64) *types.Block {
header := testutils.RandomHeader(rng)
header.Number = big.NewInt(num)
return types.NewBlock(header, nil, nil, nil, trie.NewStackTrie(nil))
}
type blockRequest struct {
ctx context.Context
blockHash common.Hash
}
type stubBlockSource struct {
requests []blockRequest
nextErr error
nextResult *types.Block
}
func (s *stubBlockSource) BlockByHash(ctx context.Context, blockHash common.Hash) (*types.Block, error) {
s.requests = append(s.requests, blockRequest{
ctx: ctx,
blockHash: blockHash,
})
return s.nextResult, s.nextErr
}
type callContextRequest struct {
ctx context.Context
method string
args []interface{}
}
type stubCallContext struct {
nextResult any
nextErr error
requests []callContextRequest
}
func (c *stubCallContext) CallContext(ctx context.Context, result any, method string, args ...interface{}) error {
if result != nil && reflect.TypeOf(result).Kind() != reflect.Ptr {
return fmt.Errorf("call result parameter must be pointer or nil interface: %v", result)
}
c.requests = append(c.requests, callContextRequest{ctx: ctx, method: method, args: args})
if c.nextErr != nil {
return c.nextErr
}
res, err := json.Marshal(c.nextResult)
if err != nil {
return fmt.Errorf("json marshal: %w", err)
}
err = json.Unmarshal(res, result)
if err != nil {
return fmt.Errorf("json unmarshal: %w", err)
}
return nil
}
func newFetcher(blockSource BlockSource, callContext CallContext) *FetchingL2Oracle {
rng := rand.New(rand.NewSource(int64(1)))
head := testutils.MakeBlockInfo(func(i *testutils.MockBlockInfo) {
i.InfoNum = headBlockNumber
})(rng)
return &FetchingL2Oracle{
ctx: context.Background(),
logger: log.New(),
head: head,
blockSource: blockSource,
callContext: callContext,
}
......
......@@ -19,7 +19,7 @@ func NewFetchingEngine(ctx context.Context, logger log.Logger, cfg *config.Confi
if err != nil {
return nil, err
}
oracle, err := NewFetchingL2Oracle(ctx, logger, cfg.L2URL)
oracle, err := NewFetchingL2Oracle(ctx, logger, cfg.L2URL, cfg.L2Head)
if err != nil {
return nil, fmt.Errorf("connect l2 oracle: %w", err)
}
......
......@@ -67,10 +67,10 @@ type ETHBackend interface {
// NonceAt returns the account nonce of the given account.
// The block number can be nil, in which case the nonce is taken from the latest known block.
NonceAt(ctx context.Context, account common.Address, blockNumber *big.Int) (uint64, error)
// PendingNonce returns the pending nonce.
// PendingNonceAt returns the pending nonce.
PendingNonceAt(ctx context.Context, account common.Address) (uint64, error)
/// EstimateGas returns an estimate of the amount of gas needed to execute the given
/// transaction against the current pending block.
// EstimateGas returns an estimate of the amount of gas needed to execute the given
// transaction against the current pending block.
EstimateGas(ctx context.Context, msg ethereum.CallMsg) (uint64, error)
}
......
......@@ -207,7 +207,7 @@ function deleteL2Outputs(uint256 _l2OutputIndex) external
/**
* @notice Computes the block number of the next L2 block that needs to be checkpointed.
*/
function getNextBlockNumber() public view returns (uint256)
function nextBlockNumber() public view returns (uint256)
```
### Configuration
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment