Commit e38c5f43 authored by protolambda's avatar protolambda Committed by GitHub

Safe head fix 2 (#3382)

* op-node: sync start update func signature, port over tests from safe-head-fix pr

* op-node: sync start now starts with current heads by label

* op-node: single loop find-heads

* op-node: sync - clean up go doc

* op-node: use sync fn in engine queue

* op-node: fix engine queue finalization test

* op-node: sync off-by-one fix

* op-node: fix highest l2 block with canon origin, need to reset when l1 reorg

* op-node: handle non-standard safe/finalized not found errors

* op-node: sync start review fixes / comment typos

* op-node: seq window size check with l1 origin, update engine queue test with extra l1 origin, now that off by 1 fix applies properly to chains with multiple L2 blocks per L1 block

* op-node: start from parent block before seq nr 0

* Update op-node/rollup/sync/start.go
Co-authored-by: default avatarJoshua Gutow <jgutow@optimism.io>

* review fixes

* fix lint
Co-authored-by: default avatarJoshua Gutow <jgutow@optimism.io>
Co-authored-by: default avatarMatthew Slipper <me@matthewslipper.com>
parent 7231a0cb
......@@ -10,7 +10,6 @@ import (
"github.com/ethereum-optimism/optimism/op-node/eth"
"github.com/ethereum-optimism/optimism/op-node/rollup"
"github.com/ethereum-optimism/optimism/op-node/rollup/sync"
"github.com/ethereum/go-ethereum"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/common/hexutil"
"github.com/ethereum/go-ethereum/core/types"
......@@ -384,23 +383,11 @@ func (eq *EngineQueue) forceNextSafeAttributes(ctx context.Context) error {
// ResetStep Walks the L2 chain backwards until it finds an L2 block whose L1 origin is canonical.
// The unsafe head is set to the head of the L2 chain, unless the existing safe head is not canonical.
func (eq *EngineQueue) ResetStep(ctx context.Context, l1Fetcher L1Fetcher) error {
finalized, err := eq.engine.L2BlockRefByLabel(ctx, eth.Finalized)
if errors.Is(err, ethereum.NotFound) {
// default to genesis if we have not finalized anything before.
finalized, err = eq.engine.L2BlockRefByHash(ctx, eq.cfg.Genesis.L2.Hash)
}
if err != nil {
return NewTemporaryError(fmt.Errorf("failed to find the finalized L2 block: %w", err))
}
// TODO: this should be resetting using the safe head instead. Out of scope for L2 client bindings PR.
prevUnsafe, err := eq.engine.L2BlockRefByLabel(ctx, eth.Unsafe)
if err != nil {
return NewTemporaryError(fmt.Errorf("failed to find the L2 Head block: %w", err))
}
unsafe, safe, err := sync.FindL2Heads(ctx, prevUnsafe, eq.cfg.SeqWindowSize, l1Fetcher, eq.engine, &eq.cfg.Genesis)
result, err := sync.FindL2Heads(ctx, eq.cfg, l1Fetcher, eq.engine)
if err != nil {
return NewTemporaryError(fmt.Errorf("failed to find the L2 Heads to start from: %w", err))
}
finalized, safe, unsafe := result.Finalized, result.Safe, result.Unsafe
l1Origin, err := l1Fetcher.L1BlockRefByHash(ctx, safe.L1Origin.Hash)
if err != nil {
return NewTemporaryError(fmt.Errorf("failed to fetch the new L1 progress: origin: %v; err: %w", safe.L1Origin, err))
......
......@@ -17,20 +17,6 @@ func TestEngineQueue_Finalize(t *testing.T) {
logger := testlog.Logger(t, log.LvlInfo)
rng := rand.New(rand.NewSource(1234))
// create a short test L2 chain:
//
// L2:
// A0: genesis
// A1: finalized, incl in B
// B0: safe, incl in C
// B1: not yet included in L1
// C0: head, not included in L1 yet
//
// L1:
// A: genesis
// B: finalized, incl A1
// C: safe, incl B0
// D: unsafe, not yet referenced by L2
l1Time := uint64(2)
refA := testutils.RandomBlockRef(rng)
......@@ -53,6 +39,18 @@ func TestEngineQueue_Finalize(t *testing.T) {
ParentHash: refC.Hash,
Time: refC.Time + l1Time,
}
refE := eth.L1BlockRef{
Hash: testutils.RandomHash(rng),
Number: refD.Number + 1,
ParentHash: refD.Hash,
Time: refD.Time + l1Time,
}
refF := eth.L1BlockRef{
Hash: testutils.RandomHash(rng),
Number: refE.Number + 1,
ParentHash: refE.Hash,
Time: refE.Time + l1Time,
}
refA0 := eth.L2BlockRef{
Hash: testutils.RandomHash(rng),
......@@ -103,42 +101,138 @@ func TestEngineQueue_Finalize(t *testing.T) {
L1Origin: refC.ID(),
SequenceNumber: 0,
}
refC1 := eth.L2BlockRef{
Hash: testutils.RandomHash(rng),
Number: refC0.Number + 1,
ParentHash: refC0.Hash,
Time: refC0.Time + cfg.BlockTime,
L1Origin: refC.ID(),
SequenceNumber: 1,
}
refD0 := eth.L2BlockRef{
Hash: testutils.RandomHash(rng),
Number: refC1.Number + 1,
ParentHash: refC1.Hash,
Time: refC1.Time + cfg.BlockTime,
L1Origin: refD.ID(),
SequenceNumber: 0,
}
refD1 := eth.L2BlockRef{
Hash: testutils.RandomHash(rng),
Number: refD0.Number + 1,
ParentHash: refD0.Hash,
Time: refD0.Time + cfg.BlockTime,
L1Origin: refD.ID(),
SequenceNumber: 1,
}
refE0 := eth.L2BlockRef{
Hash: testutils.RandomHash(rng),
Number: refD1.Number + 1,
ParentHash: refD1.Hash,
Time: refD1.Time + cfg.BlockTime,
L1Origin: refE.ID(),
SequenceNumber: 0,
}
refE1 := eth.L2BlockRef{
Hash: testutils.RandomHash(rng),
Number: refE0.Number + 1,
ParentHash: refE0.Hash,
Time: refE0.Time + cfg.BlockTime,
L1Origin: refE.ID(),
SequenceNumber: 1,
}
refF0 := eth.L2BlockRef{
Hash: testutils.RandomHash(rng),
Number: refE1.Number + 1,
ParentHash: refE1.Hash,
Time: refE1.Time + cfg.BlockTime,
L1Origin: refF.ID(),
SequenceNumber: 0,
}
refF1 := eth.L2BlockRef{
Hash: testutils.RandomHash(rng),
Number: refF0.Number + 1,
ParentHash: refF0.Hash,
Time: refF0.Time + cfg.BlockTime,
L1Origin: refF.ID(),
SequenceNumber: 1,
}
t.Log("refA", refA.Hash)
t.Log("refB", refB.Hash)
t.Log("refC", refC.Hash)
t.Log("refD", refD.Hash)
t.Log("refE", refE.Hash)
t.Log("refF", refF.Hash)
t.Log("refA0", refA0.Hash)
t.Log("refA1", refA1.Hash)
t.Log("refB0", refB0.Hash)
t.Log("refB1", refB1.Hash)
t.Log("refC0", refC0.Hash)
t.Log("refC1", refC1.Hash)
t.Log("refD0", refD0.Hash)
t.Log("refD1", refD1.Hash)
t.Log("refE0", refE0.Hash)
t.Log("refE1", refE1.Hash)
t.Log("refF0", refF0.Hash)
t.Log("refF1", refF1.Hash)
metrics := &TestMetrics{}
eng := &testutils.MockEngine{}
eng.ExpectL2BlockRefByLabel(eth.Finalized, refA1, nil)
// TODO(Proto): update expectation once we're using safe block label properly for sync starting point
eng.ExpectL2BlockRefByLabel(eth.Unsafe, refC0, nil)
// we find the common point to initialize to by comparing the L1 origins in the L2 chain with the L1 chain
l1F := &testutils.MockL1Source{}
l1F.ExpectL1BlockRefByLabel(eth.Unsafe, refD, nil)
l1F.ExpectL1BlockRefByNumber(refC0.L1Origin.Number, refC, nil)
eng.ExpectL2BlockRefByHash(refC0.ParentHash, refB1, nil) // good L1 origin
eng.ExpectL2BlockRefByHash(refB1.ParentHash, refB0, nil) // need a block with seqnr == 0, don't stop at above
l1F.ExpectL1BlockRefByHash(refB0.L1Origin.Hash, refB, nil) // the origin of the safe L2 head will be the L1 starting point for derivation.
eq := NewEngineQueue(logger, cfg, eng, metrics)
require.NoError(t, RepeatResetStep(t, eq.ResetStep, l1F, 3))
eng.ExpectL2BlockRefByLabel(eth.Finalized, refA1, nil)
eng.ExpectL2BlockRefByLabel(eth.Safe, refE0, nil)
eng.ExpectL2BlockRefByLabel(eth.Unsafe, refF1, nil)
// TODO(proto): this is changing, needs to be a sequence window ago, but starting traversal back from safe block,
// safe blocks with canon origin are good, but we go back a full window to ensure they are all included in L1,
// by forcing them to be consolidated with L1 again.
require.Equal(t, eq.SafeL2Head(), refB0, "L2 reset should go back to sequence window ago")
// unsafe
l1F.ExpectL1BlockRefByNumber(refF.Number, refF, nil)
eng.ExpectL2BlockRefByHash(refF1.ParentHash, refF0, nil)
eng.ExpectL2BlockRefByHash(refF0.ParentHash, refE1, nil)
// meet previous safe, counts 1/2
l1F.ExpectL1BlockRefByNumber(refE.Number, refE, nil)
eng.ExpectL2BlockRefByHash(refE1.ParentHash, refE0, nil)
eng.ExpectL2BlockRefByHash(refE0.ParentHash, refD1, nil)
// now full seq window, inclusive
l1F.ExpectL1BlockRefByNumber(refD.Number, refD, nil)
eng.ExpectL2BlockRefByHash(refD1.ParentHash, refD0, nil)
eng.ExpectL2BlockRefByHash(refD0.ParentHash, refC1, nil)
// now one more L1 origin
l1F.ExpectL1BlockRefByNumber(refC.Number, refC, nil)
eng.ExpectL2BlockRefByHash(refC1.ParentHash, refC0, nil)
// parent of that origin will be considered safe
eng.ExpectL2BlockRefByHash(refC0.ParentHash, refB1, nil)
// and we fetch the L1 origin of that as starting point for engine queue
l1F.ExpectL1BlockRefByHash(refB.Hash, refB, nil)
eq := NewEngineQueue(logger, cfg, eng, metrics)
require.NoError(t, RepeatResetStep(t, eq.ResetStep, l1F, 20))
require.Equal(t, refB1, eq.SafeL2Head(), "L2 reset should go back to sequence window ago: blocks with origin E and D are not safe until we reconcile, C is extra, and B1 is the end we look for")
require.Equal(t, refB, eq.Progress().Origin, "Expecting to be set back derivation L1 progress to B")
require.Equal(t, refA1, eq.Finalized(), "A1 is recognized as finalized before we run any steps")
// we are not adding blocks in this test,
// but we can still trigger post-processing for the already existing safe head,
// so the engine can prepare to finalize that.
// now say C1 was included in D and became the new safe head
eq.progress.Origin = refD
eq.safeHead = refC1
eq.postProcessSafeL2()
// now say D0 was included in E and became the new safe head
eq.progress.Origin = refE
eq.safeHead = refD0
eq.postProcessSafeL2()
// let's finalize C, which included B0, but not B1
eq.Finalize(refC.ID())
// let's finalize D (current L1), from which we fully derived C1 (it was safe head), but not D0 (included in E)
eq.Finalize(refD.ID())
// Now a few steps later, without consuming any additional L1 inputs,
// we should be able to resolve that B0 is now finalized
// we should be able to resolve that B1 is now finalized, since it was included in finalized L1 block C
require.NoError(t, RepeatStep(t, eq.Step, eq.progress, 10))
require.Equal(t, refB0, eq.Finalized(), "B0 was included in finalized C, and should now be finalized")
require.Equal(t, refC1, eq.Finalized(), "C1 was included in finalized D, and should now be finalized")
l1F.AssertExpectations(t)
eng.AssertExpectations(t)
......
// The sync package is responsible for reconciling L1 and L2.
// Package sync is responsible for reconciling L1 and L2.
//
// The Ethereum chain is a DAG of blocks with the root block being the genesis block. At any given
// time, the head (or tip) of the chain can change if an offshoot/branch of the chain has a higher
......@@ -19,19 +19,6 @@
// During normal operation, both the L1 and L2 canonical chains can change, due to a re-organisation
// or due to an extension (new L1 or L2 block).
//
// When one of these changes occurs, the rollup node needs to determine what the new L2 head blocks
// should be. We track two L2 head blocks:
//
// - The *unsafe L2 block*: This is the highest L2 block whose L1 origin is a plausible (1)
// extension of the canonical L1 chain (as known to the op-node).
// - The *safe L2 block*: This is the highest L2 block whose epoch's sequencing window is
// complete within the canonical L1 chain (as known to the op-node).
//
// (1) Plausible meaning that the blockhash of the L2 block's L1 origin (as reported in the L1
//
// Attributes deposit within the L2 block) is not canonical at another height in the L1 chain,
// and the same holds for all its ancestors.
//
// In particular, in the case of L1 extension, the L2 unsafe head will generally remain the same,
// but in the case of an L1 re-org, we need to search for the new safe and unsafe L2 block.
package sync
......@@ -41,163 +28,203 @@ import (
"errors"
"fmt"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum-optimism/optimism/op-node/eth"
"github.com/ethereum-optimism/optimism/op-node/rollup"
"github.com/ethereum/go-ethereum"
"github.com/ethereum/go-ethereum/common"
)
type L1Chain interface {
L1BlockRefByLabel(ctx context.Context, label eth.BlockLabel) (eth.L1BlockRef, error)
L1BlockRefByNumber(ctx context.Context, number uint64) (eth.L1BlockRef, error)
L1BlockRefByHash(ctx context.Context, hash common.Hash) (eth.L1BlockRef, error)
}
type L2Chain interface {
L2BlockRefByHash(ctx context.Context, l2Hash common.Hash) (eth.L2BlockRef, error)
L2BlockRefByLabel(ctx context.Context, label eth.BlockLabel) (eth.L2BlockRef, error)
}
var ReorgFinalizedErr = errors.New("cannot reorg finalized block")
var WrongChainErr = errors.New("wrong chain")
var TooDeepReorgErr = errors.New("reorg is too deep")
const MaxReorgDepth = 500
// isCanonical returns the following values:
// - `aheadOrCanonical: true if the supplied block is ahead of the known head of the L1 chain,
// or canonical in the L1 chain.
// - `canonical`: true if the block is canonical in the L1 chain.
func isAheadOrCanonical(ctx context.Context, l1 L1Chain, block eth.BlockID) (aheadOrCanonical bool, canonical bool, err error) {
if l1Head, err := l1.L1BlockRefByLabel(ctx, eth.Unsafe); err != nil {
return false, false, err
} else if block.Number > l1Head.Number {
return true, false, nil
} else if canonical, err := l1.L1BlockRefByNumber(ctx, block.Number); err != nil {
return false, false, err
} else {
canonical := canonical.Hash == block.Hash
return canonical, canonical, nil
type FindHeadsResult struct {
Unsafe eth.L2BlockRef
Safe eth.L2BlockRef
Finalized eth.L2BlockRef
}
// currentHeads returns the current finalized, safe and unsafe heads of the execution engine.
// If nothing has been marked finalized yet, the finalized head defaults to the genesis block.
// If nothing has been marked safe yet, the safe head defaults to the finalized block.
func currentHeads(ctx context.Context, cfg *rollup.Config, l2 L2Chain) (*FindHeadsResult, error) {
finalized, err := l2.L2BlockRefByLabel(ctx, eth.Finalized)
if errors.Is(err, ethereum.NotFound) {
// default to genesis if we have not finalized anything before.
finalized, err = l2.L2BlockRefByHash(ctx, cfg.Genesis.L2.Hash)
}
if err != nil {
return nil, fmt.Errorf("failed to find the finalized L2 block: %w", err)
}
safe, err := l2.L2BlockRefByLabel(ctx, eth.Safe)
if errors.Is(err, ethereum.NotFound) {
safe = finalized
} else if err != nil {
return nil, fmt.Errorf("failed to find the safe L2 block: %w", err)
}
unsafe, err := l2.L2BlockRefByLabel(ctx, eth.Unsafe)
if err != nil {
return nil, fmt.Errorf("failed to find the L2 head block: %w", err)
}
return &FindHeadsResult{
Unsafe: unsafe,
Safe: safe,
Finalized: finalized,
}, nil
}
// FindL2Heads walks back from `start` (the previous unsafe L2 block) and finds the unsafe and safe
// L2 blocks.
// FindL2Heads walks back from `start` (the previous unsafe L2 block) and finds
// the finalized, unsafe and safe L2 blocks.
//
// - The *unsafe L2 block*: This is the highest L2 block whose L1 origin is a plausible (1)
// - The *unsafe L2 block*: This is the highest L2 block whose L1 origin is a *plausible*
// extension of the canonical L1 chain (as known to the op-node).
// - The *safe L2 block*: This is the highest L2 block whose epoch's sequencing window is
// complete within the canonical L1 chain (as known to the op-node).
// - The *finalized L2 block*: This is the L2 block which is known to be fully derived from
// finalized L1 block data.
//
// (1) Plausible meaning that the blockhash of the L2 block's L1 origin (as reported in the L1
//
// Attributes deposit within the L2 block) is not canonical at another height in the L1 chain,
// and the same holds for all its ancestors.
func FindL2Heads(ctx context.Context, start eth.L2BlockRef, seqWindowSize uint64,
l1 L1Chain, l2 L2Chain, genesis *rollup.Genesis) (unsafe eth.L2BlockRef, safe eth.L2BlockRef, err error) {
// Plausible: meaning that the blockhash of the L2 block's L1 origin
// (as reported in the L1 Attributes deposit within the L2 block) is not canonical at another height in the L1 chain,
// and the same holds for all its ancestors.
func FindL2Heads(ctx context.Context, cfg *rollup.Config, l1 L1Chain, l2 L2Chain) (result *FindHeadsResult, err error) {
// Fetch current L2 forkchoice state
result, err = currentHeads(ctx, cfg, l2)
if err != nil {
return nil, fmt.Errorf("failed to fetch current L2 forkchoice state: %w", err)
}
// Loop 1. Walk the L2 chain backwards until we find an L2 block whose L1 origin is canonical.
// Remember original unsafe block to determine reorg depth
prevUnsafe := result.Unsafe
// Current L2 block.
n := start
// Number of blocks between n and start.
reorgDepth := 0
n := result.Unsafe
// Blockhash of L1 origin hash for the L2 block during the previous iteration, 0 for first
// iteration. When this changes as we walk the L2 chain backwards, it means we're seeing a different
// (earlier) epoch.
var prevL1OriginHash common.Hash
var highestL2WithCanonicalL1Origin eth.L2BlockRef // the highest L2 block with confirmed canonical L1 origin
var l1Block eth.L1BlockRef // the L1 block at the height of the L1 origin of the current L2 block n.
var ahead bool // when "n", the L2 block, has a L1 origin that is not visible in our L1 chain source yet
// The highest L2 ancestor of `start` (or `start` itself) whose ancestors are not (yet) known
// to have a non-canonical L1 origin. Empty if no such candidate is known yet. Guaranteed to be
// set after exiting from Loop 1.
var highestPlausibleCanonicalOrigin eth.L2BlockRef
ready := false // when we found the block after the safe head, and we just need to return the parent block.
// Each loop iteration we traverse further from the unsafe head towards the finalized head.
// Once we pass the previous safe head and we have seen enough canonical L1 origins to fill a sequence window worth of data,
// then we return the last L2 block of the epoch before that as safe head.
// Each loop iteration we traverse a single L2 block, and we check if the L1 origins are consistent.
for {
// Check if l1Origin is canonical when we get to a new epoch.
if prevL1OriginHash != n.L1Origin.Hash {
prevL1OriginHash = n.L1Origin.Hash
if plausible, canonical, err := isAheadOrCanonical(ctx, l1, n.L1Origin); err != nil {
return eth.L2BlockRef{}, eth.L2BlockRef{}, err
} else if !plausible {
// L1 origin nor ahead of L1 head nor canonical, discard previous candidate and
// keep looking.
highestPlausibleCanonicalOrigin = eth.L2BlockRef{}
} else {
if highestPlausibleCanonicalOrigin == (eth.L2BlockRef{}) {
// No highest plausible candidate, make L2 block new candidate.
highestPlausibleCanonicalOrigin = n
}
if canonical {
break
}
// Fetch L1 information if we never had it, or if we do not have it for the current origin
if l1Block == (eth.L1BlockRef{}) || n.L1Origin.Hash != l1Block.Hash {
b, err := l1.L1BlockRefByNumber(ctx, n.L1Origin.Number)
// if L2 is ahead of L1 view, then consider it a "plausible" head
notFound := errors.Is(err, ethereum.NotFound)
if err != nil && !notFound {
return nil, fmt.Errorf("failed to retrieve block %d from L1 for comparison against %s: %w", n.L1Origin.Number, n.L1Origin.Hash, err)
}
l1Block = b
ahead = notFound
}
// Don't walk past genesis. If we were at the L2 genesis, but could not find its L1 origin,
// the L2 chain is building on the wrong L1 branch.
if n.Hash == genesis.L2.Hash || n.Number == genesis.L2.Number {
return eth.L2BlockRef{}, eth.L2BlockRef{}, WrongChainErr
if n.Number == cfg.Genesis.L2.Number {
// Check L2 traversal against L2 Genesis data, to make sure the engine is on the correct chain, instead of attempting sync with different L2 destination.
if n.Hash != cfg.Genesis.L2.Hash {
return nil, fmt.Errorf("%w L2: genesis: %s, got %s", WrongChainErr, cfg.Genesis.L2, n)
}
// Check L1 comparison against L1 Genesis data, to make sure the L1 data is from the correct chain, instead of attempting sync with different L1 source.
if !ahead && l1Block.Hash != cfg.Genesis.L1.Hash {
return nil, fmt.Errorf("%w L1: genesis: %s, got %s", WrongChainErr, cfg.Genesis.L1, l1Block)
}
}
// Pull L2 parent for next iteration
n, err = l2.L2BlockRefByHash(ctx, n.ParentHash)
if err != nil {
return eth.L2BlockRef{}, eth.L2BlockRef{},
fmt.Errorf("failed to fetch L2 block by hash %v: %w", n.ParentHash, err)
// Check L2 traversal against finalized data
if (n.Number == result.Finalized.Number) && (n.Hash != result.Finalized.Hash) {
return nil, fmt.Errorf("%w: finalized %s, got: %s", ReorgFinalizedErr, result.Finalized, n)
}
reorgDepth++
if reorgDepth >= MaxReorgDepth {
// Check we are not reorging L2 incredibly deep
if n.Number+MaxReorgDepth < prevUnsafe.Number {
// If the reorg depth is too large, something is fishy.
// This can legitimately happen if L1 goes down for a while. But in that case,
// restarting the L2 node with a bigger configured MaxReorgDepth is an acceptable
// stopgap solution.
// Currently this can also happen if the L2 node is down for a while, but in the future
// state sync should prevent this issue.
return eth.L2BlockRef{}, eth.L2BlockRef{}, TooDeepReorgErr
return nil, fmt.Errorf("%w: traversed back to L2 block %s, but too deep compared to previous unsafe block %s", TooDeepReorgErr, n, prevUnsafe)
}
}
// Loop 2. Walk from the L1 origin of the `n` block (*) back to the L1 block that starts the
// sequencing window ending at that block. Instead of iterating on L1 blocks, we actually
// iterate on L2 blocks, because we want to find the safe L2 head, i.e. the highest L2 block
// whose L1 origin is the start of the sequencing window.
// If we don't have a usable unsafe head, then set it
if result.Unsafe == (eth.L2BlockRef{}) {
result.Unsafe = n
}
// (*) `n` being at this stage the highest L2 block whose L1 origin is canonical.
if ahead {
// keep the unsafe head if we can't tell if its L1 origin is canonical or not yet.
} else if l1Block.Hash == n.L1Origin.Hash {
// if L2 matches canonical chain, even if unsafe,
// then we can start finding a span of L1 blocks to cover the sequence window,
// which may help avoid rewinding the existing safe head unnecessarily.
if highestL2WithCanonicalL1Origin == (eth.L2BlockRef{}) {
highestL2WithCanonicalL1Origin = n
}
} else {
// L1 origin not ahead of L1 head nor canonical, discard previous candidate and keep looking.
result.Unsafe = eth.L2BlockRef{}
highestL2WithCanonicalL1Origin = eth.L2BlockRef{}
}
// Depth counter: we need to walk back `seqWindowSize` L1 blocks in order to find the start
// of the sequencing window.
depth := uint64(1)
// If the L2 block is at least as old as the previous safe head, and we have seen at least a full sequence window worth of L1 blocks to confirm
if n.Number <= result.Safe.Number && n.L1Origin.Number+cfg.SeqWindowSize < highestL2WithCanonicalL1Origin.L1Origin.Number && n.SequenceNumber == 0 {
ready = true
}
// Before entering the loop: `prevL1OriginHash == n.L1Origin.Hash`
// The original definitions of `n` and `prevL1OriginHash` still hold.
for {
// Advance depth if we change to a different (earlier) epoch.
if n.L1Origin.Hash != prevL1OriginHash {
depth++
prevL1OriginHash = n.L1Origin.Hash
// Don't traverse further than the finalized head to find a safe head
if n.Number == result.Finalized.Number {
result.Safe = n
return result, nil
}
// Found an L2 block whose L1 origin is the start of the sequencing window.
// Note: We also ensure that we are on the block number with the 0 seq number.
// This is a little hacky, but kinda works. The issue is about where the
// batch queue should start building.
if depth == seqWindowSize && n.SequenceNumber == 0 {
return highestPlausibleCanonicalOrigin, n, nil
// Pull L2 parent for next iteration
parent, err := l2.L2BlockRefByHash(ctx, n.ParentHash)
if err != nil {
return nil, fmt.Errorf("failed to fetch L2 block by hash %v: %w", n.ParentHash, err)
}
// Genesis is always safe.
if n.Hash == genesis.L2.Hash || n.Number == genesis.L2.Number {
safe = eth.L2BlockRef{Hash: genesis.L2.Hash, Number: genesis.L2.Number,
Time: genesis.L2Time, L1Origin: genesis.L1, SequenceNumber: 0}
return highestPlausibleCanonicalOrigin, safe, nil
// Check the L1 origin relation
if parent.L1Origin != n.L1Origin {
// sanity check that the L1 origin block number is coherent
if parent.L1Origin.Number+1 != n.L1Origin.Number {
return nil, fmt.Errorf("l2 parent %s of %s has L1 origin %s that is not before %s", parent, n, parent.L1Origin, n.L1Origin)
}
// sanity check that the later sequence number is 0, if it changed between the L2 blocks
if n.SequenceNumber != 0 {
return nil, fmt.Errorf("l2 block %s has parent %s with different L1 origin %s, but non-zero sequence number %d", n, parent, parent.L1Origin, n.SequenceNumber)
}
// if the L1 origin is known to be canonical, then the parent must be too
if l1Block.Hash == n.L1Origin.Hash && l1Block.ParentHash != parent.L1Origin.Hash {
return nil, fmt.Errorf("parent L2 block %s has origin %s but expected %s", parent, parent.L1Origin, l1Block.ParentHash)
}
} else {
if parent.SequenceNumber+1 != n.SequenceNumber {
return nil, fmt.Errorf("sequence number inconsistency %d <> %d between l2 blocks %s and %s", parent.SequenceNumber, n.SequenceNumber, parent, n)
}
}
// Pull L2 parent for next iteration.
n, err = l2.L2BlockRefByHash(ctx, n.ParentHash)
if err != nil {
return eth.L2BlockRef{}, eth.L2BlockRef{},
fmt.Errorf("failed to fetch L2 block by hash %v: %w", n.ParentHash, err)
n = parent
// once we found the block at seq nr 0 that is more than a full seq window behind the common chain post-reorg, then use the parent block as safe head.
if ready {
result.Safe = n
return result, nil
}
}
}
......@@ -4,11 +4,11 @@ import (
"context"
"testing"
"github.com/ethereum-optimism/optimism/op-node/testlog"
"github.com/ethereum-optimism/optimism/op-node/testutils"
"github.com/ethereum-optimism/optimism/op-node/eth"
"github.com/ethereum-optimism/optimism/op-node/rollup"
"github.com/ethereum-optimism/optimism/op-node/testlog"
"github.com/ethereum-optimism/optimism/op-node/testutils"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/log"
"github.com/stretchr/testify/require"
)
......@@ -20,19 +20,23 @@ var _ L2Chain = (*testutils.FakeChainSource)(nil)
// - The L2 chain is based off of the L1 chain
// - The actual L1 chain is the New L1 chain
// - Both heads are at the tip of their respective chains
func (c *syncStartTestCase) generateFakeL2(t *testing.T) (*testutils.FakeChainSource, eth.L2BlockRef, rollup.Genesis) {
func (c *syncStartTestCase) generateFakeL2(t *testing.T) (*testutils.FakeChainSource, rollup.Genesis) {
log := testlog.Logger(t, log.LvlError)
chain := testutils.NewFakeChainSource([]string{c.L1, c.NewL1}, []string{c.L2}, int(c.GenesisL1Num), log)
chain.SetL2Head(len(c.L2) - 1)
genesis := testutils.FakeGenesis(c.GenesisL1, c.GenesisL2, int(c.GenesisL1Num))
head, err := chain.L2BlockRefByNumber(context.Background(), nil)
require.Nil(t, err)
chain.ReorgL1()
for i := 0; i < len(c.NewL1)-1; i++ {
chain.AdvanceL1()
}
return chain, head, genesis
return chain, genesis
}
func runeToHash(id rune) common.Hash {
var h common.Hash
copy(h[:], string(id))
return h
}
type syncStartTestCase struct {
......@@ -42,6 +46,9 @@ type syncStartTestCase struct {
L2 string // L2 Chain that follows from L1Chain
NewL1 string // New L1 chain
PreFinalizedL2 rune
PreSafeL2 rune
GenesisL1 rune
GenesisL1Num uint64
GenesisL2 rune
......@@ -57,188 +64,221 @@ func refToRune(r eth.BlockID) rune {
}
func (c *syncStartTestCase) Run(t *testing.T) {
chain, l2Head, genesis := c.generateFakeL2(t)
unsafeL2Head, safeHead, err := FindL2Heads(context.Background(), l2Head, c.SeqWindowSize, chain, chain, &genesis)
chain, genesis := c.generateFakeL2(t)
chain.SetL2Finalized(runeToHash(c.PreFinalizedL2))
chain.SetL2Safe(runeToHash(c.PreSafeL2))
cfg := &rollup.Config{
Genesis: genesis,
SeqWindowSize: c.SeqWindowSize,
}
result, err := FindL2Heads(context.Background(), cfg, chain, chain)
if c.ExpectedErr != nil {
require.Error(t, err, "Expecting an error in this test case")
require.ErrorIs(t, c.ExpectedErr, err, "Unexpected error")
require.ErrorIs(t, err, c.ExpectedErr, "expected error")
return
} else {
require.NoError(t, err, "expected no error")
}
require.NoError(t, err)
expectedUnsafeHead := refToRune(unsafeL2Head.ID())
require.Equal(t, string(c.UnsafeL2Head), string(expectedUnsafeHead), "Unsafe L2 Head not equal")
gotUnsafeHead := refToRune(result.Unsafe.ID())
require.Equal(t, string(c.UnsafeL2Head), string(gotUnsafeHead), "Unsafe L2 Head not equal")
expectedSafeHead := refToRune(safeHead.ID())
require.Equal(t, string(c.SafeL2Head), string(expectedSafeHead), "Safe L2 Head not equal")
}
gotSafeHead := refToRune(result.Safe.ID())
require.Equal(t, string(c.SafeL2Head), string(gotSafeHead), "Safe L2 Head not equal")
}
func TestFindSyncStart(t *testing.T) {
testCases := []syncStartTestCase{
{
Name: "already synced",
GenesisL1Num: 0,
L1: "ab",
L2: "AB",
NewL1: "ab",
GenesisL1: 'a',
GenesisL2: 'A',
UnsafeL2Head: 'B',
SeqWindowSize: 2,
SafeL2Head: 'A',
ExpectedErr: nil,
Name: "already synced",
GenesisL1Num: 0,
L1: "ab",
L2: "AB",
NewL1: "ab",
PreFinalizedL2: 'A',
PreSafeL2: 'A',
GenesisL1: 'a',
GenesisL2: 'A',
UnsafeL2Head: 'B',
SeqWindowSize: 2,
SafeL2Head: 'A',
ExpectedErr: nil,
},
{
Name: "small reorg long chain",
GenesisL1Num: 0,
L1: "abcdefgh",
L2: "ABCDEFGH",
NewL1: "abcdefgx",
GenesisL1: 'a',
GenesisL2: 'A',
UnsafeL2Head: 'G',
SeqWindowSize: 2,
SafeL2Head: 'F',
ExpectedErr: nil,
Name: "small reorg long chain",
GenesisL1Num: 0,
L1: "abcdefgh",
L2: "ABCDEFGH",
NewL1: "abcdefgx",
PreFinalizedL2: 'B',
PreSafeL2: 'H',
GenesisL1: 'a',
GenesisL2: 'A',
UnsafeL2Head: 'G',
SeqWindowSize: 2,
SafeL2Head: 'C',
ExpectedErr: nil,
},
{
Name: "L1 Chain ahead",
GenesisL1Num: 0,
L1: "abcde",
L2: "ABCD",
NewL1: "abcde",
GenesisL1: 'a',
GenesisL2: 'A',
UnsafeL2Head: 'D',
SeqWindowSize: 3,
SafeL2Head: 'B',
ExpectedErr: nil,
Name: "L1 Chain ahead",
GenesisL1Num: 0,
L1: "abcdef",
L2: "ABCDE",
NewL1: "abcdef",
PreFinalizedL2: 'A',
PreSafeL2: 'D',
GenesisL1: 'a',
GenesisL2: 'A',
UnsafeL2Head: 'E',
SeqWindowSize: 2,
SafeL2Head: 'A',
ExpectedErr: nil,
},
{
Name: "L2 Chain ahead after reorg",
GenesisL1Num: 0,
L1: "abxyz",
L2: "ABXYZ",
NewL1: "abx",
GenesisL1: 'a',
GenesisL2: 'A',
UnsafeL2Head: 'Z',
SeqWindowSize: 2,
SafeL2Head: 'B',
ExpectedErr: nil,
Name: "L2 Chain ahead after reorg",
GenesisL1Num: 0,
L1: "abcxyz",
L2: "ABCXYZ",
NewL1: "abcx",
PreFinalizedL2: 'B',
PreSafeL2: 'X',
GenesisL1: 'a',
GenesisL2: 'A',
UnsafeL2Head: 'Z',
SeqWindowSize: 2,
SafeL2Head: 'B',
ExpectedErr: nil,
},
{
Name: "genesis",
GenesisL1Num: 0,
L1: "a",
L2: "A",
NewL1: "a",
GenesisL1: 'a',
GenesisL2: 'A',
UnsafeL2Head: 'A',
SeqWindowSize: 2,
SafeL2Head: 'A',
ExpectedErr: nil,
Name: "genesis",
GenesisL1Num: 0,
L1: "a",
L2: "A",
NewL1: "a",
PreFinalizedL2: 'A',
PreSafeL2: 'A',
GenesisL1: 'a',
GenesisL2: 'A',
UnsafeL2Head: 'A',
SeqWindowSize: 2,
SafeL2Head: 'A',
ExpectedErr: nil,
},
{
Name: "reorg one step back",
GenesisL1Num: 0,
L1: "abcd",
L2: "ABCD",
NewL1: "abcx",
GenesisL1: 'a',
GenesisL2: 'A',
UnsafeL2Head: 'C',
SeqWindowSize: 3,
SafeL2Head: 'A',
ExpectedErr: nil,
Name: "reorg one step back",
GenesisL1Num: 0,
L1: "abcdefg",
L2: "ABCDEFG",
NewL1: "abcdefx",
PreFinalizedL2: 'A',
PreSafeL2: 'E',
GenesisL1: 'a',
GenesisL2: 'A',
UnsafeL2Head: 'F',
SeqWindowSize: 3,
SafeL2Head: 'A',
ExpectedErr: nil,
},
{
Name: "reorg two steps back",
GenesisL1Num: 0,
L1: "abc",
L2: "ABC",
NewL1: "axy",
GenesisL1: 'a',
GenesisL2: 'A',
UnsafeL2Head: 'A',
SeqWindowSize: 2,
SafeL2Head: 'A',
ExpectedErr: nil,
Name: "reorg two steps back, clip genesis and finalized",
GenesisL1Num: 0,
L1: "abc",
L2: "ABC",
PreFinalizedL2: 'A',
PreSafeL2: 'B',
NewL1: "axy",
GenesisL1: 'a',
GenesisL2: 'A',
UnsafeL2Head: 'A',
SeqWindowSize: 2,
SafeL2Head: 'A',
ExpectedErr: nil,
},
{
Name: "reorg three steps back",
GenesisL1Num: 0,
L1: "abcdef",
L2: "ABCDEF",
NewL1: "abcxyz",
GenesisL1: 'a',
GenesisL2: 'A',
UnsafeL2Head: 'C',
SeqWindowSize: 2,
SafeL2Head: 'B',
ExpectedErr: nil,
Name: "reorg three steps back",
GenesisL1Num: 0,
L1: "abcdefgh",
L2: "ABCDEFGH",
NewL1: "abcdexyz",
PreFinalizedL2: 'A',
PreSafeL2: 'D',
GenesisL1: 'a',
GenesisL2: 'A',
UnsafeL2Head: 'E',
SeqWindowSize: 2,
SafeL2Head: 'A',
ExpectedErr: nil,
},
{
Name: "unexpected L1 chain",
GenesisL1Num: 0,
L1: "abcdef",
L2: "ABCDEF",
NewL1: "xyzwio",
GenesisL1: 'a',
GenesisL2: 'A',
UnsafeL2Head: 0,
ExpectedErr: WrongChainErr,
Name: "unexpected L1 chain",
GenesisL1Num: 0,
L1: "abcdef",
L2: "ABCDEF",
NewL1: "xyzwio",
PreFinalizedL2: 'A',
PreSafeL2: 'B',
GenesisL1: 'a',
GenesisL2: 'A',
UnsafeL2Head: 0,
ExpectedErr: WrongChainErr,
},
{
Name: "unexpected L2 chain",
GenesisL1Num: 0,
L1: "abcdef",
L2: "ABCDEF",
NewL1: "xyzwio",
GenesisL1: 'a',
GenesisL2: 'X',
UnsafeL2Head: 0,
ExpectedErr: WrongChainErr,
Name: "unexpected L2 chain",
GenesisL1Num: 0,
L1: "abcdef",
L2: "ABCDEF",
NewL1: "xyzwio",
PreFinalizedL2: 'A',
PreSafeL2: 'B',
GenesisL1: 'a',
GenesisL2: 'X',
UnsafeL2Head: 0,
ExpectedErr: WrongChainErr,
},
{
Name: "offset L2 genesis",
GenesisL1Num: 3,
L1: "abcdef",
L2: "DEF",
NewL1: "abcdef",
GenesisL1: 'd',
GenesisL2: 'D',
UnsafeL2Head: 'F',
SeqWindowSize: 2,
SafeL2Head: 'E',
ExpectedErr: nil,
Name: "offset L2 genesis",
GenesisL1Num: 3,
L1: "abcdefghi",
L2: "DEFGHI",
NewL1: "abcdefghi",
PreFinalizedL2: 'E',
PreSafeL2: 'H',
GenesisL1: 'd',
GenesisL2: 'D',
UnsafeL2Head: 'I',
SeqWindowSize: 2,
SafeL2Head: 'E',
ExpectedErr: nil,
},
{
Name: "offset L2 genesis reorg",
GenesisL1Num: 3,
L1: "abcdefgh",
L2: "DEFGH",
NewL1: "abcdxyzw",
GenesisL1: 'd',
GenesisL2: 'D',
UnsafeL2Head: 'D',
SeqWindowSize: 2,
SafeL2Head: 'D',
ExpectedErr: nil,
Name: "offset L2 genesis reorg",
GenesisL1Num: 3,
L1: "abcdefgh",
L2: "DEFGH",
NewL1: "abcdxyzw",
PreFinalizedL2: 'D',
PreSafeL2: 'D',
GenesisL1: 'd',
GenesisL2: 'D',
UnsafeL2Head: 'D',
SeqWindowSize: 2,
SafeL2Head: 'D',
ExpectedErr: nil,
},
{
Name: "reorg past offset genesis",
GenesisL1Num: 3,
L1: "abcdefgh",
L2: "DEFGH",
NewL1: "abxyzwio",
GenesisL1: 'd',
GenesisL2: 'D',
UnsafeL2Head: 0,
ExpectedErr: WrongChainErr,
Name: "reorg past offset genesis",
GenesisL1Num: 3,
L1: "abcdefgh",
L2: "DEFGH",
NewL1: "abxyzwio",
PreFinalizedL2: 'D',
PreSafeL2: 'D',
GenesisL1: 'd',
GenesisL2: 'D',
UnsafeL2Head: 0,
SeqWindowSize: 2,
SafeL2Head: 'D',
ExpectedErr: WrongChainErr,
},
}
......
......@@ -3,11 +3,13 @@ package sources
import (
"context"
"fmt"
"strings"
"github.com/ethereum-optimism/optimism/op-node/client"
"github.com/ethereum-optimism/optimism/op-node/eth"
"github.com/ethereum-optimism/optimism/op-node/rollup"
"github.com/ethereum-optimism/optimism/op-node/sources/caching"
"github.com/ethereum/go-ethereum"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/log"
)
......@@ -67,6 +69,11 @@ func NewL1Client(client client.RPC, log log.Logger, metrics caching.Metrics, con
func (s *L1Client) L1BlockRefByLabel(ctx context.Context, label eth.BlockLabel) (eth.L1BlockRef, error) {
info, err := s.InfoByLabel(ctx, label)
if err != nil {
// Both geth and erigon like to serve non-standard errors for the safe and finalized heads, correct that.
// This happens when the chain just started and nothing is marked as safe/finalized yet.
if strings.Contains(err.Error(), "block not found") || strings.Contains(err.Error(), "Unknown block") {
err = ethereum.NotFound
}
return eth.L1BlockRef{}, fmt.Errorf("failed to fetch head header: %w", err)
}
ref := eth.InfoToL1BlockRef(info)
......
......@@ -3,12 +3,14 @@ package sources
import (
"context"
"fmt"
"strings"
"github.com/ethereum-optimism/optimism/op-node/client"
"github.com/ethereum-optimism/optimism/op-node/eth"
"github.com/ethereum-optimism/optimism/op-node/rollup"
"github.com/ethereum-optimism/optimism/op-node/rollup/derive"
"github.com/ethereum-optimism/optimism/op-node/sources/caching"
"github.com/ethereum/go-ethereum"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/log"
)
......@@ -77,6 +79,11 @@ func NewL2Client(client client.RPC, log log.Logger, metrics caching.Metrics, con
func (s *L2Client) L2BlockRefByLabel(ctx context.Context, label eth.BlockLabel) (eth.L2BlockRef, error) {
payload, err := s.PayloadByLabel(ctx, label)
if err != nil {
// Both geth and erigon like to serve non-standard errors for the safe and finalized heads, correct that.
// This happens when the chain just started and nothing is marked as safe/finalized yet.
if strings.Contains(err.Error(), "block not found") || strings.Contains(err.Error(), "Unknown block") {
err = ethereum.NotFound
}
// w%: wrap to preserve ethereum.NotFound case
return eth.L2BlockRef{}, fmt.Errorf("failed to determine L2BlockRef of %s, could not get payload: %w", label, err)
}
......
......@@ -85,13 +85,17 @@ func NewFakeChainSource(l1 []string, l2 []string, l1GenesisNumber int, log log.L
// what the head block is of the L1 and L2 chains. In addition, it enables re-orgs
// to easily be implemented
type FakeChainSource struct {
l1reorg int // Index of the L1 chain to be operating on
l2reorg int // Index of the L2 chain to be operating on
l1head int // Head block of the L1 chain
l2head int // Head block of the L2 chain
l1s [][]eth.L1BlockRef // l1s[reorg] is the L1 chain in that specific re-org configuration
l2s [][]eth.L2BlockRef // l2s[reorg] is the L2 chain in that specific re-org configuration
log log.Logger
l1reorg int // Index of the L1 chain to be operating on
l2reorg int // Index of the L2 chain to be operating on
l1head int // Head block of the L1 chain
l2head int // Head block of the L2 chain
l1safe int
l2safe int
l1finalized int
l2finalized int
l1s [][]eth.L1BlockRef // l1s[reorg] is the L1 chain in that specific re-org configuration
l2s [][]eth.L2BlockRef // l2s[reorg] is the L2 chain in that specific re-org configuration
log log.Logger
}
func (m *FakeChainSource) L1Range(ctx context.Context, base eth.BlockID, max uint64) ([]eth.BlockID, error) {
......@@ -134,26 +138,39 @@ func (m *FakeChainSource) L1BlockRefByHash(ctx context.Context, l1Hash common.Ha
}
func (m *FakeChainSource) L1BlockRefByLabel(ctx context.Context, label eth.BlockLabel) (eth.L1BlockRef, error) {
if label != eth.Unsafe {
return eth.L1BlockRef{}, fmt.Errorf("testutil FakeChainSource does not support L1BlockRefByLabel(%s)", label)
}
m.log.Trace("L1HeadBlockRef", "l1Head", m.l1head, "reorg", m.l1reorg)
m.log.Trace("L1BlockRefByLabel", "l1Head", m.l1head, "l1Safe", m.l1safe, "l1Finalized", m.l1finalized, "reorg", m.l1reorg)
l := len(m.l1s[m.l1reorg])
if l == 0 {
return eth.L1BlockRef{}, ethereum.NotFound
}
return m.l1s[m.l1reorg][m.l1head], nil
switch label {
case eth.Unsafe:
return m.l1s[m.l1reorg][m.l1head], nil
case eth.Safe:
return m.l1s[m.l1reorg][m.l1safe], nil
case eth.Finalized:
return m.l1s[m.l1reorg][m.l1finalized], nil
default:
return eth.L1BlockRef{}, fmt.Errorf("testutil FakeChainSource does not support L1BlockRefByLabel(%s)", label)
}
}
func (m *FakeChainSource) L2BlockRefByLabel(ctx context.Context, label eth.BlockLabel) (eth.L2BlockRef, error) {
if label != eth.Unsafe {
return eth.L2BlockRef{}, fmt.Errorf("testutil FakeChainSource does not support L2BlockRefByLabel(%s)", label)
m.log.Trace("L2BlockRefByLabel", "l2Head", m.l2head, "l2Safe", m.l2safe, "l2Finalized", m.l2finalized, "reorg", m.l2reorg)
l := len(m.l2s[m.l2reorg])
if l == 0 {
return eth.L2BlockRef{}, ethereum.NotFound
}
m.log.Trace("L2BlockRefHead", "l2Head", m.l2head, "reorg", m.l2reorg)
if len(m.l2s[m.l2reorg]) == 0 {
panic("bad test, no l2 chain")
switch label {
case eth.Unsafe:
return m.l2s[m.l2reorg][m.l2head], nil
case eth.Safe:
return m.l2s[m.l2reorg][m.l2safe], nil
case eth.Finalized:
return m.l2s[m.l2reorg][m.l2finalized], nil
default:
return eth.L2BlockRef{}, fmt.Errorf("testutil FakeChainSource does not support L2BlockRefByLabel(%s)", label)
}
return m.l2s[m.l2reorg][m.l2head], nil
}
func (m *FakeChainSource) L2BlockRefByNumber(ctx context.Context, l2Num *big.Int) (eth.L2BlockRef, error) {
......@@ -204,6 +221,28 @@ func (m *FakeChainSource) ReorgL1() {
}
}
func (m *FakeChainSource) SetL2Safe(safe common.Hash) {
m.log.Trace("Set L2 safe head", "new_safe", safe, "old_safe", m.l2safe)
for i, v := range m.l2s[m.l2reorg] {
if v.Hash == safe {
m.l2safe = i
return
}
}
panic(fmt.Errorf("unknown safe block: %s", safe))
}
func (m *FakeChainSource) SetL2Finalized(finalized common.Hash) {
m.log.Trace("Set L2 finalized head", "new_finalized", finalized, "old_finalized", m.l2finalized)
for i, v := range m.l2s[m.l2reorg] {
if v.Hash == finalized {
m.l2finalized = i
return
}
}
panic(fmt.Errorf("unknown finalized block: %s", finalized))
}
func (m *FakeChainSource) SetL2Head(head int) eth.L2BlockRef {
m.log.Trace("Set L2 head", "new_head", head, "old_head", m.l2head)
m.l2head = head
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment