Skip to content

N
nebula
Commit 9e3a8847 authored by Joshua Gutow's avatar Joshua Gutow
Browse files
Options

op-node: Switch channel bank to be pull based 

This again requires a fair amount of changes to channel_in_reader.go
for the channel in reader to maintain its progress state.
parent 5aa961dd
Expand all Show whitespace changes
Inline Side-by-side
Showing with 205 additions and 298 deletions
op-node/rollup/derive/channel_bank.go
View file @ 9e3a8847
...@@ -2,6 +2,7 @@ package derive ...@@ -2,6 +2,7 @@ package derive
import ( import (
"context" "context"
"errors"
"fmt" "fmt"
"io" "io"
...@@ -11,6 +12,11 @@ import ( ...@@ -11,6 +12,11 @@ import (
"github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/log"
) )
type NextDataProvider interface {
NextData(ctx context.Context) ([]byte, error)
Origin() eth.L1BlockRef
}
// ChannelBank is a stateful stage that does the following: // ChannelBank is a stateful stage that does the following:
// 1. Unmarshalls frames from L1 transaction data // 1. Unmarshalls frames from L1 transaction data
// 2. Applies those frames to a channel // 2. Applies those frames to a channel
...@@ -22,11 +28,6 @@ import ( ...@@ -22,11 +28,6 @@ import (
// Specifically, the channel bank is not allowed to become too large between successive calls // Specifically, the channel bank is not allowed to become too large between successive calls
// to `IngestData`. This means that we can do an ingest and then do a read while becoming too large. // to `IngestData`. This means that we can do an ingest and then do a read while becoming too large.
type ChannelBankOutput interface {
StageProgress
WriteChannel(data []byte)
}
// ChannelBank buffers channel frames, and emits full channel data // ChannelBank buffers channel frames, and emits full channel data
type ChannelBank struct { type ChannelBank struct {
log log.Logger log log.Logger
...@@ -35,82 +36,79 @@ type ChannelBank struct { ...@@ -35,82 +36,79 @@ type ChannelBank struct {
channels map[ChannelID]*Channel // channels by ID channels map[ChannelID]*Channel // channels by ID
channelQueue []ChannelID // channels in FIFO order channelQueue []ChannelID // channels in FIFO order
progress Progress origin eth.L1BlockRef
next ChannelBankOutput prev NextDataProvider
prev *L1Retrieval fetcher L1Fetcher
} }
var _ Stage = (*ChannelBank)(nil) var _ PullStage = (*ChannelBank)(nil)
// NewChannelBank creates a ChannelBank, which should be Reset(origin) before use. // NewChannelBank creates a ChannelBank, which should be Reset(origin) before use.
func NewChannelBank(log log.Logger, cfg *rollup.Config, next ChannelBankOutput, prev *L1Retrieval) *ChannelBank { func NewChannelBank(log log.Logger, cfg *rollup.Config, prev NextDataProvider, fetcher L1Fetcher) *ChannelBank {
return &ChannelBank{ return &ChannelBank{
log: log, log: log,
cfg: cfg, cfg: cfg,
channels: make(map[ChannelID]*Channel), channels: make(map[ChannelID]*Channel),
channelQueue: make([]ChannelID, 0, 10), channelQueue: make([]ChannelID, 0, 10),
next: next,
prev: prev, prev: prev,
fetcher: fetcher,
} }
} }
func (ib *ChannelBank) Progress() Progress { func (cb *ChannelBank) Origin() eth.L1BlockRef {
return ib.progress return cb.prev.Origin()
} }
func (ib *ChannelBank) prune() { func (cb *ChannelBank) prune() {
// check total size // check total size
totalSize := uint64(0) totalSize := uint64(0)
for _, ch := range ib.channels { for _, ch := range cb.channels {
totalSize += ch.size totalSize += ch.size
} }
// prune until it is reasonable again. The high-priority channel failed to be read, so we start pruning there. // prune until it is reasonable again. The high-priority channel failed to be read, so we start pruning there.
for totalSize > MaxChannelBankSize { for totalSize > MaxChannelBankSize {
id := ib.channelQueue[0] id := cb.channelQueue[0]
ch := ib.channels[id] ch := cb.channels[id]
ib.channelQueue = ib.channelQueue[1:] cb.channelQueue = cb.channelQueue[1:]
delete(ib.channels, id) delete(cb.channels, id)
totalSize -= ch.size totalSize -= ch.size
} }
} }
// IngestData adds new L1 data to the channel bank. // IngestData adds new L1 data to the channel bank.
// Read() should be called repeatedly first, until everything has been read, before adding new data.\ // Read() should be called repeatedly first, until everything has been read, before adding new data.\
func (ib *ChannelBank) IngestData(data []byte) { func (cb *ChannelBank) IngestData(data []byte) {
if ib.progress.Closed { cb.log.Debug("channel bank got new data", "origin", cb.origin, "data_len", len(data))
panic("write data to bank while closed")
}
ib.log.Debug("channel bank got new data", "origin", ib.progress.Origin, "data_len", len(data))
// TODO: Why is the prune here? // TODO: Why is the prune here?
ib.prune() cb.prune()
frames, err := ParseFrames(data) frames, err := ParseFrames(data)
if err != nil { if err != nil {
ib.log.Warn("malformed frame", "err", err) cb.log.Warn("malformed frame", "err", err)
return return
} }
// Process each frame // Process each frame
for _, f := range frames { for _, f := range frames {
currentCh, ok := ib.channels[f.ID] currentCh, ok := cb.channels[f.ID]
if !ok { if !ok {
// create new channel if it doesn't exist yet // create new channel if it doesn't exist yet
currentCh = NewChannel(f.ID, ib.progress.Origin) currentCh = NewChannel(f.ID, cb.origin)
ib.channels[f.ID] = currentCh cb.channels[f.ID] = currentCh
ib.channelQueue = append(ib.channelQueue, f.ID) cb.channelQueue = append(cb.channelQueue, f.ID)
} }
// check if the channel is not timed out // check if the channel is not timed out
if currentCh.OpenBlockNumber()+ib.cfg.ChannelTimeout < ib.progress.Origin.Number { if currentCh.OpenBlockNumber()+cb.cfg.ChannelTimeout < cb.origin.Number {
ib.log.Warn("channel is timed out, ignore frame", "channel", f.ID, "frame", f.FrameNumber) cb.log.Warn("channel is timed out, ignore frame", "channel", f.ID, "frame", f.FrameNumber)
continue continue
} }
ib.log.Trace("ingesting frame", "channel", f.ID, "frame_number", f.FrameNumber, "length", len(f.Data)) cb.log.Trace("ingesting frame", "channel", f.ID, "frame_number", f.FrameNumber, "length", len(f.Data))
if err := currentCh.AddFrame(f, ib.progress.Origin); err != nil { if err := currentCh.AddFrame(f, cb.origin); err != nil {
ib.log.Warn("failed to ingest frame into channel", "channel", f.ID, "frame_number", f.FrameNumber, "err", err) cb.log.Warn("failed to ingest frame into channel", "channel", f.ID, "frame_number", f.FrameNumber, "err", err)
continue continue
} }
} }
...@@ -118,78 +116,60 @@ func (ib *ChannelBank) IngestData(data []byte) { ...@@ -118,78 +116,60 @@ func (ib *ChannelBank) IngestData(data []byte) {
// Read the raw data of the first channel, if it's timed-out or closed. // Read the raw data of the first channel, if it's timed-out or closed.
// Read returns io.EOF if there is nothing new to read. // Read returns io.EOF if there is nothing new to read.
func (ib *ChannelBank) Read() (data []byte, err error) { func (cb *ChannelBank) Read() (data []byte, err error) {
if len(ib.channelQueue) == 0 { if len(cb.channelQueue) == 0 {
return nil, io.EOF return nil, io.EOF
} }
first := ib.channelQueue[0] first := cb.channelQueue[0]
ch := ib.channels[first] ch := cb.channels[first]
timedOut := ch.OpenBlockNumber()+ib.cfg.ChannelTimeout < ib.progress.Origin.Number timedOut := ch.OpenBlockNumber()+cb.cfg.ChannelTimeout < cb.origin.Number
if timedOut { if timedOut {
ib.log.Debug("channel timed out", "channel", first, "frames", len(ch.inputs)) cb.log.Debug("channel timed out", "channel", first, "frames", len(ch.inputs))
delete(ib.channels, first) delete(cb.channels, first)
ib.channelQueue = ib.channelQueue[1:] cb.channelQueue = cb.channelQueue[1:]
return nil, io.EOF return nil, io.EOF
} }
if !ch.IsReady() { if !ch.IsReady() {
return nil, io.EOF return nil, io.EOF
} }
delete(ib.channels, first) delete(cb.channels, first)
ib.channelQueue = ib.channelQueue[1:] cb.channelQueue = cb.channelQueue[1:]
r := ch.Reader() r := ch.Reader()
// Suprress error here. io.ReadAll does return nil instead of io.EOF though. // Suprress error here. io.ReadAll does return nil instead of io.EOF though.
data, _ = io.ReadAll(r) data, _ = io.ReadAll(r)
return data, nil return data, nil
} }
// Step does the advancement for the channel bank. // NextData pulls the next piece of data from the channel bank.
// Channel bank as the first non-pull stage does it's own progress maintentance. // Note that it attempts to pull data out of the channel bank prior to
// When closed, it checks against the previous origin to determine if to open itself // loading data in (unlike most other stages). This is to ensure maintain
func (ib *ChannelBank) Step(ctx context.Context, _ Progress) error { // consistency around channel bank pruning which depends upon the order
// Open ourselves // of operations.
// This is ok to do b/c we would not have yielded control to the lower stages func (cb *ChannelBank) NextData(ctx context.Context) ([]byte, error) {
// of the pipeline without being completely done reading from L1. if cb.origin != cb.prev.Origin() {
if ib.progress.Closed { cb.origin = cb.prev.Origin()
if ib.progress.Origin != ib.prev.Origin() {
ib.progress.Closed = false
ib.progress.Origin = ib.prev.Origin()
return nil
}
} }
skipIngest := ib.next.Progress().Origin.Number > ib.progress.Origin.Number // Do the read from the channel bank first
outOfData := false data, err := cb.Read()
if err == io.EOF {
if data, err := ib.prev.NextData(ctx); err == io.EOF { // continue - We will attempt to load data into the channel bank
outOfData = true
} else if err != nil { } else if err != nil {
return err return nil, err
} else { } else {
ib.IngestData(data) return data, nil
} }
// otherwise, read the next channel data from the bank // Then load data into the channel bank
data, err := ib.Read() if data, err := cb.prev.NextData(ctx); err == io.EOF {
if err == io.EOF { // need new L1 data in the bank before we can read more channel data return nil, io.EOF
if outOfData {
if !ib.progress.Closed {
ib.progress.Closed = true
return nil
}
return io.EOF
} else {
return nil
}
} else if err != nil { } else if err != nil {
return err return nil, err
} else { } else {
if !skipIngest { cb.IngestData(data)
ib.next.WriteChannel(data) return nil, NewTemporaryError(errors.New("not enough data"))
return nil
}
} }
return nil
} }
// ResetStep walks back the L1 chain, starting at the origin of the next stage, // ResetStep walks back the L1 chain, starting at the origin of the next stage,
...@@ -197,19 +177,18 @@ func (ib *ChannelBank) Step(ctx context.Context, _ Progress) error { ...@@ -197,19 +177,18 @@ func (ib *ChannelBank) Step(ctx context.Context, _ Progress) error {
// to get consistent reads starting at origin. // to get consistent reads starting at origin.
// Any channel data before this origin will be timed out by the time the channel bank is synced up to the origin, // Any channel data before this origin will be timed out by the time the channel bank is synced up to the origin,
// so it is not relevant to replay it into the bank. // so it is not relevant to replay it into the bank.
func (ib *ChannelBank) ResetStep(ctx context.Context, l1Fetcher L1Fetcher) error { func (cb *ChannelBank) Reset(ctx context.Context, base eth.L1BlockRef) error {
ib.progress = ib.next.Progress() cb.log.Debug("walking back to find reset origin for channel bank", "origin", base)
ib.log.Debug("walking back to find reset origin for channel bank", "origin", ib.progress.Origin)
// go back in history if we are not distant enough from the next stage // go back in history if we are not distant enough from the next stage
resetBlock := ib.progress.Origin.Number - ib.cfg.ChannelTimeout resetBlock := base.Number - cb.cfg.ChannelTimeout
if ib.progress.Origin.Number < ib.cfg.ChannelTimeout { if base.Number < cb.cfg.ChannelTimeout {
resetBlock = 0 // don't underflow resetBlock = 0 // don't underflow
} }
parent, err := l1Fetcher.L1BlockRefByNumber(ctx, resetBlock) parent, err := cb.fetcher.L1BlockRefByNumber(ctx, resetBlock)
if err != nil { if err != nil {
return NewTemporaryError(fmt.Errorf("failed to find channel bank block, failed to retrieve L1 reference: %w", err)) return NewTemporaryError(fmt.Errorf("failed to find channel bank block, failed to retrieve L1 reference: %w", err))
} }
ib.progress.Origin = parent cb.origin = parent
return io.EOF return io.EOF
} }
......
op-node/rollup/derive/channel_bank_test.go
View file @ 9e3a8847
This diff is collapsed.
op-node/rollup/derive/channel_in_reader.go
View file @ 9e3a8847
...@@ -26,13 +26,18 @@ type ChannelInReader struct { ...@@ -26,13 +26,18 @@ type ChannelInReader struct {
progress Progress progress Progress
next BatchQueueStage next BatchQueueStage
prev *ChannelBank
} }
var _ ChannelBankOutput = (*ChannelInReader)(nil) var _ Stage = (*ChannelInReader)(nil)
// NewChannelInReader creates a ChannelInReader, which should be Reset(origin) before use. // NewChannelInReader creates a ChannelInReader, which should be Reset(origin) before use.
func NewChannelInReader(log log.Logger, next BatchQueueStage) *ChannelInReader { func NewChannelInReader(log log.Logger, next BatchQueueStage, prev *ChannelBank) *ChannelInReader {
return &ChannelInReader{log: log, next: next} return &ChannelInReader{
log: log,
next: next,
prev: prev,
}
} }
func (cr *ChannelInReader) Progress() Progress { func (cr *ChannelInReader) Progress() Progress {
...@@ -58,19 +63,35 @@ func (cr *ChannelInReader) NextChannel() { ...@@ -58,19 +63,35 @@ func (cr *ChannelInReader) NextChannel() {
} }
func (cr *ChannelInReader) Step(ctx context.Context, outer Progress) error { func (cr *ChannelInReader) Step(ctx context.Context, outer Progress) error {
if changed, err := cr.progress.Update(outer); err != nil || changed { // Close ourselves if required
return err if cr.progress.Closed {
if cr.progress.Origin != cr.prev.Origin() {
cr.progress.Closed = false
cr.progress.Origin = cr.prev.Origin()
return nil
}
} }
if cr.nextBatchFn == nil { if cr.nextBatchFn == nil {
if data, err := cr.prev.NextData(ctx); err == io.EOF {
if !cr.progress.Closed {
cr.progress.Closed = true
return nil
} else {
return io.EOF return io.EOF
} }
} else if err != nil {
return err
} else {
cr.WriteChannel(data)
return nil
}
} else {
// TODO: can batch be non nil while err == io.EOF // TODO: can batch be non nil while err == io.EOF
// This depends on the behavior of rlp.Stream // This depends on the behavior of rlp.Stream
batch, err := cr.nextBatchFn() batch, err := cr.nextBatchFn()
if err == io.EOF { if err == io.EOF {
cr.NextChannel()
return io.EOF return io.EOF
} else if err != nil { } else if err != nil {
cr.log.Warn("failed to read batch from channel reader, skipping to next channel now", "err", err) cr.log.Warn("failed to read batch from channel reader, skipping to next channel now", "err", err)
...@@ -79,6 +100,8 @@ func (cr *ChannelInReader) Step(ctx context.Context, outer Progress) error { ...@@ -79,6 +100,8 @@ func (cr *ChannelInReader) Step(ctx context.Context, outer Progress) error {
} }
cr.next.AddBatch(batch.Batch) cr.next.AddBatch(batch.Batch)
return nil return nil
}
} }
func (cr *ChannelInReader) ResetStep(ctx context.Context, l1Fetcher L1Fetcher) error { func (cr *ChannelInReader) ResetStep(ctx context.Context, l1Fetcher L1Fetcher) error {
......
op-node/rollup/derive/pipeline.go
View file @ 9e3a8847
...@@ -98,16 +98,16 @@ func NewDerivationPipeline(log log.Logger, cfg *rollup.Config, l1Fetcher L1Fetch ...@@ -98,16 +98,16 @@ func NewDerivationPipeline(log log.Logger, cfg *rollup.Config, l1Fetcher L1Fetch
l1Traversal := NewL1Traversal(log, l1Fetcher) l1Traversal := NewL1Traversal(log, l1Fetcher)
dataSrc := NewDataSourceFactory(log, cfg, l1Fetcher) // auxiliary stage for L1Retrieval dataSrc := NewDataSourceFactory(log, cfg, l1Fetcher) // auxiliary stage for L1Retrieval
l1Src := NewL1Retrieval(log, dataSrc, l1Traversal) l1Src := NewL1Retrieval(log, dataSrc, l1Traversal)
bank := NewChannelBank(log, cfg, l1Src, l1Fetcher)
// Push stages (that act like pull stages b/c we push from the innermost stages prior to the outermost stages) // Push stages (that act like pull stages b/c we push from the innermost stages prior to the outermost stages)
eng := NewEngineQueue(log, cfg, engine, metrics) eng := NewEngineQueue(log, cfg, engine, metrics)
attributesQueue := NewAttributesQueue(log, cfg, l1Fetcher, eng) attributesQueue := NewAttributesQueue(log, cfg, l1Fetcher, eng)
batchQueue := NewBatchQueue(log, cfg, attributesQueue) batchQueue := NewBatchQueue(log, cfg, attributesQueue)
chInReader := NewChannelInReader(log, batchQueue) chInReader := NewChannelInReader(log, batchQueue, bank)
bank := NewChannelBank(log, cfg, chInReader, l1Src)
stages := []Stage{eng, attributesQueue, batchQueue, chInReader, bank} stages := []Stage{eng, attributesQueue, batchQueue, chInReader}
pullStages := []PullStage{l1Src, l1Traversal} pullStages := []PullStage{bank, l1Src, l1Traversal}
return &DerivationPipeline{ return &DerivationPipeline{
log: log, log: log,
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment