Skip to content
Projects
Groups
Snippets
Help
Loading...
Help
Support
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
N
nebula
Project
Project
Details
Activity
Releases
Cycle Analytics
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Charts
Issues
0
Issues
0
List
Boards
Labels
Milestones
Merge Requests
0
Merge Requests
0
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Charts
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Charts
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
exchain
nebula
Commits
6b6bc5ff
Unverified
Commit
6b6bc5ff
authored
May 12, 2023
by
Michael de Hoog
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
Default to Ratio compressor
parent
6f831fdc
Changes
11
Hide whitespace changes
Inline
Side-by-side
Showing
11 changed files
with
77 additions
and
106 deletions
+77
-106
channel_builder.go
op-batcher/batcher/channel_builder.go
+1
-3
channel_builder_test.go
op-batcher/batcher/channel_builder_test.go
+1
-2
channel_manager_test.go
op-batcher/batcher/channel_manager_test.go
+0
-7
config.go
op-batcher/batcher/config.go
+0
-3
driver.go
op-batcher/batcher/driver.go
+1
-7
cli.go
op-batcher/compressor/cli.go
+28
-24
compressors.go
op-batcher/compressor/compressors.go
+23
-0
config.go
op-batcher/compressor/config.go
+17
-3
factory.go
op-batcher/compressor/factory.go
+0
-45
migration_test.go
op-e2e/migration_test.go
+3
-6
setup.go
op-e2e/setup.go
+3
-6
No files found.
op-batcher/batcher/channel_builder.go
View file @
6b6bc5ff
...
@@ -58,8 +58,6 @@ type ChannelConfig struct {
...
@@ -58,8 +58,6 @@ type ChannelConfig struct {
// CompressorConfig contains the configuration for creating new compressors.
// CompressorConfig contains the configuration for creating new compressors.
CompressorConfig
compressor
.
Config
CompressorConfig
compressor
.
Config
// CompressorFactory creates new compressors.
CompressorFactory
compressor
.
FactoryFunc
}
}
// Check validates the [ChannelConfig] parameters.
// Check validates the [ChannelConfig] parameters.
...
@@ -128,7 +126,7 @@ type channelBuilder struct {
...
@@ -128,7 +126,7 @@ type channelBuilder struct {
// newChannelBuilder creates a new channel builder or returns an error if the
// newChannelBuilder creates a new channel builder or returns an error if the
// channel out could not be created.
// channel out could not be created.
func
newChannelBuilder
(
cfg
ChannelConfig
)
(
*
channelBuilder
,
error
)
{
func
newChannelBuilder
(
cfg
ChannelConfig
)
(
*
channelBuilder
,
error
)
{
c
,
err
:=
cfg
.
Compressor
Factory
(
cfg
.
CompressorConfig
)
c
,
err
:=
cfg
.
Compressor
Config
.
NewCompressor
(
)
if
err
!=
nil
{
if
err
!=
nil
{
return
nil
,
err
return
nil
,
err
}
}
...
...
op-batcher/batcher/channel_builder_test.go
View file @
6b6bc5ff
...
@@ -34,7 +34,6 @@ var defaultTestChannelConfig = ChannelConfig{
...
@@ -34,7 +34,6 @@ var defaultTestChannelConfig = ChannelConfig{
TargetNumFrames
:
1
,
TargetNumFrames
:
1
,
ApproxComprRatio
:
0.4
,
ApproxComprRatio
:
0.4
,
},
},
CompressorFactory
:
compressor
.
NewRatioCompressor
,
}
}
// TestChannelConfig_Check tests the [ChannelConfig] [Check] function.
// TestChannelConfig_Check tests the [ChannelConfig] [Check] function.
...
@@ -420,7 +419,7 @@ func TestChannelBuilder_OutputWrongFramePanic(t *testing.T) {
...
@@ -420,7 +419,7 @@ func TestChannelBuilder_OutputWrongFramePanic(t *testing.T) {
// Mock the internals of `channelBuilder.outputFrame`
// Mock the internals of `channelBuilder.outputFrame`
// to construct a single frame
// to construct a single frame
c
,
err
:=
channelConfig
.
Compressor
Factory
(
channelConfig
.
CompressorConfig
)
c
,
err
:=
channelConfig
.
Compressor
Config
.
NewCompressor
(
)
require
.
NoError
(
t
,
err
)
require
.
NoError
(
t
,
err
)
co
,
err
:=
derive
.
NewChannelOut
(
c
)
co
,
err
:=
derive
.
NewChannelOut
(
c
)
require
.
NoError
(
t
,
err
)
require
.
NoError
(
t
,
err
)
...
...
op-batcher/batcher/channel_manager_test.go
View file @
6b6bc5ff
...
@@ -104,7 +104,6 @@ func TestChannelManagerReturnsErrReorgWhenDrained(t *testing.T) {
...
@@ -104,7 +104,6 @@ func TestChannelManagerReturnsErrReorgWhenDrained(t *testing.T) {
TargetFrameSize
:
0
,
TargetFrameSize
:
0
,
ApproxComprRatio
:
1.0
,
ApproxComprRatio
:
1.0
,
},
},
CompressorFactory
:
compressor
.
NewRatioCompressor
,
})
})
a
:=
newMiniL2Block
(
0
)
a
:=
newMiniL2Block
(
0
)
...
@@ -180,7 +179,6 @@ func TestChannelManager_Clear(t *testing.T) {
...
@@ -180,7 +179,6 @@ func TestChannelManager_Clear(t *testing.T) {
TargetNumFrames
:
1
,
TargetNumFrames
:
1
,
ApproxComprRatio
:
1.0
,
ApproxComprRatio
:
1.0
,
},
},
CompressorFactory
:
compressor
.
NewRatioCompressor
,
})
})
// Channel Manager state should be empty by default
// Channel Manager state should be empty by default
...
@@ -344,7 +342,6 @@ func TestChannelManager_TxResend(t *testing.T) {
...
@@ -344,7 +342,6 @@ func TestChannelManager_TxResend(t *testing.T) {
TargetFrameSize
:
0
,
TargetFrameSize
:
0
,
ApproxComprRatio
:
1.0
,
ApproxComprRatio
:
1.0
,
},
},
CompressorFactory
:
compressor
.
NewRatioCompressor
,
})
})
a
,
_
:=
derivetest
.
RandomL2Block
(
rng
,
4
)
a
,
_
:=
derivetest
.
RandomL2Block
(
rng
,
4
)
...
@@ -389,7 +386,6 @@ func TestChannelManagerCloseBeforeFirstUse(t *testing.T) {
...
@@ -389,7 +386,6 @@ func TestChannelManagerCloseBeforeFirstUse(t *testing.T) {
TargetFrameSize
:
0
,
TargetFrameSize
:
0
,
ApproxComprRatio
:
1.0
,
ApproxComprRatio
:
1.0
,
},
},
CompressorFactory
:
compressor
.
NewRatioCompressor
,
})
})
a
,
_
:=
derivetest
.
RandomL2Block
(
rng
,
4
)
a
,
_
:=
derivetest
.
RandomL2Block
(
rng
,
4
)
...
@@ -418,7 +414,6 @@ func TestChannelManagerCloseNoPendingChannel(t *testing.T) {
...
@@ -418,7 +414,6 @@ func TestChannelManagerCloseNoPendingChannel(t *testing.T) {
TargetNumFrames
:
1
,
TargetNumFrames
:
1
,
ApproxComprRatio
:
1.0
,
ApproxComprRatio
:
1.0
,
},
},
CompressorFactory
:
compressor
.
NewRatioCompressor
,
})
})
a
:=
newMiniL2Block
(
0
)
a
:=
newMiniL2Block
(
0
)
b
:=
newMiniL2BlockWithNumberParent
(
0
,
big
.
NewInt
(
1
),
a
.
Hash
())
b
:=
newMiniL2BlockWithNumberParent
(
0
,
big
.
NewInt
(
1
),
a
.
Hash
())
...
@@ -458,7 +453,6 @@ func TestChannelManagerClosePendingChannel(t *testing.T) {
...
@@ -458,7 +453,6 @@ func TestChannelManagerClosePendingChannel(t *testing.T) {
TargetFrameSize
:
1000
,
TargetFrameSize
:
1000
,
ApproxComprRatio
:
1.0
,
ApproxComprRatio
:
1.0
,
},
},
CompressorFactory
:
compressor
.
NewRatioCompressor
,
})
})
a
:=
newMiniL2Block
(
50
_000
)
a
:=
newMiniL2Block
(
50
_000
)
...
@@ -504,7 +498,6 @@ func TestChannelManagerCloseAllTxsFailed(t *testing.T) {
...
@@ -504,7 +498,6 @@ func TestChannelManagerCloseAllTxsFailed(t *testing.T) {
TargetFrameSize
:
1000
,
TargetFrameSize
:
1000
,
ApproxComprRatio
:
1.0
,
ApproxComprRatio
:
1.0
,
},
},
CompressorFactory
:
compressor
.
NewRatioCompressor
,
})
})
a
:=
newMiniL2Block
(
50
_000
)
a
:=
newMiniL2Block
(
50
_000
)
...
...
op-batcher/batcher/config.go
View file @
6b6bc5ff
...
@@ -111,9 +111,6 @@ func (c CLIConfig) Check() error {
...
@@ -111,9 +111,6 @@ func (c CLIConfig) Check() error {
if
err
:=
c
.
TxMgrConfig
.
Check
();
err
!=
nil
{
if
err
:=
c
.
TxMgrConfig
.
Check
();
err
!=
nil
{
return
err
return
err
}
}
if
err
:=
c
.
CompressorConfig
.
Check
();
err
!=
nil
{
return
err
}
return
nil
return
nil
}
}
...
...
op-batcher/batcher/driver.go
View file @
6b6bc5ff
...
@@ -75,11 +75,6 @@ func NewBatchSubmitterFromCLIConfig(cfg CLIConfig, l log.Logger, m metrics.Metri
...
@@ -75,11 +75,6 @@ func NewBatchSubmitterFromCLIConfig(cfg CLIConfig, l log.Logger, m metrics.Metri
return
nil
,
err
return
nil
,
err
}
}
compressorFactory
,
err
:=
cfg
.
CompressorConfig
.
Factory
()
if
err
!=
nil
{
return
nil
,
err
}
batcherCfg
:=
Config
{
batcherCfg
:=
Config
{
L1Client
:
l1Client
,
L1Client
:
l1Client
,
L2Client
:
l2Client
,
L2Client
:
l2Client
,
...
@@ -95,8 +90,7 @@ func NewBatchSubmitterFromCLIConfig(cfg CLIConfig, l log.Logger, m metrics.Metri
...
@@ -95,8 +90,7 @@ func NewBatchSubmitterFromCLIConfig(cfg CLIConfig, l log.Logger, m metrics.Metri
MaxChannelDuration
:
cfg
.
MaxChannelDuration
,
MaxChannelDuration
:
cfg
.
MaxChannelDuration
,
SubSafetyMargin
:
cfg
.
SubSafetyMargin
,
SubSafetyMargin
:
cfg
.
SubSafetyMargin
,
MaxFrameSize
:
cfg
.
MaxL1TxSize
-
1
,
// subtract 1 byte for version
MaxFrameSize
:
cfg
.
MaxL1TxSize
-
1
,
// subtract 1 byte for version
CompressorConfig
:
cfg
.
CompressorConfig
.
Config
,
CompressorConfig
:
cfg
.
CompressorConfig
.
Config
(),
CompressorFactory
:
compressorFactory
,
},
},
}
}
...
...
op-batcher/compressor/cli.go
View file @
6b6bc5ff
package
compressor
package
compressor
import
(
import
(
"
fmt
"
"
strings
"
opservice
"github.com/ethereum-optimism/optimism/op-service"
opservice
"github.com/ethereum-optimism/optimism/op-service"
"github.com/urfave/cli"
"github.com/urfave/cli"
...
@@ -11,7 +11,7 @@ const (
...
@@ -11,7 +11,7 @@ const (
TargetL1TxSizeBytesFlagName
=
"target-l1-tx-size-bytes"
TargetL1TxSizeBytesFlagName
=
"target-l1-tx-size-bytes"
TargetNumFramesFlagName
=
"target-num-frames"
TargetNumFramesFlagName
=
"target-num-frames"
ApproxComprRatioFlagName
=
"approx-compr-ratio"
ApproxComprRatioFlagName
=
"approx-compr-ratio"
Type
FlagName
=
"compressor"
Kind
FlagName
=
"compressor"
)
)
func
CLIFlags
(
envPrefix
string
)
[]
cli
.
Flag
{
func
CLIFlags
(
envPrefix
string
)
[]
cli
.
Flag
{
...
@@ -35,40 +35,44 @@ func CLIFlags(envPrefix string) []cli.Flag {
...
@@ -35,40 +35,44 @@ func CLIFlags(envPrefix string) []cli.Flag {
EnvVar
:
opservice
.
PrefixEnvVar
(
envPrefix
,
"APPROX_COMPR_RATIO"
),
EnvVar
:
opservice
.
PrefixEnvVar
(
envPrefix
,
"APPROX_COMPR_RATIO"
),
},
},
cli
.
StringFlag
{
cli
.
StringFlag
{
Name
:
Type
FlagName
,
Name
:
Kind
FlagName
,
Usage
:
"The type of compressor. Valid options: "
+
FactoryFlags
(
),
Usage
:
"The type of compressor. Valid options: "
+
strings
.
Join
(
KindKeys
,
", "
),
EnvVar
:
opservice
.
PrefixEnvVar
(
envPrefix
,
"COMPRESSOR"
),
EnvVar
:
opservice
.
PrefixEnvVar
(
envPrefix
,
"COMPRESSOR"
),
Value
:
Ratio
.
FlagValue
,
Value
:
Ratio
Kind
,
},
},
}
}
}
}
type
CLIConfig
struct
{
type
CLIConfig
struct
{
Type
string
// TargetL1TxSizeBytes to target when creating channel frames. Note that if the
Config
Config
// realized compression ratio is worse than the approximate, more frames may
// actually be created. This also depends on how close the target is to the
// max frame size.
TargetL1TxSizeBytes
uint64
// TargetNumFrames to create in this channel. If the realized compression ratio
// is worse than approxComprRatio, additional leftover frame(s) might get created.
TargetNumFrames
int
// ApproxComprRatio to assume. Should be slightly smaller than average from
// experiments to avoid the chances of creating a small additional leftover frame.
ApproxComprRatio
float64
// Type of compressor to use. Must be one of KindKeys.
Kind
string
}
}
func
(
c
*
CLIConfig
)
Check
()
error
{
func
(
c
*
CLIConfig
)
Config
()
Config
{
_
,
err
:=
c
.
Factory
()
return
Config
{
return
err
TargetFrameSize
:
c
.
TargetL1TxSizeBytes
-
1
,
// subtract 1 byte for version
}
TargetNumFrames
:
c
.
TargetNumFrames
,
ApproxComprRatio
:
c
.
ApproxComprRatio
,
func
(
c
*
CLIConfig
)
Factory
()
(
FactoryFunc
,
error
)
{
Kind
:
c
.
Kind
,
for
_
,
f
:=
range
Factories
{
if
f
.
FlagValue
==
c
.
Type
{
return
f
.
FactoryFunc
,
nil
}
}
}
return
nil
,
fmt
.
Errorf
(
"unknown compressor kind: %q"
,
c
.
Type
)
}
}
func
ReadCLIConfig
(
ctx
*
cli
.
Context
)
CLIConfig
{
func
ReadCLIConfig
(
ctx
*
cli
.
Context
)
CLIConfig
{
return
CLIConfig
{
return
CLIConfig
{
Type
:
ctx
.
GlobalString
(
TypeFlagName
),
Kind
:
ctx
.
GlobalString
(
KindFlagName
),
Config
:
Config
{
TargetL1TxSizeBytes
:
ctx
.
GlobalUint64
(
TargetL1TxSizeBytesFlagName
),
TargetFrameSize
:
ctx
.
GlobalUint64
(
TargetL1TxSizeBytesFlagName
)
-
1
,
// subtract 1 byte for version,
TargetNumFrames
:
ctx
.
GlobalInt
(
TargetNumFramesFlagName
),
TargetNumFrames
:
ctx
.
GlobalInt
(
TargetNumFramesFlagName
),
ApproxComprRatio
:
ctx
.
GlobalFloat64
(
ApproxComprRatioFlagName
),
ApproxComprRatio
:
ctx
.
GlobalFloat64
(
ApproxComprRatioFlagName
),
},
}
}
}
}
op-batcher/compressor/compressors.go
0 → 100644
View file @
6b6bc5ff
package
compressor
import
(
"github.com/ethereum-optimism/optimism/op-node/rollup/derive"
)
type
FactoryFunc
func
(
Config
)
(
derive
.
Compressor
,
error
)
const
RatioKind
=
"ratio"
const
ShadowKind
=
"shadow"
var
Kinds
=
map
[
string
]
FactoryFunc
{
RatioKind
:
NewRatioCompressor
,
ShadowKind
:
NewShadowCompressor
,
}
var
KindKeys
[]
string
func
init
()
{
for
k
:=
range
Kinds
{
KindKeys
=
append
(
KindKeys
,
k
)
}
}
op-batcher/compressor/config.go
View file @
6b6bc5ff
package
compressor
package
compressor
import
(
"github.com/ethereum-optimism/optimism/op-node/rollup/derive"
)
type
Config
struct
{
type
Config
struct
{
//
FrameSizeTarget
to target when creating channel frames. Note that if the
//
TargetFrameSize
to target when creating channel frames. Note that if the
// realized compression ratio is worse than the approximate, more frames may
// realized compression ratio is worse than the approximate, more frames may
// actually be created. This also depends on how close the target is to the
// actually be created. This also depends on how close the target is to the
// max frame size.
// max frame size.
TargetFrameSize
uint64
TargetFrameSize
uint64
//
NumFramesTarget
to create in this channel. If the realized compression ratio
//
TargetNumFrames
to create in this channel. If the realized compression ratio
// is worse than approxComprRatio, additional leftover frame(s) might get created.
// is worse than approxComprRatio, additional leftover frame(s) might get created.
TargetNumFrames
int
TargetNumFrames
int
// ApproxCompRatio to assume. Should be slightly smaller than average from
// ApproxComp
r
Ratio to assume. Should be slightly smaller than average from
// experiments to avoid the chances of creating a small additional leftover frame.
// experiments to avoid the chances of creating a small additional leftover frame.
ApproxComprRatio
float64
ApproxComprRatio
float64
// Kind of compressor to use. Must
Kind
string
}
func
(
c
Config
)
NewCompressor
()
(
derive
.
Compressor
,
error
)
{
if
k
,
ok
:=
Kinds
[
c
.
Kind
];
ok
{
return
k
(
c
)
}
// default to RatioCompressor
return
Kinds
[
RatioKind
](
c
)
}
}
op-batcher/compressor/factory.go
deleted
100644 → 0
View file @
6f831fdc
package
compressor
import
(
"strings"
"github.com/ethereum-optimism/optimism/op-node/rollup/derive"
)
type
FactoryFunc
func
(
Config
)
(
derive
.
Compressor
,
error
)
type
FactoryFlag
struct
{
FlagValue
string
FactoryFunc
FactoryFunc
}
var
(
// The Ratio Factory creates new RatioCompressor's (see NewRatioCompressor
// for a description).
Ratio
=
FactoryFlag
{
FlagValue
:
"ratio"
,
FactoryFunc
:
NewRatioCompressor
,
}
// The Shadow Factory creates new ShadowCompressor's (see NewShadowCompressor
// for a description).
Shadow
=
FactoryFlag
{
FlagValue
:
"shadow"
,
FactoryFunc
:
NewShadowCompressor
,
}
)
var
Factories
=
[]
FactoryFlag
{
Ratio
,
Shadow
,
}
func
FactoryFlags
()
string
{
var
out
strings
.
Builder
for
i
,
v
:=
range
Factories
{
out
.
WriteString
(
v
.
FlagValue
)
if
i
+
1
<
len
(
Factories
)
{
out
.
WriteString
(
", "
)
}
}
return
out
.
String
()
}
op-e2e/migration_test.go
View file @
6b6bc5ff
...
@@ -337,12 +337,9 @@ func TestMigration(t *testing.T) {
...
@@ -337,12 +337,9 @@ func TestMigration(t *testing.T) {
MaxChannelDuration
:
1
,
MaxChannelDuration
:
1
,
MaxL1TxSize
:
120
_000
,
MaxL1TxSize
:
120
_000
,
CompressorConfig
:
compressor
.
CLIConfig
{
CompressorConfig
:
compressor
.
CLIConfig
{
Type
:
compressor
.
Ratio
.
FlagValue
,
TargetL1TxSizeBytes
:
100
_000
,
Config
:
compressor
.
Config
{
TargetNumFrames
:
1
,
TargetFrameSize
:
100
_000
-
1
,
ApproxComprRatio
:
0.4
,
TargetNumFrames
:
1
,
ApproxComprRatio
:
0.4
,
},
},
},
SubSafetyMargin
:
4
,
SubSafetyMargin
:
4
,
PollInterval
:
50
*
time
.
Millisecond
,
PollInterval
:
50
*
time
.
Millisecond
,
...
...
op-e2e/setup.go
View file @
6b6bc5ff
...
@@ -601,12 +601,9 @@ func (cfg SystemConfig) Start(_opts ...SystemConfigOption) (*System, error) {
...
@@ -601,12 +601,9 @@ func (cfg SystemConfig) Start(_opts ...SystemConfigOption) (*System, error) {
MaxChannelDuration
:
1
,
MaxChannelDuration
:
1
,
MaxL1TxSize
:
120
_000
,
MaxL1TxSize
:
120
_000
,
CompressorConfig
:
compressor
.
CLIConfig
{
CompressorConfig
:
compressor
.
CLIConfig
{
Type
:
compressor
.
Ratio
.
FlagValue
,
TargetL1TxSizeBytes
:
100
_000
,
Config
:
compressor
.
Config
{
TargetNumFrames
:
1
,
TargetFrameSize
:
100
_000
-
1
,
ApproxComprRatio
:
0.4
,
TargetNumFrames
:
1
,
ApproxComprRatio
:
0.4
,
},
},
},
SubSafetyMargin
:
4
,
SubSafetyMargin
:
4
,
PollInterval
:
50
*
time
.
Millisecond
,
PollInterval
:
50
*
time
.
Millisecond
,
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment