Skip to content

N
nebula
Commit e6b022b2 authored by Axel Kingsley's avatar Axel Kingsley Committed by GitHub
Browse files
Options

Fix Benchmarking ; add Compression Algo to Test Case Definitions (#10693) 

* Fix Benchmarking ; add Compression Algo to Test Case Definitions

* Correct to Brotli9
parent 03289ebf
Hide whitespace changes
Inline Side-by-side
Showing with 165 additions and 123 deletions
op-node/benchmarks/batchbuilding_test.go
View file @ e6b022b2
...@@ -19,52 +19,81 @@ const ( ...@@ -19,52 +19,81 @@ const (
targetOuput_real = uint64(780120) targetOuput_real = uint64(780120)
) )
var ( // compressorDetails is a helper struct to create compressors or supply the configuration for span batches
// compressors used in the benchmark type compressorDetails struct {
rc, _ = compressor.NewRatioCompressor(compressor.Config{ name string
TargetOutputSize: targetOutput_huge, compressorFn func(compressor.Config) (derive.Compressor, error)
ApproxComprRatio: 0.4, config compressor.Config
}) }
sc, _ = compressor.NewShadowCompressor(compressor.Config{
TargetOutputSize: targetOutput_huge,
})
nc, _ = compressor.NewNonCompressor(compressor.Config{
TargetOutputSize: targetOutput_huge,
})
realsc, _ = compressor.NewShadowCompressor(compressor.Config{
TargetOutputSize: targetOuput_real,
})
// compressors used in the benchmark mapped by their name func (cd compressorDetails) String() string {
// they come paired with a target output size so span batches can use the target size directly return fmt.Sprintf("%s-%s-%d", cd.name, cd.config.CompressionAlgo, cd.config.TargetOutputSize)
compressors = map[string]compressorAndTarget{ }
"NonCompressor": {nc, targetOutput_huge}, func (cd compressorDetails) Compressor() (derive.Compressor, error) {
"RatioCompressor": {rc, targetOutput_huge}, return cd.compressorFn(cd.config)
"ShadowCompressor": {sc, targetOutput_huge}, }
"RealShadowCompressor": {realsc, targetOuput_real},
} var (
// batch types used in the benchmark // batch types used in the benchmark
batchTypes = []uint{ batchTypes = []uint{
derive.SpanBatchType, derive.SpanBatchType,
// uncomment to include singular batches in the benchmark
// singular batches are not included by default because they are not the target of the benchmark
// derive.SingularBatchType,
} }
)
type compressorAndTarget struct { compAlgos = []derive.CompressionAlgo{
compressor derive.Compressor derive.Zlib,
targetOutput uint64 derive.Brotli,
} derive.Brotli9,
derive.Brotli11,
}
// compressors used in the benchmark
// they are all configured to Zlib compression, which may be overridden in the test cases
compressors = map[string]compressorDetails{
"NonCompressor": {
name: "NonCompressor",
compressorFn: compressor.NewNonCompressor,
config: compressor.Config{
TargetOutputSize: targetOutput_huge,
CompressionAlgo: derive.Zlib,
},
},
"RatioCompressor": {
name: "RatioCompressor",
compressorFn: compressor.NewRatioCompressor,
config: compressor.Config{
TargetOutputSize: targetOutput_huge,
CompressionAlgo: derive.Zlib,
},
},
"ShadowCompressor": {
name: "ShadowCompressor",
compressorFn: compressor.NewShadowCompressor,
config: compressor.Config{
TargetOutputSize: targetOutput_huge,
CompressionAlgo: derive.Zlib,
},
},
"RealShadowCompressor": {
name: "ShadowCompressor",
compressorFn: compressor.NewShadowCompressor,
config: compressor.Config{
TargetOutputSize: targetOuput_real,
CompressionAlgo: derive.Zlib,
},
},
}
)
// channelOutByType returns a channel out of the given type as a helper for the benchmarks // channelOutByType returns a channel out of the given type as a helper for the benchmarks
func channelOutByType(batchType uint, compKey string, algo derive.CompressionAlgo) (derive.ChannelOut, error) { func channelOutByType(b *testing.B, batchType uint, cd compressorDetails) (derive.ChannelOut, error) {
chainID := big.NewInt(333) chainID := big.NewInt(333)
if batchType == derive.SingularBatchType { if batchType == derive.SingularBatchType {
return derive.NewSingularChannelOut(compressors[compKey].compressor) compressor, err := cd.Compressor()
require.NoError(b, err)
return derive.NewSingularChannelOut(compressor)
} }
if batchType == derive.SpanBatchType { if batchType == derive.SpanBatchType {
return derive.NewSpanChannelOut(0, chainID, compressors[compKey].targetOutput, algo) return derive.NewSpanChannelOut(0, chainID, cd.config.TargetOutputSize, cd.config.CompressionAlgo)
} }
return nil, fmt.Errorf("unsupported batch type: %d", batchType) return nil, fmt.Errorf("unsupported batch type: %d", batchType)
} }
...@@ -75,7 +104,7 @@ type BatchingBenchmarkTC struct { ...@@ -75,7 +104,7 @@ type BatchingBenchmarkTC struct {
BatchType uint BatchType uint
BatchCount int BatchCount int
txPerBatch int txPerBatch int
compKey string cd compressorDetails
} }
func (t BatchingBenchmarkTC) String() string { func (t BatchingBenchmarkTC) String() string {
...@@ -86,7 +115,7 @@ func (t BatchingBenchmarkTC) String() string { ...@@ -86,7 +115,7 @@ func (t BatchingBenchmarkTC) String() string {
if t.BatchType == derive.SpanBatchType { if t.BatchType == derive.SpanBatchType {
btype = "Span" btype = "Span"
} }
return fmt.Sprintf("BatchType=%s, txPerBatch=%d, BatchCount=%d, Compressor=%s", btype, t.txPerBatch, t.BatchCount, t.compKey) return fmt.Sprintf("BatchType=%s, txPerBatch=%d, BatchCount=%d, Compressor=%s", btype, t.txPerBatch, t.BatchCount, t.cd.String())
} }
// BenchmarkChannelOut benchmarks the performance of adding singular batches to a channel out // BenchmarkChannelOut benchmarks the performance of adding singular batches to a channel out
...@@ -97,6 +126,7 @@ func (t BatchingBenchmarkTC) String() string { ...@@ -97,6 +126,7 @@ func (t BatchingBenchmarkTC) String() string {
// Hint: Raise the rollup.MaxRLPBytesPerChannel to 10_000_000_000 to avoid hitting limits if adding larger test cases // Hint: Raise the rollup.MaxRLPBytesPerChannel to 10_000_000_000 to avoid hitting limits if adding larger test cases
func BenchmarkFinalBatchChannelOut(b *testing.B) { func BenchmarkFinalBatchChannelOut(b *testing.B) {
// Targets define the number of batches and transactions per batch to test // Targets define the number of batches and transactions per batch to test
// they will be multiplied by various compressors
type target struct{ bs, tpb int } type target struct{ bs, tpb int }
targets := []target{ targets := []target{
{10, 1}, {10, 1},
...@@ -107,12 +137,16 @@ func BenchmarkFinalBatchChannelOut(b *testing.B) { ...@@ -107,12 +137,16 @@ func BenchmarkFinalBatchChannelOut(b *testing.B) {
{100, 100}, {100, 100},
} }
// build a set of test cases for each batch type, compressor, and target-pair // make test-cases for every batch type, compressor, compressorAlgo, and target-pair
tests := []BatchingBenchmarkTC{} tests := []BatchingBenchmarkTC{}
for _, bt := range batchTypes { for _, bt := range batchTypes {
for compkey := range compressors { for _, compDetails := range compressors {
for _, t := range targets { for _, algo := range compAlgos {
tests = append(tests, BatchingBenchmarkTC{bt, t.bs, t.tpb, compkey}) for _, t := range targets {
cd := compDetails
cd.config.CompressionAlgo = algo
tests = append(tests, BatchingBenchmarkTC{bt, t.bs, t.tpb, cd})
}
} }
} }
} }
...@@ -129,27 +163,24 @@ func BenchmarkFinalBatchChannelOut(b *testing.B) { ...@@ -129,27 +163,24 @@ func BenchmarkFinalBatchChannelOut(b *testing.B) {
// to leverage optimizations in the Batch Linked List // to leverage optimizations in the Batch Linked List
batches[i].Timestamp = uint64(t.Add(time.Duration(i) * time.Second).Unix()) batches[i].Timestamp = uint64(t.Add(time.Duration(i) * time.Second).Unix())
} }
for _, algo := range derive.CompressionAlgos { b.Run(tc.String(), func(b *testing.B) {
b.Run(tc.String()+"_"+algo.String(), func(b *testing.B) { // reset the compressor used in the test case
// reset the compressor used in the test case for bn := 0; bn < b.N; bn++ {
for bn := 0; bn < b.N; bn++ { // don't measure the setup time
// don't measure the setup time b.StopTimer()
b.StopTimer() cout, _ := channelOutByType(b, tc.BatchType, tc.cd)
compressors[tc.compKey].compressor.Reset() // add all but the final batch to the channel out
cout, _ := channelOutByType(tc.BatchType, tc.compKey, algo) for i := 0; i < tc.BatchCount-1; i++ {
// add all but the final batch to the channel out err := cout.AddSingularBatch(batches[i], 0)
for i := 0; i < tc.BatchCount-1; i++ {
err := cout.AddSingularBatch(batches[i], 0)
require.NoError(b, err)
}
// measure the time to add the final batch
b.StartTimer()
// add the final batch to the channel out
err := cout.AddSingularBatch(batches[tc.BatchCount-1], 0)
require.NoError(b, err) require.NoError(b, err)
} }
}) // measure the time to add the final batch
} b.StartTimer()
// add the final batch to the channel out
err := cout.AddSingularBatch(batches[tc.BatchCount-1], 0)
require.NoError(b, err)
}
})
} }
} }
...@@ -165,40 +196,50 @@ func BenchmarkIncremental(b *testing.B) { ...@@ -165,40 +196,50 @@ func BenchmarkIncremental(b *testing.B) {
// use batchCount as the number of batches to add in each benchmark iteration // use batchCount as the number of batches to add in each benchmark iteration
// and use txPerBatch as the number of transactions per batch // and use txPerBatch as the number of transactions per batch
tcs := []BatchingBenchmarkTC{ tcs := []BatchingBenchmarkTC{
{derive.SpanBatchType, 5, 1, "RealBlindCompressor"}, {derive.SpanBatchType, 5, 1, compressorDetails{
//{derive.SingularBatchType, 100, 1, "RealShadowCompressor"}, name: "RealThreshold",
config: compressor.Config{
TargetOutputSize: targetOuput_real,
CompressionAlgo: derive.Zlib,
},
}},
{derive.SpanBatchType, 5, 1, compressorDetails{
name: "RealThreshold",
config: compressor.Config{
TargetOutputSize: targetOuput_real,
CompressionAlgo: derive.Brotli10,
},
}},
} }
for _, algo := range derive.CompressionAlgos { for _, tc := range tcs {
for _, tc := range tcs { cout, err := channelOutByType(b, tc.BatchType, tc.cd)
cout, err := channelOutByType(tc.BatchType, tc.compKey, algo) if err != nil {
if err != nil { b.Fatal(err)
b.Fatal(err) }
} done := false
done := false for base := 0; !done; base += tc.BatchCount {
for base := 0; !done; base += tc.BatchCount { rangeName := fmt.Sprintf("Incremental %s: %d-%d", tc.String(), base, base+tc.BatchCount)
rangeName := fmt.Sprintf("Incremental %s-%s: %d-%d", algo, tc.String(), base, base+tc.BatchCount) b.Run(rangeName, func(b *testing.B) {
b.Run(rangeName+"_"+algo.String(), func(b *testing.B) { b.StopTimer()
b.StopTimer() // prepare the batches
// prepare the batches t := time.Now()
t := time.Now() batches := make([]*derive.SingularBatch, tc.BatchCount)
batches := make([]*derive.SingularBatch, tc.BatchCount) for i := 0; i < tc.BatchCount; i++ {
for i := 0; i < tc.BatchCount; i++ { t := t.Add(time.Second)
t := t.Add(time.Second) batches[i] = derive.RandomSingularBatch(rng, tc.txPerBatch, chainID)
batches[i] = derive.RandomSingularBatch(rng, tc.txPerBatch, chainID) // set the timestamp to increase with each batch
// set the timestamp to increase with each batch // to leverage optimizations in the Batch Linked List
// to leverage optimizations in the Batch Linked List batches[i].Timestamp = uint64(t.Unix())
batches[i].Timestamp = uint64(t.Unix()) }
} b.StartTimer()
b.StartTimer() for i := 0; i < tc.BatchCount; i++ {
for i := 0; i < tc.BatchCount; i++ { err := cout.AddSingularBatch(batches[i], 0)
err := cout.AddSingularBatch(batches[i], 0) if err != nil {
if err != nil { done = true
done = true return
return
}
} }
}) }
} })
} }
} }
} }
...@@ -224,42 +265,43 @@ func BenchmarkAllBatchesChannelOut(b *testing.B) { ...@@ -224,42 +265,43 @@ func BenchmarkAllBatchesChannelOut(b *testing.B) {
// build a set of test cases for each batch type, compressor, and target-pair // build a set of test cases for each batch type, compressor, and target-pair
tests := []BatchingBenchmarkTC{} tests := []BatchingBenchmarkTC{}
for _, bt := range batchTypes { for _, bt := range batchTypes {
for compkey := range compressors { for _, compDetails := range compressors {
for _, t := range targets { for _, algo := range compAlgos {
tests = append(tests, BatchingBenchmarkTC{bt, t.bs, t.tpb, compkey}) for _, t := range targets {
cd := compDetails
cd.config.CompressionAlgo = algo
tests = append(tests, BatchingBenchmarkTC{bt, t.bs, t.tpb, cd})
}
} }
} }
} }
for _, algo := range derive.CompressionAlgos { for _, tc := range tests {
for _, tc := range tests { chainID := big.NewInt(333)
chainID := big.NewInt(333) rng := rand.New(rand.NewSource(0x543331))
rng := rand.New(rand.NewSource(0x543331)) // pre-generate batches to keep the benchmark from including the random generation
// pre-generate batches to keep the benchmark from including the random generation batches := make([]*derive.SingularBatch, tc.BatchCount)
batches := make([]*derive.SingularBatch, tc.BatchCount) t := time.Now()
t := time.Now() for i := 0; i < tc.BatchCount; i++ {
for i := 0; i < tc.BatchCount; i++ { batches[i] = derive.RandomSingularBatch(rng, tc.txPerBatch, chainID)
batches[i] = derive.RandomSingularBatch(rng, tc.txPerBatch, chainID) // set the timestamp to increase with each batch
// set the timestamp to increase with each batch // to leverage optimizations in the Batch Linked List
// to leverage optimizations in the Batch Linked List batches[i].Timestamp = uint64(t.Add(time.Duration(i) * time.Second).Unix())
batches[i].Timestamp = uint64(t.Add(time.Duration(i) * time.Second).Unix())
}
b.Run(tc.String()+"_"+algo.String(), func(b *testing.B) {
// reset the compressor used in the test case
for bn := 0; bn < b.N; bn++ {
// don't measure the setup time
b.StopTimer()
compressors[tc.compKey].compressor.Reset()
cout, _ := channelOutByType(tc.BatchType, tc.compKey, algo)
b.StartTimer()
// add all batches to the channel out
for i := 0; i < tc.BatchCount; i++ {
err := cout.AddSingularBatch(batches[i], 0)
require.NoError(b, err)
}
}
})
} }
b.Run(tc.String(), func(b *testing.B) {
// reset the compressor used in the test case
for bn := 0; bn < b.N; bn++ {
// don't measure the setup time
b.StopTimer()
cout, _ := channelOutByType(b, tc.BatchType, tc.cd)
b.StartTimer()
// add all batches to the channel out
for i := 0; i < tc.BatchCount; i++ {
err := cout.AddSingularBatch(batches[i], 0)
require.NoError(b, err)
}
}
})
} }
} }
...@@ -282,7 +324,7 @@ func BenchmarkGetRawSpanBatch(b *testing.B) { ...@@ -282,7 +324,7 @@ func BenchmarkGetRawSpanBatch(b *testing.B) {
tests := []BatchingBenchmarkTC{} tests := []BatchingBenchmarkTC{}
for _, t := range targets { for _, t := range targets {
tests = append(tests, BatchingBenchmarkTC{derive.SpanBatchType, t.bs, t.tpb, "NonCompressor"}) tests = append(tests, BatchingBenchmarkTC{derive.SpanBatchType, t.bs, t.tpb, compressors["NonCompressor"]})
} }
for _, tc := range tests { for _, tc := range tests {
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment