Commit 17f516ff authored by Yann Hodique's avatar Yann Hodique Committed by GitHub

feat(kurtosis-devnet): devnet manifest implementation (#13823)

This change adds a "compilation" phase for devnet manifest.
Bottom line, we:

- treat the devnet manifest as a high-level specification of an
  expected deployment

- open up the possibility to generate an actionable input for an
  arbitrary deployer

- use kurtosis as a first target, by generating a valid kurtosis input
  that reflects the properties defined in the manifest

pkg/devnet/cmd/main.go is a toy tool to perform that last step.

Going forward, we could imagine a k8s deployment being "compiled"
following a similar process.
Or this being used during automated tests setup in order to create the
right target environment.

At a high-level, this is part of an effort to standardize our sources
of truth across the board.
parent 469577f7
package main
import (
"fmt"
"os"
"github.com/ethereum-optimism/optimism/kurtosis-devnet/pkg/devnet/kt"
"github.com/ethereum-optimism/optimism/kurtosis-devnet/pkg/devnet/manifest"
"github.com/urfave/cli/v2"
"gopkg.in/yaml.v3"
)
func main() {
app := &cli.App{
Name: "devnet",
Usage: "Generate Kurtosis parameters from a devnet manifest",
Flags: []cli.Flag{
&cli.StringFlag{
Name: "manifest",
Aliases: []string{"m"},
Usage: "Path to the manifest YAML file",
Required: true,
},
&cli.StringFlag{
Name: "output",
Aliases: []string{"o"},
Usage: "Path to write the Kurtosis parameters file (default: stdout)",
},
},
Action: func(c *cli.Context) error {
// Read manifest file
manifestPath := c.String("manifest")
manifestBytes, err := os.ReadFile(manifestPath)
if err != nil {
return fmt.Errorf("failed to read manifest file: %w", err)
}
// Parse manifest YAML
var m manifest.Manifest
if err := yaml.Unmarshal(manifestBytes, &m); err != nil {
return fmt.Errorf("failed to parse manifest YAML: %w", err)
}
// Create visitor and process manifest
visitor := kt.NewKurtosisVisitor()
m.Accept(visitor)
// Get params and write to file or stdout
params := visitor.GetParams()
paramsBytes, err := yaml.Marshal(params)
if err != nil {
return fmt.Errorf("failed to marshal params: %w", err)
}
outputPath := c.String("output")
if outputPath != "" {
if err := os.WriteFile(outputPath, paramsBytes, 0644); err != nil {
return fmt.Errorf("failed to write params file: %w", err)
}
} else {
fmt.Print(string(paramsBytes))
}
return nil
},
}
if err := app.Run(os.Args); err != nil {
fmt.Fprintf(os.Stderr, "error: %v\n", err)
os.Exit(1)
}
}
package images
import "fmt"
// Repository maps component versions to their corresponding Docker image URLs
type Repository struct {
mapping map[string]string
}
const (
opLabsToolsRegistry = "us-docker.pkg.dev/oplabs-tools-artifacts/images"
paradigmRegistry = "ghcr.io/paradigmxyz"
)
// NewRepository creates a new Repository instance with predefined mappings
func NewRepository() *Repository {
return &Repository{
mapping: map[string]string{
// OP Labs images
"op-deployer": opLabsToolsRegistry,
"op-geth": opLabsToolsRegistry,
"op-node": opLabsToolsRegistry,
"op-batcher": opLabsToolsRegistry,
"op-proposer": opLabsToolsRegistry,
"op-challenger": opLabsToolsRegistry,
// Paradigm images
"op-reth": paradigmRegistry,
},
}
}
// GetImage returns the full Docker image URL for a given component and version
func (r *Repository) GetImage(component string, version string) string {
if imageTemplate, ok := r.mapping[component]; ok {
if version == "" {
version = "latest"
}
return fmt.Sprintf("%s/%s:%s", imageTemplate, component, version)
}
// TODO: that's our way to convey that the "default" image should be used.
// We should probably have a more explicit way to do this.
return ""
}
package kt
// KurtosisParams represents the top-level Kurtosis configuration
type KurtosisParams struct {
OptimismPackage OptimismPackage `yaml:"optimism_package"`
EthereumPackage EthereumPackage `yaml:"ethereum_package"`
}
// OptimismPackage represents the Optimism-specific configuration
type OptimismPackage struct {
Chains []ChainConfig `yaml:"chains"`
OpContractDeployerParams OpContractDeployerParams `yaml:"op_contract_deployer_params"`
Persistent bool `yaml:"persistent"`
}
// ChainConfig represents a single chain configuration
type ChainConfig struct {
Participants []ParticipantConfig `yaml:"participants"`
NetworkParams NetworkParams `yaml:"network_params"`
BatcherParams BatcherParams `yaml:"batcher_params"`
ChallengerParams ChallengerParams `yaml:"challenger_params"`
ProposerParams ProposerParams `yaml:"proposer_params"`
}
// ParticipantConfig represents a participant in the network
type ParticipantConfig struct {
ElType string `yaml:"el_type"`
ElImage string `yaml:"el_image"`
ClType string `yaml:"cl_type"`
ClImage string `yaml:"cl_image"`
Count int `yaml:"count"`
}
// TimeOffsets represents a map of time offset values
type TimeOffsets map[string]int
// NetworkParams represents network-specific parameters
type NetworkParams struct {
Network string `yaml:"network"`
NetworkID string `yaml:"network_id"`
SecondsPerSlot int `yaml:"seconds_per_slot"`
Name string `yaml:"name"`
FundDevAccounts bool `yaml:"fund_dev_accounts"`
TimeOffsets `yaml:",inline"`
}
// BatcherParams represents batcher-specific parameters
type BatcherParams struct {
Image string `yaml:"image"`
}
// ChallengerParams represents challenger-specific parameters
type ChallengerParams struct {
Image string `yaml:"image"`
CannonPrestatesURL string `yaml:"cannon_prestates_url,omitempty"`
}
// ProposerParams represents proposer-specific parameters
type ProposerParams struct {
Image string `yaml:"image"`
GameType int `yaml:"game_type"`
ProposalInterval string `yaml:"proposal_interval"`
}
// OpContractDeployerParams represents contract deployer parameters
type OpContractDeployerParams struct {
Image string `yaml:"image"`
L1ArtifactsLocator string `yaml:"l1_artifacts_locator"`
L2ArtifactsLocator string `yaml:"l2_artifacts_locator"`
}
// EthereumPackage represents Ethereum-specific configuration
type EthereumPackage struct {
NetworkParams EthereumNetworkParams `yaml:"network_params"`
}
// EthereumNetworkParams represents Ethereum network parameters
type EthereumNetworkParams struct {
Preset string `yaml:"preset"`
GenesisDelay int `yaml:"genesis_delay"`
AdditionalPreloadedContracts string `yaml:"additional_preloaded_contracts"`
}
package kt
import (
"strconv"
"strings"
"github.com/ethereum-optimism/optimism/kurtosis-devnet/pkg/devnet/images"
"github.com/ethereum-optimism/optimism/kurtosis-devnet/pkg/devnet/manifest"
)
const (
defaultProposalInterval = "10m"
defaultGameType = 1
defaultPreset = "minimal"
defaultGenesisDelay = 5
defaultPreloadedContracts = `{
"0x4e59b44847b379578588920cA78FbF26c0B4956C": {
"balance": "0ETH",
"code": "0x7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe03601600081602082378035828234f58015156039578182fd5b8082525050506014600cf3",
"storage": {},
"nonce": "1"
}
}`
)
// KurtosisVisitor implements the manifest.ManifestVisitor interface
type KurtosisVisitor struct {
params *KurtosisParams
repository *images.Repository
l2Visitor *l2Visitor
}
// Component visitor for handling component versions
type componentVisitor struct {
name string
version string
}
// Chain visitor for handling chain configuration
type chainVisitor struct {
name string
id uint64
}
// Contracts visitor for handling contract configuration
type contractsVisitor struct {
locator string
}
// Overrides represents deployment overrides
type Overrides struct {
SecondsPerSlot int `yaml:"seconds_per_slot"`
TimeOffsets `yaml:",inline"`
}
// Deployment visitor for handling deployment configuration
type deploymentVisitor struct {
deployer *componentVisitor
l1Contracts *contractsVisitor
l2Contracts *contractsVisitor
overrides *Overrides
}
// L2 visitor for handling L2 configuration
type l2Visitor struct {
components map[string]*componentVisitor
deployment *deploymentVisitor
chains []*chainVisitor
}
// NewKurtosisVisitor creates a new KurtosisVisitor
func NewKurtosisVisitor() *KurtosisVisitor {
return &KurtosisVisitor{
params: &KurtosisParams{
OptimismPackage: OptimismPackage{
Chains: make([]ChainConfig, 0),
Persistent: false,
},
EthereumPackage: EthereumPackage{
NetworkParams: EthereumNetworkParams{
Preset: defaultPreset,
GenesisDelay: defaultGenesisDelay,
AdditionalPreloadedContracts: defaultPreloadedContracts,
},
},
},
repository: images.NewRepository(),
}
}
func (v *KurtosisVisitor) VisitName(name string) {}
func (v *KurtosisVisitor) VisitType(manifestType string) {}
func (v *KurtosisVisitor) VisitL1() manifest.ChainVisitor {
return &chainVisitor{}
}
func (v *KurtosisVisitor) VisitL2() manifest.L2Visitor {
v.l2Visitor = &l2Visitor{
components: make(map[string]*componentVisitor),
deployment: &deploymentVisitor{
deployer: &componentVisitor{},
l1Contracts: &contractsVisitor{},
l2Contracts: &contractsVisitor{},
overrides: &Overrides{
TimeOffsets: make(TimeOffsets),
},
},
chains: make([]*chainVisitor, 0),
}
return v.l2Visitor
}
// Component visitor implementation
func (v *componentVisitor) VisitVersion(version string) {
// Strip the component name from the version string
parts := strings.SplitN(version, "/", 2)
if len(parts) == 2 {
v.version = parts[1]
} else {
v.version = version
}
}
// Chain visitor implementation
func (v *chainVisitor) VisitName(name string) {
v.name = name
}
func (v *chainVisitor) VisitID(id uint64) {
// TODO: this is horrible but unfortunately the funding script breaks for
// chain IDs larger than 32 bits.
v.id = id & 0xFFFFFFFF
}
// Contracts visitor implementation
func (v *contractsVisitor) VisitVersion(version string) {
if v.locator == "" {
v.locator = "tag://" + version
}
}
func (v *contractsVisitor) VisitLocator(locator string) {
v.locator = locator
}
// Deployment visitor implementation
func (v *deploymentVisitor) VisitDeployer() manifest.ComponentVisitor {
return v.deployer
}
func (v *deploymentVisitor) VisitL1Contracts() manifest.ContractsVisitor {
return v.l1Contracts
}
func (v *deploymentVisitor) VisitL2Contracts() manifest.ContractsVisitor {
return v.l2Contracts
}
func (v *deploymentVisitor) VisitOverride(key string, value interface{}) {
if key == "seconds_per_slot" {
if intValue, ok := value.(int); ok {
v.overrides.SecondsPerSlot = intValue
}
} else if strings.HasSuffix(key, "_time_offset") {
if intValue, ok := value.(int); ok {
v.overrides.TimeOffsets[key] = intValue
}
}
}
// L2 visitor implementation
func (v *l2Visitor) VisitL2Component(name string) manifest.ComponentVisitor {
comp := &componentVisitor{name: name}
v.components[name] = comp
return comp
}
func (v *l2Visitor) VisitL2Deployment() manifest.DeploymentVisitor {
return v.deployment
}
func (v *l2Visitor) VisitL2Chain(idx int) manifest.ChainVisitor {
chain := &chainVisitor{}
if idx >= len(v.chains) {
v.chains = append(v.chains, chain)
} else {
v.chains[idx] = chain
}
return chain
}
// GetParams returns the generated Kurtosis parameters
func (v *KurtosisVisitor) GetParams() *KurtosisParams {
if v.l2Visitor != nil {
v.BuildKurtosisParams(v.l2Visitor)
}
return v.params
}
// getComponentVersion returns the version for a component, or empty string if not found
func (l2 *l2Visitor) getComponentVersion(name string) string {
if comp, ok := l2.components[name]; ok {
return comp.version
}
return ""
}
// getComponentImage returns the image for a component, or empty string if component doesn't exist
func (v *KurtosisVisitor) getComponentImage(l2 *l2Visitor, name string) string {
if _, ok := l2.components[name]; ok {
return v.repository.GetImage(name, l2.getComponentVersion(name))
}
return ""
}
// BuildKurtosisParams builds the final Kurtosis parameters from the collected visitor data
func (v *KurtosisVisitor) BuildKurtosisParams(l2 *l2Visitor) {
// Set deployer params
v.params.OptimismPackage.OpContractDeployerParams = OpContractDeployerParams{
Image: v.repository.GetImage("op-deployer", l2.deployment.deployer.version),
L1ArtifactsLocator: l2.deployment.l1Contracts.locator,
L2ArtifactsLocator: l2.deployment.l2Contracts.locator,
}
// Build chain configs
for _, chain := range l2.chains {
// Create network params with embedded map
networkParams := NetworkParams{
Network: "kurtosis",
NetworkID: strconv.FormatUint(chain.id, 10),
SecondsPerSlot: l2.deployment.overrides.SecondsPerSlot,
Name: chain.name,
FundDevAccounts: true,
TimeOffsets: l2.deployment.overrides.TimeOffsets,
}
chainConfig := ChainConfig{
Participants: []ParticipantConfig{
{
ElType: "op-geth",
ElImage: v.getComponentImage(l2, "op-geth"),
ClType: "op-node",
ClImage: v.getComponentImage(l2, "op-node"),
Count: 1,
},
},
NetworkParams: networkParams,
BatcherParams: BatcherParams{
Image: v.getComponentImage(l2, "op-batcher"),
},
ChallengerParams: ChallengerParams{
Image: v.getComponentImage(l2, "op-challenger"),
},
ProposerParams: ProposerParams{
Image: v.getComponentImage(l2, "op-proposer"),
GameType: defaultGameType,
ProposalInterval: defaultProposalInterval,
},
}
v.params.OptimismPackage.Chains = append(v.params.OptimismPackage.Chains, chainConfig)
}
}
package kt
import (
"testing"
"github.com/ethereum-optimism/optimism/kurtosis-devnet/pkg/devnet/manifest"
"github.com/stretchr/testify/require"
"gopkg.in/yaml.v3"
)
func TestKurtosisVisitor_TransformsManifest(t *testing.T) {
input := `
name: alpaca
type: alphanet
l1:
name: sepolia
chain_id: 11155111
l2:
deployment:
op-deployer:
version: op-deployer/v0.0.11
l1-contracts:
locator: https://storage.googleapis.com/oplabs-contract-artifacts/artifacts-v1-c3f2e2adbd52a93c2c08cab018cd637a4e203db53034e59c6c139c76b4297953.tar.gz
version: 984bae9146398a2997ec13757bfe2438ca8f92eb
l2-contracts:
version: op-contracts/v.1.7.0-beta.1+l2-contracts
overrides:
seconds_per_slot: 2
fjord_time_offset: 0
granite_time_offset: 0
holocene_time_offset: 0
components:
op-node:
version: op-node/v1.10.2
op-geth:
version: op-geth/v1.101411.4-rc.4
op-reth:
version: op-reth/v1.1.5
op-proposer:
version: op-proposer/v1.10.0-rc.2
op-batcher:
version: op-batcher/v1.10.0
op-challenger:
version: op-challenger/v1.3.1-rc.4
chains:
- name: alpaca-0
chain_id: 11155111100000
`
// Then the output should match the expected YAML structure
expected := KurtosisParams{
OptimismPackage: OptimismPackage{
Chains: []ChainConfig{
{
Participants: []ParticipantConfig{
{
ElType: "op-geth",
ElImage: "us-docker.pkg.dev/oplabs-tools-artifacts/images/op-geth:v1.101411.4-rc.4",
ClType: "op-node",
ClImage: "us-docker.pkg.dev/oplabs-tools-artifacts/images/op-node:v1.10.2",
Count: 1,
},
},
NetworkParams: NetworkParams{
Network: "kurtosis",
NetworkID: "11155111100000",
SecondsPerSlot: 2,
Name: "alpaca-0",
FundDevAccounts: true,
TimeOffsets: TimeOffsets{
"fjord_time_offset": 0,
"granite_time_offset": 0,
"holocene_time_offset": 0,
},
},
BatcherParams: BatcherParams{
Image: "us-docker.pkg.dev/oplabs-tools-artifacts/images/op-batcher:v1.10.0",
},
ChallengerParams: ChallengerParams{
Image: "us-docker.pkg.dev/oplabs-tools-artifacts/images/op-challenger:v1.3.1-rc.4",
CannonPrestatesURL: "",
},
ProposerParams: ProposerParams{
Image: "us-docker.pkg.dev/oplabs-tools-artifacts/images/op-proposer:v1.10.0-rc.2",
GameType: 1,
ProposalInterval: "10m",
},
},
},
OpContractDeployerParams: OpContractDeployerParams{
Image: "us-docker.pkg.dev/oplabs-tools-artifacts/images/op-deployer:v0.0.11",
L1ArtifactsLocator: "https://storage.googleapis.com/oplabs-contract-artifacts/artifacts-v1-c3f2e2adbd52a93c2c08cab018cd637a4e203db53034e59c6c139c76b4297953.tar.gz",
L2ArtifactsLocator: "op-contracts/v.1.7.0-beta.1+l2-contracts",
},
Persistent: false,
},
EthereumPackage: EthereumPackage{
NetworkParams: EthereumNetworkParams{
Preset: "minimal",
GenesisDelay: 5,
AdditionalPreloadedContracts: defaultPreloadedContracts,
},
},
}
// Convert the input to a manifest
var manifest manifest.Manifest
err := yaml.Unmarshal([]byte(input), &manifest)
require.NoError(t, err)
// Create visitor and have manifest accept it
visitor := NewKurtosisVisitor()
manifest.Accept(visitor)
// Get the generated params
actual := *visitor.GetParams()
// Compare the actual and expected params
require.Equal(t, expected, actual, "Generated params should match expected params")
}
package manifest
type ManifestAcceptor interface {
Accept(visitor ManifestVisitor)
}
type ChainAcceptor interface {
Accept(visitor ChainVisitor)
}
type L2Acceptor interface {
Accept(visitor L2Visitor)
}
type DeploymentAcceptor interface {
Accept(visitor DeploymentVisitor)
}
type ContractsAcceptor interface {
Accept(visitor ContractsVisitor)
}
type ComponentAcceptor interface {
Accept(visitor ComponentVisitor)
}
package manifest
// L1Config represents L1 configuration
type L1Config struct {
Name string `yaml:"name"`
ChainID uint64 `yaml:"chain_id"`
}
func (c *L1Config) Accept(visitor ChainVisitor) {
visitor.VisitName(c.Name)
visitor.VisitID(c.ChainID)
}
var _ ChainAcceptor = (*L1Config)(nil)
type Component struct {
Version string `yaml:"version"`
}
func (c *Component) Accept(visitor ComponentVisitor) {
visitor.VisitVersion(c.Version)
}
var _ ComponentAcceptor = (*Component)(nil)
type Contracts struct {
Version string `yaml:"version"`
Locator string `yaml:"locator"`
}
func (c *Contracts) Accept(visitor ContractsVisitor) {
visitor.VisitLocator(c.Locator)
visitor.VisitVersion(c.Version)
}
var _ ContractsAcceptor = (*Contracts)(nil)
// L2Deployment represents deployment configuration
type L2Deployment struct {
OpDeployer *Component `yaml:"op-deployer"`
L1Contracts *Contracts `yaml:"l1-contracts"`
L2Contracts *Contracts `yaml:"l2-contracts"`
Overrides map[string]interface{} `yaml:"overrides"`
}
func (d *L2Deployment) Accept(visitor DeploymentVisitor) {
d.OpDeployer.Accept(visitor.VisitDeployer())
d.L1Contracts.Accept(visitor.VisitL1Contracts())
d.L2Contracts.Accept(visitor.VisitL2Contracts())
for key, value := range d.Overrides {
visitor.VisitOverride(key, value)
}
}
var _ DeploymentAcceptor = (*L2Deployment)(nil)
// L2Chain represents an L2 chain configuration
type L2Chain struct {
Name string `yaml:"name"`
ChainID uint64 `yaml:"chain_id"`
}
func (c *L2Chain) Accept(visitor ChainVisitor) {
visitor.VisitName(c.Name)
visitor.VisitID(c.ChainID)
}
var _ ChainAcceptor = (*L2Chain)(nil)
// L2Config represents L2 configuration
type L2Config struct {
Deployment *L2Deployment `yaml:"deployment"`
Components map[string]*Component `yaml:"components"`
Chains []*L2Chain `yaml:"chains"`
}
func (c *L2Config) Accept(visitor L2Visitor) {
for name, component := range c.Components {
component.Accept(visitor.VisitL2Component(name))
}
for i, chain := range c.Chains {
chain.Accept(visitor.VisitL2Chain(i))
}
c.Deployment.Accept(visitor.VisitL2Deployment())
}
var _ L2Acceptor = (*L2Config)(nil)
// Manifest represents the top-level manifest configuration
type Manifest struct {
Name string `yaml:"name"`
Type string `yaml:"type"`
L1 *L1Config `yaml:"l1"`
L2 *L2Config `yaml:"l2"`
}
func (m *Manifest) Accept(visitor ManifestVisitor) {
visitor.VisitName(m.Name)
visitor.VisitType(m.Type)
m.L1.Accept(visitor.VisitL1())
m.L2.Accept(visitor.VisitL2())
}
var _ ManifestAcceptor = (*Manifest)(nil)
package manifest
type ManifestVisitor interface {
VisitName(name string)
VisitType(manifestType string)
VisitL1() ChainVisitor
VisitL2() L2Visitor
}
type L2Visitor interface {
VisitL2Component(name string) ComponentVisitor
VisitL2Deployment() DeploymentVisitor
VisitL2Chain(int) ChainVisitor
}
type ComponentVisitor interface {
VisitVersion(version string)
}
type DeploymentVisitor interface {
VisitDeployer() ComponentVisitor
VisitL1Contracts() ContractsVisitor
VisitL2Contracts() ContractsVisitor
VisitOverride(string, interface{})
}
type ContractsVisitor interface {
VisitVersion(version string)
VisitLocator(locator string)
}
type ChainVisitor interface {
VisitName(name string)
VisitID(id uint64)
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment