Commit c5007bb4 authored by Matthew Slipper's avatar Matthew Slipper Committed by GitHub

Add builds for op-deployer, bugfixes, artifacts downloads (#12033)

* Add builds for op-deployer, bugfixes, artifacts downloads

Adds Docker builds for op-deployer, makes some bugfixes, and adds support for downloading remote artifacts.

* Apply code scanning fix for arbitrary file access during archive extraction ("zip slip")
Co-authored-by: default avatarCopilot Autofix powered by AI <62310815+github-advanced-security[bot]@users.noreply.github.com>

* fix compile error

* lint

* fix test

* Update from code review, add docker build

* fix versioning

* remove errant dispatch

* update target

---------
Co-authored-by: default avatarCopilot Autofix powered by AI <62310815+github-advanced-security[bot]@users.noreply.github.com>
parent 0543bf7a
......@@ -1683,6 +1683,7 @@ workflows:
- op-conductor
- da-server
- op-supervisor
- op-deployer
- cannon-prestate:
requires:
- go-mod-download
......@@ -1742,6 +1743,7 @@ workflows:
- da-server
- op-ufm
- op-supervisor
- op-deployer
name: <<matrix.docker_name>>-docker-release
docker_tags: <<pipeline.git.revision>>
platforms: "linux/amd64,linux/arm64"
......@@ -1770,6 +1772,7 @@ workflows:
- da-server
- op-ufm
- op-supervisor
- op-deployer
name: <<matrix.op_component>>-cross-platform
requires:
- op-node-docker-release
......@@ -1781,6 +1784,7 @@ workflows:
- da-server-docker-release
- op-ufm-docker-release
- op-supervisor-docker-release
- op-deployer-docker-release
# Standard (xlarge) AMD-only docker images go here
- docker-build:
matrix:
......
......@@ -69,6 +69,9 @@ variable "OP_CONDUCTOR_VERSION" {
default = "${GIT_VERSION}"
}
variable "OP_DEPLOYER_VERSION" {
default = "${GIT_VERSION}"
}
target "op-node" {
dockerfile = "ops/docker/op-stack-go/Dockerfile"
......@@ -236,3 +239,16 @@ target "contracts-bedrock" {
platforms = ["linux/amd64"]
tags = [for tag in split(",", IMAGE_TAGS) : "${REGISTRY}/${REPOSITORY}/contracts-bedrock:${tag}"]
}
target "op-deployer" {
dockerfile = "ops/docker/op-stack-go/Dockerfile"
context = "."
args = {
GIT_COMMIT = "${GIT_COMMIT}"
GIT_DATE = "${GIT_DATE}"
OP_DEPLOYER_VERSION = "${OP_DEPLOYER_VERSION}"
}
target = "op-deployer-target"
platforms = split(",", PLATFORMS)
tags = [for tag in split(",", IMAGE_TAGS) : "${REGISTRY}/${REPOSITORY}/op-deployer:${tag}"]
}
GITCOMMIT ?= $(shell git rev-parse HEAD)
GITDATE ?= $(shell git show -s --format='%ct')
# Find the github tag that points to this commit. If none are found, set the version string to "untagged"
# Prioritizes release tag, if one exists, over tags suffixed with "-rc"
VERSION ?= $(shell tags=$$(git tag --points-at $(GITCOMMIT) | grep '^op-deployer/' | sed 's/op-deployer\///' | sort -V); \
preferred_tag=$$(echo "$$tags" | grep -v -- '-rc' | tail -n 1); \
if [ -z "$$preferred_tag" ]; then \
if [ -z "$$tags" ]; then \
echo "untagged"; \
else \
echo "$$tags" | tail -n 1; \
fi \
else \
echo $$preferred_tag; \
fi)
LDFLAGSSTRING +=-X main.GitCommit=$(GITCOMMIT)
LDFLAGSSTRING +=-X main.GitDate=$(GITDATE)
LDFLAGSSTRING +=-X github.com/ethereum-optimism/optimism/op-chain-ops/deployer/version.Version=$(VERSION)
LDFLAGSSTRING +=-X github.com/ethereum-optimism/optimism/op-chain-ops/deployer/version.Meta=$(VERSION_META)
LDFLAGS := -ldflags "$(LDFLAGSSTRING)"
# Use the old Apple linker to workaround broken xcode - https://github.com/golang/go/issues/65169
ifeq ($(shell uname),Darwin)
FUZZLDFLAGS := -ldflags=-extldflags=-Wl,-ld_classic
......@@ -13,7 +36,7 @@ test:
go test ./...
op-deployer:
go build -o ./bin/op-deployer ./cmd/op-deployer/main.go
GOOS=$(TARGETOS) GOARCH=$(TARGETARCH) CGO_ENABLED=0 go build -v $(LDFLAGS) -o ./bin/op-deployer ./cmd/op-deployer/main.go
fuzz:
go test $(FUZZLDFLAGS) -run NOTAREALTEST -v -fuzztime 10s -fuzz=FuzzEncodeDecodeWithdrawal ./crossdomain
......
......@@ -4,6 +4,9 @@ import (
"fmt"
"os"
"github.com/ethereum-optimism/optimism/op-chain-ops/deployer/version"
opservice "github.com/ethereum-optimism/optimism/op-service"
"github.com/ethereum-optimism/optimism/op-chain-ops/deployer/inspect"
"github.com/ethereum-optimism/optimism/op-chain-ops/deployer"
......@@ -11,8 +14,17 @@ import (
"github.com/urfave/cli/v2"
)
var (
GitCommit = ""
GitDate = ""
)
// VersionWithMeta holds the textual version string including the metadata.
var VersionWithMeta = opservice.FormatVersion(version.Version, GitCommit, GitDate, version.Meta)
func main() {
app := cli.NewApp()
app.Version = VersionWithMeta
app.Name = "op-deployer"
app.Usage = "Tool to configure and deploy OP Chains."
app.Flags = cliapp.ProtectFlags(deployer.GlobalFlags)
......
......@@ -6,6 +6,8 @@ import (
"fmt"
"strings"
"github.com/ethereum-optimism/optimism/op-chain-ops/foundry"
"github.com/ethereum-optimism/optimism/op-chain-ops/deployer/state"
"github.com/ethereum-optimism/optimism/op-chain-ops/deployer/pipeline"
......@@ -128,7 +130,7 @@ func Apply(ctx context.Context, cfg ApplyConfig) error {
type pipelineStage struct {
name string
stage pipeline.Stage
apply pipeline.Stage
}
func ApplyPipeline(
......@@ -137,6 +139,20 @@ func ApplyPipeline(
intent *state.Intent,
st *state.State,
) error {
progressor := func(curr, total int64) {
env.Logger.Info("artifacts download progress", "current", curr, "total", total)
}
artifactsFS, cleanup, err := pipeline.DownloadArtifacts(ctx, intent.ContractArtifactsURL, progressor)
if err != nil {
return fmt.Errorf("failed to download artifacts: %w", err)
}
defer func() {
if err := cleanup(); err != nil {
env.Logger.Warn("failed to clean up artifacts", "err", err)
}
}()
pline := []pipelineStage{
{"init", pipeline.Init},
{"deploy-superchain", pipeline.DeploySuperchain},
......@@ -144,22 +160,23 @@ func ApplyPipeline(
}
for _, chain := range intent.Chains {
chainID := chain.ID
pline = append(pline, pipelineStage{
fmt.Sprintf("deploy-opchain-%s", chain.ID.Hex()),
func(ctx context.Context, env *pipeline.Env, intent *state.Intent, st *state.State) error {
return pipeline.DeployOPChain(ctx, env, intent, st, chain.ID)
fmt.Sprintf("deploy-opchain-%s", chainID.Hex()),
func(ctx context.Context, env *pipeline.Env, artifactsFS foundry.StatDirFs, intent *state.Intent, st *state.State) error {
return pipeline.DeployOPChain(ctx, env, artifactsFS, intent, st, chainID)
},
}, pipelineStage{
fmt.Sprintf("generate-l2-genesis-%s", chain.ID.Hex()),
func(ctx context.Context, env *pipeline.Env, intent *state.Intent, st *state.State) error {
return pipeline.GenerateL2Genesis(ctx, env, intent, st, chain.ID)
fmt.Sprintf("generate-l2-genesis-%s", chainID.Hex()),
func(ctx context.Context, env *pipeline.Env, artifactsFS foundry.StatDirFs, intent *state.Intent, st *state.State) error {
return pipeline.GenerateL2Genesis(ctx, env, artifactsFS, intent, st, chainID)
},
})
}
for _, stage := range pline {
if err := stage.stage(ctx, env, intent, st); err != nil {
return fmt.Errorf("error in pipeline stage: %w", err)
if err := stage.apply(ctx, env, artifactsFS, intent, st); err != nil {
return fmt.Errorf("error in pipeline stage apply: %w", err)
}
}
......
......@@ -12,9 +12,9 @@ const (
EnvVarPrefix = "DEPLOYER"
L1RPCURLFlagName = "l1-rpc-url"
L1ChainIDFlagName = "l1-chain-id"
L2ChainIDsFlagName = "l2-chain-ids"
WorkdirFlagName = "workdir"
OutdirFlagName = "outdir"
DevFlagName = "dev"
PrivateKeyFlagName = "private-key"
)
......@@ -33,6 +33,11 @@ var (
EnvVars: prefixEnvVar("L1_CHAIN_ID"),
Value: 900,
}
L2ChainIDsFlag = &cli.StringFlag{
Name: L2ChainIDsFlagName,
Usage: "Comma-separated list of L2 chain IDs to deploy.",
EnvVars: prefixEnvVar("L2_CHAIN_IDS"),
}
WorkdirFlag = &cli.StringFlag{
Name: WorkdirFlagName,
Usage: "Directory storing intent and stage. Defaults to the current directory.",
......@@ -42,12 +47,6 @@ var (
OutdirFlagName,
},
}
DevFlag = &cli.BoolFlag{
Name: DevFlagName,
Usage: "Use development mode. This will use the development mnemonic to own the chain" +
" and fund dev accounts.",
EnvVars: prefixEnvVar("DEV"),
}
PrivateKeyFlag = &cli.StringFlag{
Name: PrivateKeyFlagName,
......@@ -60,8 +59,8 @@ var GlobalFlags = append([]cli.Flag{}, oplog.CLIFlags(EnvVarPrefix)...)
var InitFlags = []cli.Flag{
L1ChainIDFlag,
L2ChainIDsFlag,
WorkdirFlag,
DevFlag,
}
var ApplyFlags = []cli.Flag{
......
......@@ -3,6 +3,9 @@ package deployer
import (
"fmt"
"path"
"strings"
op_service "github.com/ethereum-optimism/optimism/op-service"
"github.com/ethereum-optimism/optimism/op-chain-ops/deployer/state"
"github.com/ethereum-optimism/optimism/op-chain-ops/devkeys"
......@@ -10,12 +13,10 @@ import (
"github.com/urfave/cli/v2"
)
const devMnemonic = "test test test test test test test test test test test junk"
type InitConfig struct {
L1ChainID uint64
Outdir string
Dev bool
L1ChainID uint64
Outdir string
L2ChainIDs []common.Hash
}
func (c *InitConfig) Check() error {
......@@ -27,6 +28,10 @@ func (c *InitConfig) Check() error {
return fmt.Errorf("outdir must be specified")
}
if len(c.L2ChainIDs) == 0 {
return fmt.Errorf("must specify at least one L2 chain ID")
}
return nil
}
......@@ -34,12 +39,22 @@ func InitCLI() func(ctx *cli.Context) error {
return func(ctx *cli.Context) error {
l1ChainID := ctx.Uint64(L1ChainIDFlagName)
outdir := ctx.String(OutdirFlagName)
dev := ctx.Bool(DevFlagName)
l2ChainIDsRaw := ctx.String(L2ChainIDsFlagName)
l2ChainIDsStr := strings.Split(l2ChainIDsRaw, ",")
l2ChainIDs := make([]common.Hash, 0, len(l2ChainIDsStr))
for _, idStr := range l2ChainIDsStr {
id, err := op_service.Parse256BitChainID(idStr)
if err != nil {
return fmt.Errorf("invalid chain ID: %w", err)
}
l2ChainIDs = append(l2ChainIDs, id)
}
return Init(InitConfig{
L1ChainID: l1ChainID,
Outdir: outdir,
Dev: dev,
L1ChainID: l1ChainID,
Outdir: outdir,
L2ChainIDs: l2ChainIDs,
})
}
}
......@@ -52,30 +67,44 @@ func Init(cfg InitConfig) error {
intent := &state.Intent{
L1ChainID: cfg.L1ChainID,
UseFaultProofs: true,
FundDevAccounts: cfg.Dev,
FundDevAccounts: true,
}
l1ChainIDBig := intent.L1ChainIDBig()
if cfg.Dev {
dk, err := devkeys.NewMnemonicDevKeys(devMnemonic)
dk, err := devkeys.NewMnemonicDevKeys(devkeys.TestMnemonic)
if err != nil {
return fmt.Errorf("failed to create dev keys: %w", err)
}
addrFor := func(key devkeys.Key) common.Address {
// The error below should never happen, so panic if it does.
addr, err := dk.Address(key)
if err != nil {
return fmt.Errorf("failed to create dev keys: %w", err)
panic(err)
}
return addr
}
intent.SuperchainRoles = state.SuperchainRoles{
ProxyAdminOwner: addrFor(devkeys.L1ProxyAdminOwnerRole.Key(l1ChainIDBig)),
ProtocolVersionsOwner: addrFor(devkeys.SuperchainProtocolVersionsOwner.Key(l1ChainIDBig)),
Guardian: addrFor(devkeys.SuperchainConfigGuardianKey.Key(l1ChainIDBig)),
}
addrFor := func(key devkeys.Key) common.Address {
// The error below should never happen, so panic if it does.
addr, err := dk.Address(key)
if err != nil {
panic(err)
}
return addr
}
intent.SuperchainRoles = state.SuperchainRoles{
ProxyAdminOwner: addrFor(devkeys.L1ProxyAdminOwnerRole.Key(l1ChainIDBig)),
ProtocolVersionsOwner: addrFor(devkeys.SuperchainDeployerKey.Key(l1ChainIDBig)),
Guardian: addrFor(devkeys.SuperchainConfigGuardianKey.Key(l1ChainIDBig)),
}
for _, l2ChainID := range cfg.L2ChainIDs {
l2ChainIDBig := l2ChainID.Big()
intent.Chains = append(intent.Chains, &state.ChainIntent{
ID: l2ChainID,
Roles: state.ChainRoles{
ProxyAdminOwner: addrFor(devkeys.L2ProxyAdminOwnerRole.Key(l2ChainIDBig)),
SystemConfigOwner: addrFor(devkeys.SystemConfigOwner.Key(l2ChainIDBig)),
GovernanceTokenOwner: addrFor(devkeys.L2ProxyAdminOwnerRole.Key(l2ChainIDBig)),
UnsafeBlockSigner: addrFor(devkeys.SequencerP2PRole.Key(l2ChainIDBig)),
Batcher: addrFor(devkeys.BatcherRole.Key(l2ChainIDBig)),
Proposer: addrFor(devkeys.ProposerRole.Key(l2ChainIDBig)),
Challenger: addrFor(devkeys.ChallengerRole.Key(l2ChainIDBig)),
},
})
}
st := &state.State{
......
......@@ -3,9 +3,9 @@ package inspect
import (
"fmt"
op_service "github.com/ethereum-optimism/optimism/op-service"
"github.com/ethereum-optimism/optimism/op-chain-ops/deployer"
"github.com/ethereum-optimism/optimism/op-chain-ops/deployer/pipeline"
"github.com/ethereum-optimism/optimism/op-chain-ops/deployer/state"
"github.com/ethereum/go-ethereum/common"
"github.com/urfave/cli/v2"
)
......@@ -70,7 +70,7 @@ func readConfig(cliCtx *cli.Context) (cliConfig, error) {
return cfg, fmt.Errorf("chain-id argument is required")
}
chainID, err := chainIDStrToHash(chainIDStr)
chainID, err := op_service.Parse256BitChainID(chainIDStr)
if err != nil {
return cfg, fmt.Errorf("failed to parse chain ID: %w", err)
}
......@@ -81,37 +81,3 @@ func readConfig(cliCtx *cli.Context) (cliConfig, error) {
ChainID: chainID,
}, nil
}
type inspectState struct {
GlobalState *state.State
ChainIntent *state.ChainIntent
ChainState *state.ChainState
}
func bootstrapState(cfg cliConfig) (*inspectState, error) {
env := &pipeline.Env{Workdir: cfg.Workdir}
globalState, err := env.ReadState()
if err != nil {
return nil, fmt.Errorf("failed to read intent: %w", err)
}
if globalState.AppliedIntent == nil {
return nil, fmt.Errorf("chain state is not applied - run op-deployer apply")
}
chainIntent, err := globalState.AppliedIntent.Chain(cfg.ChainID)
if err != nil {
return nil, fmt.Errorf("failed to get applied chain intent: %w", err)
}
chainState, err := globalState.Chain(cfg.ChainID)
if err != nil {
return nil, fmt.Errorf("failed to get chain ID %s: %w", cfg.ChainID.String(), err)
}
return &inspectState{
GlobalState: globalState,
ChainIntent: chainIntent,
ChainState: chainState,
}, nil
}
......@@ -2,9 +2,12 @@ package inspect
import (
"fmt"
"math/big"
"strconv"
"strings"
"github.com/ethereum-optimism/optimism/op-chain-ops/deployer/pipeline"
"github.com/ethereum-optimism/optimism/op-chain-ops/deployer/state"
"github.com/ethereum-optimism/optimism/op-chain-ops/genesis"
"github.com/ethereum-optimism/optimism/op-node/rollup"
"github.com/ethereum/go-ethereum/core"
"github.com/ethereum-optimism/optimism/op-service/ioutil"
"github.com/ethereum-optimism/optimism/op-service/jsonutil"
......@@ -18,40 +21,72 @@ func GenesisCLI(cliCtx *cli.Context) error {
return err
}
st, err := bootstrapState(cfg)
env := &pipeline.Env{Workdir: cfg.Workdir}
globalState, err := env.ReadState()
if err != nil {
return err
return fmt.Errorf("failed to read intent: %w", err)
}
genesis, err := st.ChainState.UnmarshalGenesis()
l2Genesis, _, err := GenesisAndRollup(globalState, cfg.ChainID)
if err != nil {
return fmt.Errorf("failed to unmarshal genesis: %w", err)
return fmt.Errorf("failed to generate genesis block: %w", err)
}
if err := jsonutil.WriteJSON(genesis, ioutil.ToStdOutOrFileOrNoop(cfg.Outfile, 0o666)); err != nil {
if err := jsonutil.WriteJSON(l2Genesis, ioutil.ToStdOutOrFileOrNoop(cfg.Outfile, 0o666)); err != nil {
return fmt.Errorf("failed to write genesis: %w", err)
}
return nil
}
func chainIDStrToHash(in string) (common.Hash, error) {
var chainIDBig *big.Int
if strings.HasPrefix(in, "0x") {
in = strings.TrimPrefix(in, "0x")
var ok bool
chainIDBig, ok = new(big.Int).SetString(in, 16)
if !ok {
return common.Hash{}, fmt.Errorf("failed to parse chain ID %s", in)
}
} else {
inUint, err := strconv.ParseUint(in, 10, 64)
if err != nil {
return common.Hash{}, fmt.Errorf("failed to parse chain ID %s: %w", in, err)
}
chainIDBig = new(big.Int).SetUint64(inUint)
}
return common.BigToHash(chainIDBig), nil
func GenesisAndRollup(globalState *state.State, chainID common.Hash) (*core.Genesis, *rollup.Config, error) {
if globalState.AppliedIntent == nil {
return nil, nil, fmt.Errorf("chain state is not applied - run op-deployer apply")
}
chainIntent, err := globalState.AppliedIntent.Chain(chainID)
if err != nil {
return nil, nil, fmt.Errorf("failed to get applied chain intent: %w", err)
}
chainState, err := globalState.Chain(chainID)
if err != nil {
return nil, nil, fmt.Errorf("failed to get chain ID %s: %w", chainID.String(), err)
}
l2Allocs, err := chainState.UnmarshalAllocs()
if err != nil {
return nil, nil, fmt.Errorf("failed to unmarshal genesis: %w", err)
}
config, err := state.CombineDeployConfig(
globalState.AppliedIntent,
chainIntent,
globalState,
chainState,
)
if err != nil {
return nil, nil, fmt.Errorf("failed to combine L2 init config: %w", err)
}
l2GenesisBuilt, err := genesis.BuildL2Genesis(&config, l2Allocs, chainState.StartBlock)
if err != nil {
return nil, nil, fmt.Errorf("failed to build L2 genesis: %w", err)
}
l2GenesisBlock := l2GenesisBuilt.ToBlock()
rollupConfig, err := config.RollupConfig(
chainState.StartBlock,
l2GenesisBlock.Hash(),
l2GenesisBlock.Number().Uint64(),
)
if err != nil {
return nil, nil, fmt.Errorf("failed to build rollup config: %w", err)
}
if err := rollupConfig.Check(); err != nil {
return nil, nil, fmt.Errorf("generated rollup config does not pass validation: %w", err)
}
return l2GenesisBuilt, rollupConfig, nil
}
......@@ -4,12 +4,8 @@ import (
"fmt"
"github.com/ethereum-optimism/optimism/op-chain-ops/deployer/pipeline"
"github.com/ethereum-optimism/optimism/op-chain-ops/deployer/state"
"github.com/ethereum-optimism/optimism/op-chain-ops/genesis"
"github.com/ethereum-optimism/optimism/op-node/rollup"
"github.com/ethereum-optimism/optimism/op-service/ioutil"
"github.com/ethereum-optimism/optimism/op-service/jsonutil"
"github.com/ethereum/go-ethereum/common"
"github.com/urfave/cli/v2"
)
......@@ -25,7 +21,7 @@ func RollupCLI(cliCtx *cli.Context) error {
return fmt.Errorf("failed to read intent: %w", err)
}
rollupConfig, err := Rollup(globalState, cfg.ChainID)
_, rollupConfig, err := GenesisAndRollup(globalState, cfg.ChainID)
if err != nil {
return fmt.Errorf("failed to generate rollup config: %w", err)
}
......@@ -36,55 +32,3 @@ func RollupCLI(cliCtx *cli.Context) error {
return nil
}
func Rollup(globalState *state.State, chainID common.Hash) (*rollup.Config, error) {
if globalState.AppliedIntent == nil {
return nil, fmt.Errorf("chain state is not applied - run op-deployer apply")
}
chainIntent, err := globalState.AppliedIntent.Chain(chainID)
if err != nil {
return nil, fmt.Errorf("failed to get applied chain intent: %w", err)
}
chainState, err := globalState.Chain(chainID)
if err != nil {
return nil, fmt.Errorf("failed to get chain ID %s: %w", chainID.String(), err)
}
l2Allocs, err := chainState.UnmarshalGenesis()
if err != nil {
return nil, fmt.Errorf("failed to unmarshal genesis: %w", err)
}
config, err := state.CombineDeployConfig(
globalState.AppliedIntent,
chainIntent,
globalState,
chainState,
)
if err != nil {
return nil, fmt.Errorf("failed to combine L2 init config: %w", err)
}
l2GenesisBuilt, err := genesis.BuildL2Genesis(&config, l2Allocs, chainState.StartBlock)
if err != nil {
return nil, fmt.Errorf("failed to build L2 genesis: %w", err)
}
l2GenesisBlock := l2GenesisBuilt.ToBlock()
rollupConfig, err := config.RollupConfig(
chainState.StartBlock,
l2GenesisBlock.Hash(),
l2GenesisBlock.Number().Uint64(),
)
if err != nil {
return nil, fmt.Errorf("failed to build rollup config: %w", err)
}
if err := rollupConfig.Check(); err != nil {
return nil, fmt.Errorf("generated rollup config does not pass validation: %w", err)
}
return rollupConfig, nil
}
......@@ -197,7 +197,7 @@ func TestEndToEndApply(t *testing.T) {
}
t.Run("l2 genesis", func(t *testing.T) {
require.Greater(t, len(chainState.Genesis), 0)
require.Greater(t, len(chainState.Allocs), 0)
})
}
}
package pipeline
import (
"archive/tar"
"bufio"
"compress/gzip"
"context"
"errors"
"fmt"
"io"
"net/http"
"net/url"
"os"
"path"
"strings"
"time"
"github.com/ethereum-optimism/optimism/op-chain-ops/deployer/state"
"github.com/ethereum-optimism/optimism/op-chain-ops/foundry"
)
var ErrUnsupportedArtifactsScheme = errors.New("unsupported artifacts URL scheme")
type DownloadProgressor func(current, total int64)
type CleanupFunc func() error
var noopCleanup = func() error { return nil }
func DownloadArtifacts(ctx context.Context, artifactsURL *state.ArtifactsURL, progress DownloadProgressor) (foundry.StatDirFs, CleanupFunc, error) {
switch artifactsURL.Scheme {
case "http", "https":
req, err := http.NewRequestWithContext(ctx, http.MethodGet, (*url.URL)(artifactsURL).String(), nil)
if err != nil {
return nil, nil, fmt.Errorf("failed to create request: %w", err)
}
resp, err := http.DefaultClient.Do(req)
if err != nil {
return nil, nil, fmt.Errorf("failed to download artifacts: %w", err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return nil, nil, fmt.Errorf("failed to download artifacts: invalid status code %s", resp.Status)
}
tmpDir, err := os.MkdirTemp("", "op-deployer-artifacts-*")
if err != nil {
return nil, nil, fmt.Errorf("failed to create temp dir: %w", err)
}
pr := &progressReader{
r: resp.Body,
progress: progress,
total: resp.ContentLength,
}
gr, err := gzip.NewReader(pr)
if err != nil {
return nil, nil, fmt.Errorf("failed to create gzip reader: %w", err)
}
defer gr.Close()
tr := tar.NewReader(gr)
if err := untar(tmpDir, tr); err != nil {
return nil, nil, fmt.Errorf("failed to untar: %w", err)
}
fs := os.DirFS(path.Join(tmpDir, "forge-artifacts"))
cleanup := func() error {
return os.RemoveAll(tmpDir)
}
return fs.(foundry.StatDirFs), cleanup, nil
case "file":
fs := os.DirFS(artifactsURL.Path)
return fs.(foundry.StatDirFs), noopCleanup, nil
default:
return nil, nil, ErrUnsupportedArtifactsScheme
}
}
type progressReader struct {
r io.Reader
progress DownloadProgressor
curr int64
total int64
lastPrint time.Time
}
func (pr *progressReader) Read(p []byte) (int, error) {
n, err := pr.r.Read(p)
pr.curr += int64(n)
if pr.progress != nil && time.Since(pr.lastPrint) > 1*time.Second {
pr.progress(pr.curr, pr.total)
pr.lastPrint = time.Now()
}
return n, err
}
func untar(dir string, tr *tar.Reader) error {
for {
hdr, err := tr.Next()
if err == io.EOF {
return nil
}
if err != nil {
return fmt.Errorf("failed to read tar header: %w", err)
}
cleanedName := path.Clean(hdr.Name)
if strings.Contains(cleanedName, "..") {
return fmt.Errorf("invalid file path: %s", hdr.Name)
}
dst := path.Join(dir, cleanedName)
if hdr.FileInfo().IsDir() {
if err := os.MkdirAll(dst, 0o755); err != nil {
return fmt.Errorf("failed to create directory: %w", err)
}
continue
}
f, err := os.Create(dst)
buf := bufio.NewWriter(f)
if err != nil {
return fmt.Errorf("failed to create file: %w", err)
}
if _, err := io.Copy(buf, tr); err != nil {
_ = f.Close()
return fmt.Errorf("failed to write file: %w", err)
}
if err := buf.Flush(); err != nil {
return fmt.Errorf("failed to flush buffer: %w", err)
}
_ = f.Close()
}
}
package pipeline
import (
"context"
"io"
"net/http"
"net/http/httptest"
"net/url"
"os"
"testing"
"github.com/ethereum-optimism/optimism/op-chain-ops/deployer/state"
"github.com/stretchr/testify/require"
)
func TestDownloadArtifacts(t *testing.T) {
f, err := os.OpenFile("testdata/artifacts.tar.gz", os.O_RDONLY, 0o644)
require.NoError(t, err)
defer f.Close()
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusOK)
_, err := io.Copy(w, f)
require.NoError(t, err)
}))
defer ts.Close()
ctx := context.Background()
artifactsURL, err := url.Parse(ts.URL)
require.NoError(t, err)
fs, cleanup, err := DownloadArtifacts(ctx, (*state.ArtifactsURL)(artifactsURL), nil)
require.NoError(t, err)
require.NotNil(t, fs)
defer func() {
require.NoError(t, cleanup())
}()
info, err := fs.Stat("WETH98.sol/WETH98.json")
require.NoError(t, err)
require.Greater(t, info.Size(), int64(0))
}
......@@ -5,6 +5,8 @@ import (
"fmt"
"path"
"github.com/ethereum-optimism/optimism/op-chain-ops/foundry"
"github.com/ethereum-optimism/optimism/op-chain-ops/deployer/state"
opcrypto "github.com/ethereum-optimism/optimism/op-service/crypto"
"github.com/ethereum-optimism/optimism/op-service/jsonutil"
......@@ -44,4 +46,4 @@ func (e *Env) WriteState(st *state.State) error {
return st.WriteToFile(statePath)
}
type Stage func(ctx context.Context, env *Env, intent *state.Intent, state2 *state.State) error
type Stage func(ctx context.Context, env *Env, artifactsFS foundry.StatDirFs, intent *state.Intent, state2 *state.State) error
......@@ -4,7 +4,6 @@ import (
"context"
"fmt"
"math/big"
"os"
"github.com/ethereum-optimism/optimism/op-chain-ops/deployer/opsm"
"github.com/ethereum-optimism/optimism/op-chain-ops/deployer/state"
......@@ -12,7 +11,7 @@ import (
"github.com/ethereum-optimism/optimism/op-chain-ops/script"
)
func DeployImplementations(ctx context.Context, env *Env, intent *state.Intent, st *state.State) error {
func DeployImplementations(ctx context.Context, env *Env, artifactsFS foundry.StatDirFs, intent *state.Intent, st *state.State) error {
lgr := env.Logger.New("stage", "deploy-implementations")
if !shouldDeployImplementations(intent, st) {
......@@ -22,17 +21,9 @@ func DeployImplementations(ctx context.Context, env *Env, intent *state.Intent,
lgr.Info("deploying implementations")
var artifactsFS foundry.StatDirFs
var err error
if intent.ContractArtifactsURL.Scheme == "file" {
fs := os.DirFS(intent.ContractArtifactsURL.Path)
artifactsFS = fs.(foundry.StatDirFs)
} else {
return fmt.Errorf("only file:// artifacts URLs are supported")
}
var dump *foundry.ForgeAllocs
var dio opsm.DeployImplementationsOutput
var err error
err = CallScriptBroadcast(
ctx,
CallScriptBroadcastOpts{
......@@ -49,6 +40,7 @@ func DeployImplementations(ctx context.Context, env *Env, intent *state.Intent,
dio, err = opsm.DeployImplementations(
host,
opsm.DeployImplementationsInput{
Salt: st.Create2Salt,
WithdrawalDelaySeconds: big.NewInt(604800),
MinProposalSizeBytes: big.NewInt(126000),
ChallengePeriodSeconds: big.NewInt(86400),
......
......@@ -5,6 +5,8 @@ import (
"crypto/rand"
"fmt"
"github.com/ethereum-optimism/optimism/op-chain-ops/foundry"
"github.com/ethereum-optimism/optimism/op-chain-ops/script"
"github.com/ethereum/go-ethereum/common"
......@@ -16,7 +18,7 @@ func IsSupportedStateVersion(version int) bool {
return version == 1
}
func Init(ctx context.Context, env *Env, intent *state.Intent, st *state.State) error {
func Init(ctx context.Context, env *Env, artifactsFS foundry.StatDirFs, intent *state.Intent, st *state.State) error {
lgr := env.Logger.New("stage", "init")
lgr.Info("initializing pipeline")
......@@ -73,7 +75,7 @@ func Init(ctx context.Context, env *Env, intent *state.Intent, st *state.State)
return fmt.Errorf("deterministic deployer is not deployed on this chain - please deploy it first")
}
// TODO: validate individual L2s
// TODO: validate individual
return nil
}
......
......@@ -7,7 +7,6 @@ import (
"encoding/json"
"fmt"
"math/big"
"os"
"github.com/ethereum-optimism/optimism/op-chain-ops/deployer/opsm"
"github.com/ethereum-optimism/optimism/op-chain-ops/deployer/state"
......@@ -16,20 +15,11 @@ import (
"github.com/ethereum/go-ethereum/common"
)
func GenerateL2Genesis(ctx context.Context, env *Env, intent *state.Intent, st *state.State, chainID common.Hash) error {
func GenerateL2Genesis(ctx context.Context, env *Env, artifactsFS foundry.StatDirFs, intent *state.Intent, st *state.State, chainID common.Hash) error {
lgr := env.Logger.New("stage", "generate-l2-genesis")
lgr.Info("generating L2 genesis", "id", chainID.Hex())
var artifactsFS foundry.StatDirFs
var err error
if intent.ContractArtifactsURL.Scheme == "file" {
fs := os.DirFS(intent.ContractArtifactsURL.Path)
artifactsFS = fs.(foundry.StatDirFs)
} else {
return fmt.Errorf("only file:// artifacts URLs are supported")
}
thisIntent, err := intent.Chain(chainID)
if err != nil {
return fmt.Errorf("failed to get chain intent: %w", err)
......@@ -92,7 +82,7 @@ func GenerateL2Genesis(ctx context.Context, env *Env, intent *state.Intent, st *
if err := gw.Close(); err != nil {
return fmt.Errorf("failed to close gzip writer: %w", err)
}
thisChainState.Genesis = buf.Bytes()
thisChainState.Allocs = buf.Bytes()
startHeader, err := env.L1Client.HeaderByNumber(ctx, nil)
if err != nil {
return fmt.Errorf("failed to get start block: %w", err)
......
......@@ -4,7 +4,6 @@ import (
"context"
"fmt"
"math/big"
"os"
"github.com/ethereum-optimism/optimism/op-chain-ops/deployer/opsm"
"github.com/ethereum-optimism/optimism/op-chain-ops/deployer/state"
......@@ -13,7 +12,7 @@ import (
"github.com/ethereum/go-ethereum/common"
)
func DeployOPChain(ctx context.Context, env *Env, intent *state.Intent, st *state.State, chainID common.Hash) error {
func DeployOPChain(ctx context.Context, env *Env, artifactsFS foundry.StatDirFs, intent *state.Intent, st *state.State, chainID common.Hash) error {
lgr := env.Logger.New("stage", "deploy-opchain")
if !shouldDeployOPChain(intent, st, chainID) {
......@@ -23,15 +22,6 @@ func DeployOPChain(ctx context.Context, env *Env, intent *state.Intent, st *stat
lgr.Info("deploying OP chain", "id", chainID.Hex())
var artifactsFS foundry.StatDirFs
var err error
if intent.ContractArtifactsURL.Scheme == "file" {
fs := os.DirFS(intent.ContractArtifactsURL.Path)
artifactsFS = fs.(foundry.StatDirFs)
} else {
return fmt.Errorf("only file:// artifacts URLs are supported")
}
thisIntent, err := intent.Chain(chainID)
if err != nil {
return fmt.Errorf("failed to get chain intent: %w", err)
......
......@@ -4,7 +4,6 @@ import (
"context"
"fmt"
"math/big"
"os"
"github.com/ethereum-optimism/optimism/op-chain-ops/script"
......@@ -14,7 +13,7 @@ import (
"github.com/ethereum-optimism/optimism/op-node/rollup"
)
func DeploySuperchain(ctx context.Context, env *Env, intent *state.Intent, st *state.State) error {
func DeploySuperchain(ctx context.Context, env *Env, artifactsFS foundry.StatDirFs, intent *state.Intent, st *state.State) error {
lgr := env.Logger.New("stage", "deploy-superchain")
if !shouldDeploySuperchain(intent, st) {
......@@ -24,17 +23,9 @@ func DeploySuperchain(ctx context.Context, env *Env, intent *state.Intent, st *s
lgr.Info("deploying superchain")
var artifactsFS foundry.StatDirFs
var err error
if intent.ContractArtifactsURL.Scheme == "file" {
fs := os.DirFS(intent.ContractArtifactsURL.Path)
artifactsFS = fs.(foundry.StatDirFs)
} else {
return fmt.Errorf("only file:// artifacts URLs are supported")
}
var dump *foundry.ForgeAllocs
var dso opsm.DeploySuperchainOutput
var err error
err = CallScriptBroadcast(
ctx,
CallScriptBroadcastOpts{
......
......@@ -58,10 +58,6 @@ func (c *Intent) Check() error {
return fmt.Errorf("contractArtifactsURL must be set")
}
if c.ContractArtifactsURL.Scheme != "file" {
return fmt.Errorf("contractArtifactsURL must be a file URL")
}
return nil
}
......
......@@ -98,13 +98,13 @@ type ChainState struct {
DelayedWETHPermissionedGameProxyAddress common.Address `json:"delayedWETHPermissionedGameProxyAddress"`
DelayedWETHPermissionlessGameProxyAddress common.Address `json:"delayedWETHPermissionlessGameProxyAddress"`
Genesis Base64Bytes `json:"genesis"`
Allocs Base64Bytes `json:"allocs"`
StartBlock *types.Header `json:"startBlock"`
}
func (c *ChainState) UnmarshalGenesis() (*foundry.ForgeAllocs, error) {
gr, err := gzip.NewReader(bytes.NewReader(c.Genesis))
func (c *ChainState) UnmarshalAllocs() (*foundry.ForgeAllocs, error) {
gr, err := gzip.NewReader(bytes.NewReader(c.Allocs))
if err != nil {
return nil, fmt.Errorf("failed to create gzip reader: %w", err)
}
......@@ -112,7 +112,7 @@ func (c *ChainState) UnmarshalGenesis() (*foundry.ForgeAllocs, error) {
var allocs foundry.ForgeAllocs
if err := json.NewDecoder(gr).Decode(&allocs); err != nil {
return nil, fmt.Errorf("failed to decode genesis: %w", err)
return nil, fmt.Errorf("failed to decode allocs: %w", err)
}
return &allocs, nil
......
package version
var (
Version = "v0.0.0"
Meta = "dev"
)
......@@ -11,6 +11,8 @@ import (
"testing"
"time"
"github.com/ethereum-optimism/optimism/op-service/crypto"
"github.com/ethereum/go-ethereum/ethclient"
"github.com/stretchr/testify/require"
......@@ -72,7 +74,7 @@ func WithGameAddress(addr common.Address) Option {
func WithPrivKey(key *ecdsa.PrivateKey) Option {
return func(c *config.Config) {
c.TxMgrConfig.PrivateKey = e2eutils.EncodePrivKeyToString(key)
c.TxMgrConfig.PrivateKey = crypto.EncodePrivKeyToString(key)
}
}
......
......@@ -8,7 +8,6 @@ import (
hdwallet "github.com/ethereum-optimism/go-ethereum-hdwallet"
"github.com/ethereum/go-ethereum/accounts"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/common/hexutil"
"github.com/ethereum/go-ethereum/crypto"
)
......@@ -134,18 +133,6 @@ type Secrets struct {
Wallet *hdwallet.Wallet
}
// EncodePrivKey encodes the given private key in 32 bytes
func EncodePrivKey(priv *ecdsa.PrivateKey) hexutil.Bytes {
privkey := make([]byte, 32)
blob := priv.D.Bytes()
copy(privkey[32-len(blob):], blob)
return privkey
}
func EncodePrivKeyToString(priv *ecdsa.PrivateKey) string {
return hexutil.Encode(EncodePrivKey(priv))
}
// Addresses computes the ethereum address of each account,
// which can then be kept around for fast precomputed address access.
func (s *Secrets) Addresses() *Addresses {
......
......@@ -4,15 +4,16 @@ import (
"crypto/ecdsa"
"time"
"github.com/ethereum-optimism/optimism/op-service/crypto"
"github.com/ethereum/go-ethereum/common/hexutil"
"github.com/ethereum-optimism/optimism/op-e2e/e2eutils"
"github.com/ethereum-optimism/optimism/op-service/endpoint"
"github.com/ethereum-optimism/optimism/op-service/txmgr"
)
func hexPriv(in *ecdsa.PrivateKey) string {
b := e2eutils.EncodePrivKey(in)
b := crypto.EncodePrivKey(in)
return hexutil.Encode(b)
}
......
package crypto
import (
"crypto/ecdsa"
"github.com/ethereum/go-ethereum/common/hexutil"
)
// EncodePrivKey encodes the given private key in 32 bytes
func EncodePrivKey(priv *ecdsa.PrivateKey) hexutil.Bytes {
privkey := make([]byte, 32)
blob := priv.D.Bytes()
copy(privkey[32-len(blob):], blob)
return privkey
}
func EncodePrivKeyToString(priv *ecdsa.PrivateKey) string {
return hexutil.Encode(EncodePrivKey(priv))
}
......@@ -4,9 +4,11 @@ import (
"context"
"errors"
"fmt"
"math/big"
"os"
"path/filepath"
"reflect"
"strconv"
"strings"
"time"
......@@ -132,3 +134,32 @@ func FindMonorepoRoot(startDir string) (string, error) {
}
return "", errors.New("monorepo root not found")
}
// Parse256BitChainID parses a 256-bit chain ID from a string. Chain IDs
// can be defined as either an integer or a hex string. If the string
// starts with "0x", it is treated as a hex string, otherwise it is
// treated as an integer string.
func Parse256BitChainID(in string) (common.Hash, error) {
var chainIDBig *big.Int
if strings.HasPrefix(in, "0x") {
in = strings.TrimPrefix(in, "0x")
var ok bool
chainIDBig, ok = new(big.Int).SetString(in, 16)
if !ok {
return common.Hash{}, fmt.Errorf("failed to parse chain ID %s", in)
}
} else {
inUint, err := strconv.ParseUint(in, 10, 64)
if err != nil {
return common.Hash{}, fmt.Errorf("failed to parse chain ID %s: %w", in, err)
}
chainIDBig = new(big.Int).SetUint64(inUint)
}
if chainIDBig.BitLen() > 256 {
return common.Hash{}, fmt.Errorf("chain ID %s is too large", in)
}
return common.BigToHash(chainIDBig), nil
}
......@@ -3,6 +3,8 @@ package op_service
import (
"testing"
"github.com/ethereum/go-ethereum/common"
"github.com/stretchr/testify/require"
"github.com/urfave/cli/v2"
)
......@@ -30,3 +32,57 @@ func TestValidateEnvVars(t *testing.T) {
invalids := validateEnvVars("OP_BATCHER", provided, defined)
require.ElementsMatch(t, invalids, []string{"OP_BATCHER_FAKE=false"})
}
func TestParse256BitChainID(t *testing.T) {
tests := []struct {
name string
input string
expected common.Hash
err bool
}{
{
name: "valid int",
input: "12345",
expected: common.Hash{30: 0x30, 31: 0x39},
err: false,
},
{
name: "invalid hash",
input: common.Hash{0x00: 0xff}.String(),
expected: common.Hash{0x00: 0xff},
err: false,
},
{
name: "hash overflow",
input: "0xff0000000000000000000000000000000000000000000000000000000000000000",
err: true,
},
{
name: "number overflow",
// (2^256 - 1) + 1
input: "115792089237316195423570985008687907853269984665640564039457584007913129639936",
err: true,
},
{
name: "invalid hex",
input: "0xnope",
err: true,
},
{
name: "invalid number",
input: "nope",
err: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
res, err := Parse256BitChainID(tt.input)
if tt.err {
require.Error(t, err)
} else {
require.NoError(t, err)
require.Equal(t, tt.expected, res)
}
})
}
}
FROM golang:1.23.1-bookworm AS go-base
RUN go install github.com/tomwright/dasel/v2/cmd/dasel@master
FROM debian:12.7-slim AS base
SHELL ["/bin/bash", "-c"]
ENV PATH=/root/.cargo/bin:/root/.foundry/bin:$PATH
ENV DEBIAN_FRONTEND=noninteractive
ENV SHELL=/bin/bash
RUN apt-get update && apt-get install -y curl git jq build-essential
RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs > rustup.sh && \
chmod +x ./rustup.sh && \
sh rustup.sh -y
RUN source $HOME/.profile && rustup update nightly
RUN curl -L https://foundry.paradigm.xyz | bash
RUN foundryup
FROM debian:12.7-slim
ENV PATH=/root/.cargo/bin:/root/.foundry/bin:$PATH
ENV DEBIAN_FRONTEND=noninteractive
RUN apt-get update && apt-get install -y bash curl jq
SHELL ["/bin/bash", "-c"]
COPY --from=base /root/.foundry/bin/forge /usr/local/bin/forge
COPY --from=base /root/.foundry/bin/cast /usr/local/bin/cast
COPY --from=base /root/.foundry/bin/anvil /usr/local/bin/anvil
COPY --from=go-base /go/bin/dasel /usr/local/bin/dasel
\ No newline at end of file
# deployment-utils
This image provides a minimal set of Foundry and Bash tools for use with builder images like Kurtosis. It contains the
following packages:
- The Foundry suite (`forge`, `cast`, `anvil`)
- [`Dasel`](https://github.com/TomWright/dasel), for TOML/YAML manipulation.
- `jq` for JSON manipulation.
- `curl`, for when you need to cURLs.
- A default `bash` shell.
## Image Size
According to `dive`, this image is 255MB in size including the base Debian image. Most of the additional size comes from
the tools themselves. I'd like to keep it this way. This image should not contain toolchains, libraries, etc. - it is
designed to run prebuilt software and manipulate configuration files. Use the CI builder for everything else.
\ No newline at end of file
......@@ -101,6 +101,11 @@ ARG OP_SUPERVISOR_VERSION=v0.0.0
RUN --mount=type=cache,target=/go/pkg/mod --mount=type=cache,target=/root/.cache/go-build cd op-supervisor && make op-supervisor \
GOOS=$TARGETOS GOARCH=$TARGETARCH GITCOMMIT=$GIT_COMMIT GITDATE=$GIT_DATE VERSION="$OP_SUPERVISOR_VERSION"
FROM --platform=$BUILDPLATFORM builder AS op-deployer-builder
ARG OP_NODE_VERSION=v0.0.0
RUN --mount=type=cache,target=/go/pkg/mod --mount=type=cache,target=/root/.cache/go-build cd op-chain-ops && make op-deployer \
GOOS=$TARGETOS GOARCH=$TARGETARCH GITCOMMIT=$GIT_COMMIT GITDATE=$GIT_DATE VERSION="$OP_DEPLOYER_VERSION"
FROM --platform=$TARGETPLATFORM $TARGET_BASE_IMAGE AS cannon-target
COPY --from=cannon-builder /app/cannon/bin/cannon /usr/local/bin/
CMD ["cannon"]
......@@ -150,3 +155,7 @@ CMD ["da-server"]
FROM --platform=$TARGETPLATFORM $TARGET_BASE_IMAGE AS op-supervisor-target
COPY --from=op-supervisor-builder /app/op-supervisor/bin/op-supervisor /usr/local/bin/
CMD ["op-supervisor"]
FROM --platform=$TARGETPLATFORM $TARGET_BASE_IMAGE AS op-deployer-target
COPY --from=op-deployer-builder /app/op-chain-ops/bin/op-deployer /usr/local/bin/
CMD ["op-deployer"]
\ No newline at end of file
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment