ops: Add bedrock-devnet script (#3734)

* ops: Add bedrock-devnet script

Adds a bedrock-devnet script that deploys a local network using the Hardhat deploy scripts. I opted to write this in Python rather than Bash because we're starting to add complexity to our devnet setup, and I find Python much easier and safer to maintain.

Over time, I'll migrate the other devnet-up script to use this Python script as well.

To invoke this script, run `make devnet-up-deploy`. It's included in CI already as a separate testing job.

* review updates

* Revert "review updates"

This reverts commit d01ba16b971294b44c5bfbc3124354be30477daf.

.circleci/config.yml

@@ -508,6 +508,11 @@ jobs:
     machine:
       image: ubuntu-2004:2022.07.1
       docker_layer_caching: true
+    parameters:
+      deploy:
+        description: Deploy contracts
+        default: false
+        type: boolean
     environment:
       DOCKER_BUILDKIT: 1
     steps:
@@ -539,22 +544,48 @@ jobs:
           command: |
             yarn install
             yarn build
-      - run:
-          name: Bring up the stack
-          command: |
-            make devnet-up
-      - run:
-          name: Deposit ERC20 through the bridge
-          command: timeout 5m npx hardhat deposit-erc20 --network devnetL1
-          working_directory: packages/sdk
-      - run:
-          name: Deposit ETH through the bridge
-          command: timeout 5m npx hardhat deposit-eth --network devnetL1
-          working_directory: packages/sdk
-      - run:
-          name: Check the status
-          command: npx hardhat check-op-node
-          working_directory: packages/contracts-bedrock
+      - when:
+          condition:
+            and:
+              - equal: [ true, <<parameters.deploy>> ]
+          steps:
+            - run:
+                name: Bring up the stack
+                command: |
+                  make devnet-up-deploy
+            - run:
+                name: Deposit ERC20 through the bridge
+                command: timeout 5m npx hardhat deposit-erc20 --network devnetL1 --l1-contracts-json-path ../../.devnet/sdk-addresses.json
+                working_directory: packages/sdk
+            - run:
+                name: Deposit ETH through the bridge
+                command: timeout 5m npx hardhat deposit-eth --network devnetL1 --l1-contracts-json-path ../../.devnet/sdk-addresses.json
+                working_directory: packages/sdk
+            - run:
+                name: Check the status
+                command: npx hardhat check-op-node
+                working_directory: packages/contracts-bedrock
+      - when:
+          condition:
+            and:
+              - equal: [ false, <<parameters.deploy>> ]
+          steps:
+            - run:
+                name: Bring up the stack
+                command: |
+                  make devnet-up
+            - run:
+                name: Deposit ERC20 through the bridge
+                command: timeout 5m npx hardhat deposit-erc20 --network devnetL1
+                working_directory: packages/sdk
+            - run:
+                name: Deposit ETH through the bridge
+                command: timeout 5m npx hardhat deposit-eth --network devnetL1
+                working_directory: packages/sdk
+            - run:
+                name: Check the status
+                command: npx hardhat check-op-node
+                working_directory: packages/contracts-bedrock
 
   integration-tests:
     machine:
@@ -773,7 +804,12 @@ workflows:
       - bedrock-go-tests
       - fuzz-op-node
       - bedrock-markdown
-      - devnet
+      - devnet:
+          name: devnet (with deployed contracts)
+          deploy: true
+      - devnet:
+          name: devnet (with genesis contracts)
+          deploy: false
       - go-lint-test-build:
           name: batch-submitter-tests
           binary_name: batch-submitter

.gitignore

@@ -53,3 +53,6 @@ coverage.out
 # Ignore bedrock go bindings local output files
 op-bindings/bin
 op-exporter
+
+
+__pycache__

Makefile

@@ -68,6 +68,10 @@ devnet-up:
 	@bash ./ops-bedrock/devnet-up.sh
 .PHONY: devnet-up
 
+devnet-up-deploy:
+	PYTHONPATH=./bedrock-devnet python3 ./bedrock-devnet/main.py --monorepo-dir=.
+.PHONY: devnet-up-deploy
+
 devnet-down:
 	@(cd ./ops-bedrock && GENESIS_TIMESTAMP=$(shell date +%s) docker-compose stop)
 .PHONY: devnet-down
@@ -114,4 +118,4 @@ tag-bedrock-go-modules:
 
 update-op-geth:
 	./ops/scripts/update-op-geth.py
-.PHONY: update-op-geth
+.PHONY: update-op-geth
\ No newline at end of file

bedrock-devnet/README.md

+# bedrock-devnet
+
+This is a utility for running a local Bedrock devnet. It is designed to replace the legacy Bash-based devnet runner as part of a progressive migration away from Bash automation.
+
+The easiest way to invoke this script is to run `make devnet-up-deploy` from the root of this repository. Otherwise, to use this script run `python3 main.py --monorepo-path=<path to the monorepo>`. You may need to set `PYTHONPATH` to this directory if you are invoking the script from somewhere other than `bedrock-devnet`.
\ No newline at end of file

bedrock-devnet/devnet/__init__.py

+import argparse
+import logging
+import os
+import subprocess
+import json
+import socket
+
+import time
+
+import shutil
+
+import devnet.log_setup
+from devnet.genesis import GENESIS_TMPL
+
+parser = argparse.ArgumentParser(description='Bedrock devnet launcher')
+parser.add_argument('--monorepo-dir', help='Directory of the monorepo', default=os.getcwd())
+
+log = logging.getLogger()
+
+
+def main():
+    args = parser.parse_args()
+
+    pjoin = os.path.join
+    monorepo_dir = os.path.abspath(args.monorepo_dir)
+    devnet_dir = pjoin(monorepo_dir, '.devnet')
+    ops_bedrock_dir = pjoin(monorepo_dir, 'ops-bedrock')
+    contracts_bedrock_dir = pjoin(monorepo_dir, 'packages', 'contracts-bedrock')
+    deployment_dir = pjoin(contracts_bedrock_dir, 'deployments', 'devnetL1')
+    op_node_dir = pjoin(args.monorepo_dir, 'op-node')
+    genesis_l1_path = pjoin(devnet_dir, 'genesis-l1.json')
+    genesis_l2_path = pjoin(devnet_dir, 'genesis-l2.json')
+    addresses_json_path = pjoin(devnet_dir, 'addresses.json')
+    sdk_addresses_json_path = pjoin(devnet_dir, 'sdk-addresses.json')
+    rollup_config_path = pjoin(devnet_dir, 'rollup.json')
+    os.makedirs(devnet_dir, exist_ok=True)
+
+    if os.path.exists(genesis_l1_path):
+        log.info('L2 genesis already generated.')
+    else:
+        log.info('Generating L1 genesis.')
+        write_json(genesis_l1_path, GENESIS_TMPL)
+
+    log.info('Starting L1.')
+    run_command(['docker-compose', 'up', '-d', 'l1'], cwd=ops_bedrock_dir, env={
+        'PWD': ops_bedrock_dir
+    })
+    wait_up(8545)
+
+    log.info('Generating network config.')
+    devnet_cfg_orig = pjoin(contracts_bedrock_dir, 'deploy-config', 'devnetL1.json')
+    devnet_cfg_backup = pjoin(devnet_dir, 'devnetL1.json.bak')
+    shutil.copy(devnet_cfg_orig, devnet_cfg_backup)
+    deploy_config = read_json(devnet_cfg_orig)
+    deploy_config['l1GenesisBlockTimestamp'] = GENESIS_TMPL['timestamp']
+    deploy_config['l1StartingBlockTag'] = 'earliest'
+    write_json(devnet_cfg_orig, deploy_config)
+
+    if os.path.exists(addresses_json_path):
+        log.info('Contracts already deployed.')
+        addresses = read_json(addresses_json_path)
+    else:
+        log.info('Deploying contracts.')
+        run_command(['yarn', 'hardhat', '--network', 'devnetL1', 'deploy'], env={
+            'CHAIN_ID': '900',
+            'L1_RPC': 'http://localhost:8545',
+            'PRIVATE_KEY_DEPLOYER': 'ac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80'
+        }, cwd=contracts_bedrock_dir)
+        contracts = os.listdir(deployment_dir)
+        addresses = {}
+        for c in contracts:
+            if not c.endswith('.json'):
+                continue
+            data = read_json(pjoin(deployment_dir, c))
+            addresses[c.replace('.json', '')] = data['address']
+        sdk_addresses = {}
+        sdk_addresses.update({
+            'StateCommitmentChain': '0x0000000000000000000000000000000000000000',
+            'CanonicalTransactionChain': '0x0000000000000000000000000000000000000000',
+            'BondManager': '0x0000000000000000000000000000000000000000',
+        })
+        sdk_addresses['AddressManager'] = addresses['AddressManager']
+        sdk_addresses['L1CrossDomainMessenger'] = addresses['L1CrossDomainMessengerProxy']
+        sdk_addresses['L1StandardBridge'] = addresses['L1StandardBridgeProxy']
+        sdk_addresses['OptimismPortal'] = addresses['OptimismPortalProxy']
+        sdk_addresses['L2OutputOracle'] = addresses['L2OutputOracleProxy']
+        write_json(addresses_json_path, addresses)
+        write_json(sdk_addresses_json_path, sdk_addresses)
+
+    if os.path.exists(genesis_l2_path):
+        log.info('L2 genesis and rollup configs already generated.')
+    else:
+        log.info('Generating L2 genesis and rollup configs.')
+        run_command([
+            'go', 'run', 'cmd/main.go', 'genesis', 'l2',
+            '--l1-rpc', 'http://localhost:8545',
+            '--deploy-config', devnet_cfg_orig,
+            '--deployment-dir', deployment_dir,
+            '--outfile.l2', pjoin(devnet_dir, 'genesis-l2.json'),
+            '--outfile.rollup', pjoin(devnet_dir, 'rollup.json')
+        ], cwd=op_node_dir)
+
+    rollup_config = read_json(rollup_config_path)
+
+    if os.path.exists(devnet_cfg_backup):
+        shutil.move(devnet_cfg_backup, devnet_cfg_orig)
+
+    log.info('Bringing up L2.')
+    run_command(['docker-compose', 'up', '-d', 'l2'], cwd=ops_bedrock_dir, env={
+        'PWD': ops_bedrock_dir
+    })
+    wait_up(9545)
+
+    log.info('Bringing up everything else.')
+    run_command(['docker-compose', 'up', '-d', 'op-node', 'op-proposer', 'op-batcher'], cwd=ops_bedrock_dir, env={
+        'PWD': ops_bedrock_dir,
+        'L2OO_ADDRESS': addresses['L2OutputOracleProxy'],
+        'SEQUENCER_GENESIS_HASH': rollup_config['genesis']['l2']['hash'],
+        'SEQUENCER_BATCH_INBOX_ADDRESS': rollup_config['batch_inbox_address']
+    })
+
+    log.info('Devnet ready.')
+
+
+def run_command(args, check=True, shell=False, cwd=None, env=None):
+    env = env if env else {}
+    return subprocess.run(
+        args,
+        check=check,
+        shell=shell,
+        env={
+            **os.environ,
+            **env
+        },
+        cwd=cwd
+    )
+
+
+def wait_up(port, retries=10, wait_secs=1):
+    for i in range(0, retries):
+        log.info(f'Trying 127.0.0.1:{port}')
+        s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+        try:
+            s.connect(('127.0.0.1', int(port)))
+            s.shutdown(2)
+            log.info(f'Connected 127.0.0.1:{port}')
+            return True
+        except Exception:
+            time.sleep(wait_secs)
+
+    raise Exception(f'Timed out waiting for port {port}.')
+
+
+def write_json(path, data):
+    with open(path, 'w+') as f:
+        json.dump(data, f, indent='  ')
+
+
+def read_json(path):
+    with open(path, 'r') as f:
+        return json.load(f)

bedrock-devnet/devnet/genesis.py

+import time
+
+DEV_ACCOUNTS = [
+    '3c44cdddb6a900fa2b585dd299e03d12fa4293bc',
+    '70997970c51812dc3a010c7d01b50e0d17dc79c8',
+    'f39fd6e51aad88f6f4ce6ab8827279cfffb92266'
+]
+
+GENESIS_TMPL = {
+    'config': {
+        'chainId': 900,
+        "homesteadBlock": 0,
+        "eip150Block": 0,
+        "eip150Hash": "0x0000000000000000000000000000000000000000000000000000000000000000",
+        "eip155Block": 0,
+        "eip158Block": 0,
+        "byzantiumBlock": 0,
+        "constantinopleBlock": 0,
+        "petersburgBlock": 0,
+        "istanbulBlock": 0,
+        "muirGlacierBlock": 0,
+        "berlinBlock": 0,
+        "londonBlock": 0,
+        "arrowGlacierBlock": 0,
+        "grayGlacierBlock": 0,
+        "shanghaiBlock": None,
+        "cancunBlock": None,
+        'clique': {
+            'period': 15,
+            'epoch': 30000
+        }
+    },
+    'nonce': '0x0',
+    'timestamp': '{:#x}'.format(int(time.time())),
+    'extraData': '0x0000000000000000000000000000000000000000000000000000000000000000ca062b0fd91172d89bcd4bb084ac4e21972cc4670000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000',
+    'gasLimit': '0xE4E1C0',
+    'difficulty': '0x1',
+    'mixHash': '0x0000000000000000000000000000000000000000000000000000000000000000',
+    'coinbase': '0x0000000000000000000000000000000000000000',
+    'alloc': {
+        '{:x}'.format(i).ljust(40, '0'): {
+            'balance': '0x1'
+        } for i in range(0, 255)
+    },
+    'number': '0x0',
+    'gasUsed': '0x0',
+    'parentHash': '0x0000000000000000000000000000000000000000000000000000000000000000',
+    'baseFeePergas': '0x3B9ACA00'
+}
+
+GENESIS_TMPL['alloc'].update({
+    d: {
+        'balance': '0x200000000000000000000000000000000000000000000000000000000000000'
+    } for d in DEV_ACCOUNTS
+})

bedrock-devnet/devnet/log_setup.py

+import os
+
+from logging.config import dictConfig
+
+log_level = os.getenv('LOG_LEVEL')
+
+log_config = {
+    'version': 1,
+    'loggers': {
+        '': {
+            'handlers': ['console'],
+            'level': log_level if log_level is not None else 'INFO'
+        },
+    },
+    'handlers': {
+        'console': {
+            'formatter': 'stderr',
+            'class': 'logging.StreamHandler',
+            'stream': 'ext://sys.stdout'
+        }
+    },
+    'formatters': {
+        'stderr': {
+            'format': '[%(levelname)s|%(asctime)s] %(message)s',
+            'datefmt': '%m-%d-%Y %I:%M:%S'
+        }
+    },
+}
+
+dictConfig(log_config)

bedrock-devnet/main.py

+import devnet
+
+
+def main():
+    devnet.main()
+
+
+if __name__ == '__main__':
+    main()

op-node/cmd/genesis/cmd.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"encoding/json"
 	"errors"
+	"fmt"
 	"math/big"
 	"os"
 	"path/filepath"
@@ -125,7 +126,7 @@ var Subcommands = cli.Commands{
 				l1StartBlock, err = client.BlockByNumber(context.Background(), big.NewInt(config.L1StartingBlockTag.BlockNumber.Int64()))
 			}
 			if err != nil {
-				return err
+				return fmt.Errorf("error getting l1 start block: %w", err)
 			}
 
 			depPath, network := filepath.Split(ctx.String("deployment-dir"))
@@ -157,7 +158,7 @@ var Subcommands = cli.Commands{
 			}
 			l2Genesis, err := genesis.BuildL2DeveloperGenesis(config, l1StartBlock, l2Addrs)
 			if err != nil {
-				return err
+				return fmt.Errorf("error creating l2 developer genesis: %w", err)
 			}
 
 			rollupConfig := makeRollupConfig(config, l1StartBlock, l2Genesis, portalProxy.Address)

packages/contracts-bedrock/deploy-config/devnetL1.json

@@ -23,6 +23,8 @@
   "optimismBaseFeeRecipient": "0xBcd4042DE499D14e55001CcbB24a551F3b954096",
   "optimismL1FeeRecipient": "0x71bE63f3384f5fb98995898A86B02Fb2426c5788",
 
+  "finalizationPeriodSeconds": 2,
+
   "deploymentWaitConfirmations": 1,
   "fundDevAccounts": true
 }

packages/sdk/src/cross-chain-messenger.ts

@@ -176,6 +176,7 @@ export class CrossChainMessenger {
 
     this.bridges = getBridgeAdapters(this.l2ChainId, this, {
       overrides: opts.bridges,
+      contracts: opts.contracts,
     })
   }
 

packages/sdk/src/utils/contracts.ts

@@ -161,6 +161,7 @@ export const getBridgeAdapters = (
   messenger: CrossChainMessenger,
   opts?: {
     overrides?: BridgeAdapterData
+    contracts?: DeepPartial<OEContractsLike>
   }
 ): BridgeAdapters => {
   const adapterData: BridgeAdapterData = {
@@ -168,12 +169,16 @@ export const getBridgeAdapters = (
       ? {
           Standard: {
             Adapter: StandardBridgeAdapter,
-            l1Bridge: CONTRACT_ADDRESSES[l2ChainId].l1.L1StandardBridge,
+            l1Bridge:
+              opts.contracts?.l1?.L1StandardBridge ||
+              CONTRACT_ADDRESSES[l2ChainId].l1.L1StandardBridge,
             l2Bridge: predeploys.L2StandardBridge,
           },
           ETH: {
             Adapter: ETHBridgeAdapter,
-            l1Bridge: CONTRACT_ADDRESSES[l2ChainId].l1.L1StandardBridge,
+            l1Bridge:
+              opts.contracts?.l1?.L1StandardBridge ||
+              CONTRACT_ADDRESSES[l2ChainId].l1.L1StandardBridge,
             l2Bridge: predeploys.L2StandardBridge,
           },
         }

packages/sdk/tasks/deposit-erc20.ts

+import { promises as fs } from 'fs'
+
 import { task, types } from 'hardhat/config'
 import { HardhatRuntimeEnvironment } from 'hardhat/types'
 import '@nomiclabs/hardhat-ethers'
@@ -8,7 +10,13 @@ import {
 } from '@eth-optimism/contracts-bedrock'
 import { Event, Contract, Wallet, providers, utils } from 'ethers'
 
-import { CrossChainMessenger, MessageStatus, CONTRACT_ADDRESSES } from '../src'
+import {
+  CrossChainMessenger,
+  MessageStatus,
+  CONTRACT_ADDRESSES,
+  OEContractsLike,
+  DEFAULT_L2_CONTRACT_ADDRESSES,
+} from '../src'
 
 const deployWETH9 = async (
   hre: HardhatRuntimeEnvironment,
@@ -102,6 +110,12 @@ task('deposit-erc20', 'Deposits WETH9 onto L2.')
     'http://localhost:7545',
     types.string
   )
+  .addOptionalParam(
+    'l1ContractsJsonPath',
+    'Path to a JSON with L1 contract addresses in it',
+    '',
+    types.string
+  )
   .setAction(async (args, hre) => {
     const signers = await hre.ethers.getSigners()
     if (signers.length === 0) {
@@ -127,7 +141,14 @@ task('deposit-erc20', 'Deposits WETH9 onto L2.')
     )
 
     const l2ChainId = await l2Signer.getChainId()
-    const contractAddrs = CONTRACT_ADDRESSES[l2ChainId]
+    let contractAddrs = CONTRACT_ADDRESSES[l2ChainId]
+    if (args.l1ContractsJsonPath) {
+      const data = await fs.readFile(args.l1ContractsJsonPath)
+      contractAddrs = {
+        l1: JSON.parse(data.toString()),
+        l2: DEFAULT_L2_CONTRACT_ADDRESSES,
+      } as OEContractsLike
+    }
 
     const Artifact__L2ToL1MessagePasser = await getContractDefinition(
       'L2ToL1MessagePasser'
@@ -192,6 +213,7 @@ task('deposit-erc20', 'Deposits WETH9 onto L2.')
       l1ChainId: await signer.getChainId(),
       l2ChainId,
       bedrock: true,
+      contracts: contractAddrs,
     })
 
     console.log('Deploying WETH9 to L1')

packages/sdk/tasks/deposit-eth.ts

+import { promises as fs } from 'fs'
+
 import { task, types } from 'hardhat/config'
 import '@nomiclabs/hardhat-ethers'
 import 'hardhat-deploy'
@@ -7,7 +9,13 @@ import {
 } from '@eth-optimism/contracts-bedrock'
 import { providers, utils } from 'ethers'
 
-import { CrossChainMessenger, MessageStatus, CONTRACT_ADDRESSES } from '../src'
+import {
+  CrossChainMessenger,
+  MessageStatus,
+  CONTRACT_ADDRESSES,
+  OEContractsLike,
+  DEFAULT_L2_CONTRACT_ADDRESSES,
+} from '../src'
 
 task('deposit-eth', 'Deposits WETH9 onto L2.')
   .addParam(
@@ -35,6 +43,12 @@ task('deposit-eth', 'Deposits WETH9 onto L2.')
     true,
     types.boolean
   )
+  .addOptionalParam(
+    'l1ContractsJsonPath',
+    'Path to a JSON with L1 contract addresses in it',
+    '',
+    types.string
+  )
   .addOptionalParam('withdrawAmount', 'Amount to withdraw', '', types.string)
   .setAction(async (args, hre) => {
     const signers = await hre.ethers.getSigners()
@@ -70,7 +84,14 @@ task('deposit-eth', 'Deposits WETH9 onto L2.')
     )
 
     const l2ChainId = await l2Signer.getChainId()
-    const contractAddrs = CONTRACT_ADDRESSES[l2ChainId]
+    let contractAddrs = CONTRACT_ADDRESSES[l2ChainId]
+    if (args.l1ContractsJsonPath) {
+      const data = await fs.readFile(args.l1ContractsJsonPath)
+      contractAddrs = {
+        l1: JSON.parse(data.toString()),
+        l2: DEFAULT_L2_CONTRACT_ADDRESSES,
+      } as OEContractsLike
+    }
 
     const Artifact__L2ToL1MessagePasser = await getContractDefinition(
       'L2ToL1MessagePasser'
@@ -135,6 +156,7 @@ task('deposit-eth', 'Deposits WETH9 onto L2.')
       l1ChainId: await signer.getChainId(),
       l2ChainId,
       bedrock: true,
+      contracts: contractAddrs,
     })
 
     const opBalanceBefore = await signer.provider.getBalance(