devnet: unify tooling for how to bring up devnet

The bash script is removed in favor of unifying on the python
code. In the future we should only have 1 way to bring up the devnet
because it is difficult to maintain 2 different ways. It adds
additional CI runs plus the need to add functionality to 2 different
things when contracts are added to the system.

Makefile

@@ -74,11 +74,11 @@ nuke: clean devnet-clean
 .PHONY: nuke
 
 devnet-up:
-	@bash ./ops-bedrock/devnet-up.sh
+	PYTHONPATH=./bedrock-devnet python3 ./bedrock-devnet/main.py --monorepo-dir=.
 .PHONY: devnet-up
 
 devnet-up-deploy:
-	PYTHONPATH=./bedrock-devnet python3 ./bedrock-devnet/main.py --monorepo-dir=.
+	PYTHONPATH=./bedrock-devnet python3 ./bedrock-devnet/main.py --monorepo-dir=. --deploy
 .PHONY: devnet-up-deploy
 
 devnet-down:

bedrock-devnet/devnet/__init__.py

@@ -4,74 +4,142 @@ import os
 import subprocess
 import json
 import socket
-
+import calendar
+import datetime
 import time
-
 import shutil
 
 import devnet.log_setup
 from devnet.genesis import GENESIS_TMPL
 
+pjoin = os.path.join
+
 parser = argparse.ArgumentParser(description='Bedrock devnet launcher')
 parser.add_argument('--monorepo-dir', help='Directory of the monorepo', default=os.getcwd())
+parser.add_argument('--deploy', help='Whether the contracts should be predeployed or deployed', type=bool, action=argparse.BooleanOptionalAction)
 
 log = logging.getLogger()
 
+class Bunch:
+    def __init__(self, **kwds):
+        self.__dict__.update(kwds)
 
 def main():
     args = parser.parse_args()
 
-    pjoin = os.path.join
     monorepo_dir = os.path.abspath(args.monorepo_dir)
     devnet_dir = pjoin(monorepo_dir, '.devnet')
-    ops_bedrock_dir = pjoin(monorepo_dir, 'ops-bedrock')
     contracts_bedrock_dir = pjoin(monorepo_dir, 'packages', 'contracts-bedrock')
     deployment_dir = pjoin(contracts_bedrock_dir, 'deployments', 'devnetL1')
     op_node_dir = pjoin(args.monorepo_dir, 'op-node')
-    genesis_l1_path = pjoin(devnet_dir, 'genesis-l1.json')
-    genesis_l2_path = pjoin(devnet_dir, 'genesis-l2.json')
-    addresses_json_path = pjoin(devnet_dir, 'addresses.json')
-    sdk_addresses_json_path = pjoin(devnet_dir, 'sdk-addresses.json')
-    rollup_config_path = pjoin(devnet_dir, 'rollup.json')
+    ops_bedrock_dir=pjoin(monorepo_dir, 'ops-bedrock')
+
+    paths = Bunch(
+      mono_repo_dir=monorepo_dir,
+      devnet_dir=devnet_dir,
+      contracts_bedrock_dir=contracts_bedrock_dir,
+      deployment_dir=deployment_dir,
+      deploy_config_dir=pjoin(contracts_bedrock_dir, 'deploy-config'),
+      op_node_dir=op_node_dir,
+      ops_bedrock_dir=ops_bedrock_dir,
+      genesis_l1_path=pjoin(devnet_dir, 'genesis-l1.json'),
+      genesis_l2_path=pjoin(devnet_dir, 'genesis-l2.json'),
+      addresses_json_path=pjoin(devnet_dir, 'addresses.json'),
+      sdk_addresses_json_path=pjoin(devnet_dir, 'sdk-addresses.json'),
+      rollup_config_path=pjoin(devnet_dir, 'rollup.json')
+    )
+
     os.makedirs(devnet_dir, exist_ok=True)
 
-    if os.path.exists(genesis_l1_path):
-        log.info('L2 genesis already generated.')
+    if args.deploy:
+      log.info('Devnet with upcoming smart contract deployments')
+      devnet_deploy(paths)
+    else:
+      log.info('Devnet with smart contracts pre-deployed')
+      devnet_prestate(paths)
+
+# Bring up the devnet where the L1 contracts are in the genesis state
+def devnet_prestate(paths):
+    date = datetime.datetime.utcnow()
+    utc_time = hex(calendar.timegm(date.utctimetuple()))
+
+    done_file = pjoin(paths.devnet_dir, 'done')
+    if os.path.exists(done_file):
+        log.info('Genesis files already exist')
+    else:
+        log.info('Creating genesis files')
+        deploy_config_path = pjoin(paths.deploy_config_dir, 'devnetL1.json')
+
+        # read the json file
+        deploy_config = read_json(deploy_config_path)
+        deploy_config['l1GenesisBlockTimestamp'] = utc_time
+        temp_deploy_config = pjoin(paths.devnet_dir, 'deploy-config.json')
+        write_json(temp_deploy_config, deploy_config)
+
+        outfile_l1 = paths.genesis_l1_path
+        outfile_l2 = paths.genesis_l2_path
+        outfile_rollup = paths.rollup_config_path
+
+        run_command(['go', 'run', 'cmd/main.go', 'genesis', 'devnet', '--deploy-config', temp_deploy_config, '--outfile.l1', outfile_l1, '--outfile.l2', outfile_l2, '--outfile.rollup', outfile_rollup], cwd=paths.op_node_dir)
+        write_json(done_file, {})
+
+    log.info('Bringing up L1.')
+    run_command(['docker-compose', 'up', '-d', 'l1'], cwd=paths.ops_bedrock_dir, env={
+        'PWD': paths.ops_bedrock_dir
+    })
+    wait_up(8545)
+
+    log.info('Bringing up L2.')
+    run_command(['docker-compose', 'up', '-d', 'l2'], cwd=paths.ops_bedrock_dir, env={
+        'PWD': paths.ops_bedrock_dir
+    })
+    wait_up(9545)
+
+    log.info('Bringing up the services.')
+    run_command(['docker-compose', 'up', '-d', 'op-proposer', 'op-batcher'], cwd=paths.ops_bedrock_dir, env={
+        'PWD': paths.ops_bedrock_dir,
+        'L2OO_ADDRESS': '0x6900000000000000000000000000000000000000'
+    })
+
+# Bring up the devnet where the contracts are deployed to L1
+def devnet_deploy(paths):
+    if os.path.exists(paths.genesis_l1_path):
+        log.info('L1 genesis already generated.')
     else:
         log.info('Generating L1 genesis.')
-        write_json(genesis_l1_path, GENESIS_TMPL)
+        write_json(paths.genesis_l1_path, GENESIS_TMPL)
 
     log.info('Starting L1.')
-    run_command(['docker-compose', 'up', '-d', 'l1'], cwd=ops_bedrock_dir, env={
-        'PWD': ops_bedrock_dir
+    run_command(['docker-compose', 'up', '-d', 'l1'], cwd=paths.ops_bedrock_dir, env={
+        'PWD': paths.ops_bedrock_dir
     })
     wait_up(8545)
 
     log.info('Generating network config.')
-    devnet_cfg_orig = pjoin(contracts_bedrock_dir, 'deploy-config', 'devnetL1.json')
-    devnet_cfg_backup = pjoin(devnet_dir, 'devnetL1.json.bak')
+    devnet_cfg_orig = pjoin(paths.contracts_bedrock_dir, 'deploy-config', 'devnetL1.json')
+    devnet_cfg_backup = pjoin(paths.devnet_dir, 'devnetL1.json.bak')
     shutil.copy(devnet_cfg_orig, devnet_cfg_backup)
     deploy_config = read_json(devnet_cfg_orig)
     deploy_config['l1GenesisBlockTimestamp'] = GENESIS_TMPL['timestamp']
     deploy_config['l1StartingBlockTag'] = 'earliest'
     write_json(devnet_cfg_orig, deploy_config)
 
-    if os.path.exists(addresses_json_path):
+    if os.path.exists(paths.addresses_json_path):
         log.info('Contracts already deployed.')
-        addresses = read_json(addresses_json_path)
+        addresses = read_json(paths.addresses_json_path)
     else:
         log.info('Deploying contracts.')
         run_command(['yarn', 'hardhat', '--network', 'devnetL1', 'deploy', '--tags', 'l1'], env={
             'CHAIN_ID': '900',
             'L1_RPC': 'http://localhost:8545',
             'PRIVATE_KEY_DEPLOYER': 'ac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80'
-        }, cwd=contracts_bedrock_dir)
-        contracts = os.listdir(deployment_dir)
+        }, cwd=paths.contracts_bedrock_dir)
+        contracts = os.listdir(paths.deployment_dir)
         addresses = {}
         for c in contracts:
             if not c.endswith('.json'):
                 continue
-            data = read_json(pjoin(deployment_dir, c))
+            data = read_json(pjoin(paths.deployment_dir, c))
             addresses[c.replace('.json', '')] = data['address']
         sdk_addresses = {}
         sdk_addresses.update({
@@ -84,10 +152,10 @@ def main():
         sdk_addresses['L1StandardBridge'] = addresses['Proxy__OVM_L1StandardBridge']
         sdk_addresses['OptimismPortal'] = addresses['OptimismPortalProxy']
         sdk_addresses['L2OutputOracle'] = addresses['L2OutputOracleProxy']
-        write_json(addresses_json_path, addresses)
-        write_json(sdk_addresses_json_path, sdk_addresses)
+        write_json(paths.addresses_json_path, addresses)
+        write_json(paths.sdk_addresses_json_path, sdk_addresses)
 
-    if os.path.exists(genesis_l2_path):
+    if os.path.exists(paths.genesis_l2_path):
         log.info('L2 genesis and rollup configs already generated.')
     else:
         log.info('Generating L2 genesis and rollup configs.')
@@ -95,25 +163,25 @@ def main():
             'go', 'run', 'cmd/main.go', 'genesis', 'l2',
             '--l1-rpc', 'http://localhost:8545',
             '--deploy-config', devnet_cfg_orig,
-            '--deployment-dir', deployment_dir,
-            '--outfile.l2', pjoin(devnet_dir, 'genesis-l2.json'),
-            '--outfile.rollup', pjoin(devnet_dir, 'rollup.json')
-        ], cwd=op_node_dir)
+            '--deployment-dir', paths.deployment_dir,
+            '--outfile.l2', pjoin(paths.devnet_dir, 'genesis-l2.json'),
+            '--outfile.rollup', pjoin(paths.devnet_dir, 'rollup.json')
+        ], cwd=paths.op_node_dir)
 
-    rollup_config = read_json(rollup_config_path)
+    rollup_config = read_json(paths.rollup_config_path)
 
     if os.path.exists(devnet_cfg_backup):
         shutil.move(devnet_cfg_backup, devnet_cfg_orig)
 
     log.info('Bringing up L2.')
-    run_command(['docker-compose', 'up', '-d', 'l2'], cwd=ops_bedrock_dir, env={
-        'PWD': ops_bedrock_dir
+    run_command(['docker-compose', 'up', '-d', 'l2'], cwd=paths.ops_bedrock_dir, env={
+        'PWD': paths.ops_bedrock_dir
     })
     wait_up(9545)
 
     log.info('Bringing up everything else.')
-    run_command(['docker-compose', 'up', '-d', 'op-node', 'op-proposer', 'op-batcher'], cwd=ops_bedrock_dir, env={
-        'PWD': ops_bedrock_dir,
+    run_command(['docker-compose', 'up', '-d', 'op-node', 'op-proposer', 'op-batcher'], cwd=paths.ops_bedrock_dir, env={
+        'PWD': paths.ops_bedrock_dir,
         'L2OO_ADDRESS': addresses['L2OutputOracleProxy'],
         'SEQUENCER_BATCH_INBOX_ADDRESS': rollup_config['batch_inbox_address']
     })

ops-bedrock/devnet-up.sh

-#!/usr/bin/env bash
-
-# This script starts a local devnet using Docker Compose. We have to use
-# this more complicated Bash script rather than Compose's native orchestration
-# tooling because we need to start each service in a specific order, and specify
-# their configuration along the way. The order is:
-#
-# 1. Start L1.
-# 2. Compile contracts.
-# 3. Deploy the contracts to L1 if necessary.
-# 4. Start L2, inserting the compiled contract artifacts into the genesis.
-# 5. Get the genesis hashes and timestamps from L1/L2.
-# 6. Generate the rollup driver's config using the genesis hashes and the
-#    timestamps recovered in step 4 as well as the address of the OptimismPortal
-#    contract deployed in step 3.
-# 7. Start the rollup driver.
-# 8. Start the L2 output submitter.
-#
-# The timestamps are critically important here, since the rollup driver will fill in
-# empty blocks if the tip of L1 lags behind the current timestamp. This can lead to
-# a perceived infinite loop. To get around this, we set the timestamp to the current
-# time in this script.
-#
-# This script is safe to run multiple times. It stores state in `.devnet`, and
-# contracts-bedrock/deployments/devnetL1.
-#
-# Don't run this script directly. Run it using the makefile, e.g. `make devnet-up`.
-# To clean up your devnet, run `make devnet-clean`.
-
-set -eu
-
-L1_URL="http://localhost:8545"
-L2_URL="http://localhost:9545"
-
-OP_NODE="$PWD/op-node"
-CONTRACTS_BEDROCK="$PWD/packages/contracts-bedrock"
-NETWORK=devnetL1
-DEVNET="$PWD/.devnet"
-
-# Helper method that waits for a given URL to be up. Can't use
-# cURL's built-in retry logic because connection reset errors
-# are ignored unless you're using a very recent version of cURL
-function wait_up {
-  echo -n "Waiting for $1 to come up..."
-  i=0
-  until curl -s -f -o /dev/null "$1"
-  do
-    echo -n .
-    sleep 0.25
-
-    ((i=i+1))
-    if [ "$i" -eq 300 ]; then
-      echo " Timeout!" >&2
-      exit 1
-    fi
-  done
-  echo "Done!"
-}
-
-mkdir -p ./.devnet
-
-# Regenerate the L1 genesis file if necessary. The existence of the genesis
-# file is used to determine if we need to recreate the devnet's state folder.
-if [ ! -f "$DEVNET/done" ]; then
-  echo "Regenerating genesis files"
-
-  TIMESTAMP=$(date +%s | xargs printf '0x%x')
-  cat "$CONTRACTS_BEDROCK/deploy-config/devnetL1.json" | jq -r ".l1GenesisBlockTimestamp = \"$TIMESTAMP\"" > /tmp/bedrock-devnet-deploy-config.json
-
-  (
-    cd "$OP_NODE"
-    go run cmd/main.go genesis devnet \
-        --deploy-config /tmp/bedrock-devnet-deploy-config.json \
-        --outfile.l1 $DEVNET/genesis-l1.json \
-        --outfile.l2 $DEVNET/genesis-l2.json \
-        --outfile.rollup $DEVNET/rollup.json
-    touch "$DEVNET/done"
-  )
-fi
-
-# Bring up L1.
-(
-  cd ops-bedrock
-  echo "Bringing up L1..."
-  DOCKER_BUILDKIT=1 docker-compose build --progress plain
-  docker-compose up -d l1
-  wait_up $L1_URL
-)
-
-# Bring up L2.
-(
-  cd ops-bedrock
-  echo "Bringing up L2..."
-  docker-compose up -d l2
-  wait_up $L2_URL
-)
-
-L2OO_ADDRESS="0x6900000000000000000000000000000000000000"
-
-# Bring up everything else.
-(
-  cd ops-bedrock
-  echo "Bringing up devnet..."
-  L2OO_ADDRESS="$L2OO_ADDRESS" \
-      docker-compose up -d op-proposer op-batcher
-
-  echo "Bringing up stateviz webserver..."
-  docker-compose up -d stateviz
-)
-
-echo "Devnet ready."