ops: Add replica generator script (#3243)

Adds a Python script that generates replica configurations for networks deployed using `bedrock-regentool`. Usage instructions are in `ops-bedrock/tools/README.md`.

ops-bedrock/tools/README.md

+# tools
+
+A collection of Bedrock ops tools.
+
+## generate_replica.py
+
+This script generates a replica configuration suitable for use with a deployed Bedrock network. Given a network name and an output directory, it will:
+
+1. Pull the network's genesis, rollup config, and contract addresses from GCP.
+2. Generate P2P/JWT keys.
+3. Generate a `docker-compose.yml` file that can be used to immediately start the replica.
+
+The above files are outputted to a user-defined output directory in case further customization is desired.
+
+The network must already have been deployed using `bedrock-regen`.
+
+**Prerequisites**: Python 3.7 or above. No `pip` or `venv` is necessary, since the script does not have any dependencies outside of the standard library.
+
+**Usage:**
+
+Run `python3 generate.py <options>` to invoke the script. `python3 generate.py -h` will output the usage help text below. All configuration options except for the following are optional: `--network`, `--l1-rpc`, and `--outdir`.
+
+**Example**:
+
+```
+python3 generate_replica.py --network <network-name> --op-node-tag 068113f255fa23edcd628ed853c6e5e616af7b77 --outdir ./replica-regenesis-447cda2 --l1-rpc <removed>
+```
+
+**CLI Helptext**:
+
+```
+generate.py [-h] --network NETWORK --l1-rpc L1_RPC --outdir OUTDIR [--geth-tag GETH_TAG] [--geth-http-port GETH_HTTP_PORT] [--geth-ws-port GETH_WS_PORT] [--op-node-tag OP_NODE_TAG]
+                   [--op-node-http-port OP_NODE_HTTP_PORT] [--op-node-metrics-port OP_NODE_METRICS_PORT] [--op-node-pprof-port OP_NODE_PPROF_PORT] [--bucket BUCKET]
+
+Configure an Optimism Bedrock replica using docker-compose.
+
+optional arguments:
+  -h, --help            show this help message and exit
+  --network NETWORK     name for the network to create a replica for
+  --l1-rpc L1_RPC       l1 RPC provider
+  --outdir OUTDIR       output directory for the replica config
+  --geth-tag GETH_TAG   docker tag to use with geth
+  --geth-http-port GETH_HTTP_PORT
+                        geth http port
+  --geth-ws-port GETH_WS_PORT
+                        geth ws port
+  --op-node-tag OP_NODE_TAG
+                        docker tag to use with the rollup node
+  --op-node-http-port OP_NODE_HTTP_PORT
+                        rollup node http port
+  --op-node-metrics-port OP_NODE_METRICS_PORT
+                        rollup node http port
+  --op-node-pprof-port OP_NODE_PPROF_PORT
+                        rollup node http port
+  --bucket BUCKET       GCP bucket to pull network data from
+
+```
\ No newline at end of file

ops-bedrock/tools/generate_replica.py

+import argparse
+import hashlib
+import logging
+import os
+import shutil
+import sys
+from logging.config import dictConfig
+import urllib.request
+from secrets import token_bytes
+
+log_level = os.getenv('LOG_LEVEL')
+
+log_config = {
+    'version': 1,
+    'loggers': {
+        '': {
+            'handlers': ['console'],
+            'level': log_level if log_level is not None else 'INFO'
+        },
+    },
+    'handlers': {
+        'console': {
+            'formatter': 'stderr',
+            'class': 'logging.StreamHandler',
+            'stream': 'ext://sys.stdout'
+        }
+    },
+    'formatters': {
+        'stderr': {
+            'format': '[%(levelname)s|%(asctime)s] %(message)s',
+            'datefmt': '%m-%d-%Y %I:%M:%S'
+        }
+    },
+}
+
+dictConfig(log_config)
+
+log = logging.getLogger()
+
+parser = argparse.ArgumentParser(description='Configure an Optimism Bedrock replica using docker-compose.')
+parser.add_argument('--network', type=str, help='name for the network to create a replica for', required=True)
+parser.add_argument('--l1-rpc', type=str, help='l1 RPC provider', required=True)
+parser.add_argument('--outdir', type=str, help='output directory for the replica config', required=True)
+parser.add_argument('--geth-tag', type=str, help='docker tag to use with geth', default='optimism-history')
+parser.add_argument('--geth-http-port', type=int, help='geth http port', default=8545)
+parser.add_argument('--geth-ws-port', type=int, help='geth ws port', default=8546)
+parser.add_argument('--op-node-tag', type=str, help='docker tag to use with the rollup node', default='develop')
+parser.add_argument('--op-node-http-port', type=int, help='rollup node http port', default=9545)
+parser.add_argument('--op-node-metrics-port', type=int, help='rollup node http port', default=7300)
+parser.add_argument('--op-node-pprof-port', type=int, help='rollup node http port', default=6300)
+parser.add_argument('--bucket', type=str, help='GCP bucket to pull network data from',
+                    default='https://storage.googleapis.com/bedrock-goerli-regenesis-data')
+
+
+def main():
+    args = parser.parse_args()
+    network = args.network
+    l1_rpc = args.l1_rpc
+    outdir = args.outdir
+    bucket = args.bucket
+
+    if not os.path.isdir(outdir):
+        log.info(f'Output directory {outdir} does not exist, creating it.')
+        os.makedirs(outdir, exist_ok=False)
+
+    if len(os.listdir(outdir)) > 0 and not confirm(
+            f'Output directory {outdir} is not empty. Files may be overwritten. Proceed?'):
+        log.error('Aborted.')
+        sys.exit(1)
+
+    log.info(f'Using network {network}.')
+    log.info(f'Downloading genesis.')
+    get_network_file(outdir, bucket, network, 'genesis.json')
+    log.info(f'Downloading contracts.')
+    get_network_file(outdir, bucket, network, 'contracts.json')
+    log.info(f'Downloading rollup config.')
+    get_network_file(outdir, bucket, network, 'rollup.json')
+
+    log.info('Writing JWT secret.')
+    m = hashlib.sha3_256()
+    m.update(token_bytes(32))
+    dump_file(outdir, 'jwt-secret.txt', m.hexdigest())
+
+    log.info('Writing P2P secret.')
+    m = hashlib.sha3_256()
+    m.update(token_bytes(32))
+    dump_file(outdir, 'p2p-node-key.txt', m.hexdigest())
+
+    log.info('Writing opnode environment.')
+    dump_file(outdir, 'op-node.env', op_node_env_tmpl(l1_rpc, f'ws://l2:{args.geth_ws_port}', args.op_node_http_port))
+
+    log.info('Writing entrypoint.')
+    dump_file(outdir, 'entrypoint.sh', ENTRYPOINT)
+
+    log.info('Writing compose config.')
+    dump_file(outdir, 'docker-compose.yml', docker_compose_tmpl(
+        network,
+        args.geth_tag,
+        args.geth_http_port,
+        args.geth_ws_port,
+        args.op_node_tag,
+        args.op_node_http_port,
+        args.op_node_pprof_port,
+        args.op_node_metrics_port
+    ))
+
+
+def get_network_file(outdir, bucket, network, filename):
+    outfile, _ = urllib.request.urlretrieve(
+        f'{bucket}/{network}/{filename}'
+    )
+    shutil.move(outfile, os.path.join(outdir, filename))
+    return outfile
+
+
+def confirm(msg):
+    while True:
+        res = input(f'{msg} y/n ')
+        if res in 'y':
+            return True
+        elif res in 'n':
+            return False
+        else:
+            print('\nInvalid option, please try again.')
+
+
+def dump_file(outdir, filename, content):
+    with open(os.path.join(outdir, filename), 'w+') as f:
+        f.write(content)
+
+
+def op_node_env_tmpl(l1_rpc, l2_rpc, op_node_http_port):
+    return f"""
+OP_NODE_L1_ETH_RPC={l1_rpc}
+OP_NODE_L2_ETH_RPC={l2_rpc}
+OP_NODE_ROLLUP_CONFIG=/config/rollup.json
+OP_NODE_L2_ENGINE_RPC={l2_rpc}
+OP_NODE_RPC_ADDR=0.0.0.0
+OP_NODE_RPC_PORT={op_node_http_port}
+OP_NODE_P2P_LISTEN_IP=0.0.0.0
+OP_NODE_P2P_LISTEN_TCP_PORT=9003
+OP_NODE_P2P_LISTEN_UDP_PORT=9003
+OP_NODE_P2P_PRIV_PATH=/config/p2p-node-key.txt
+OP_NODE_P2P_PEERSTORE_PATH=/p2p/peerstore
+OP_NODE_P2P_DISCOVERY_PATH=/p2p/discovery
+OP_NODE_L2_ENGINE_AUTH=/config/jwt-secret.txt
+OP_NODE_VERIFIER_L1_CONFS=3
+OP_NODE_LOG_FORMAT=json
+
+# OP_NODE_P2P_ADVERTISE_IP=
+# OP_NODE_P2P_ADVERTISE_TCP=9003
+# OP_NODE_P2P_ADVERTISE_TCP=9003
+
+OP_NODE_METRICS_ENABLED=true
+OP_NODE_METRICS_ADDR=127.0.0.1
+OP_NODE_METRICS_PORT=7300
+
+OP_NODE_PPROF_ENABLED=true
+OP_NODE_PPROF_PORT=6666
+OP_NODE_PPROF_ADDR=127.0.0.1
+    """
+
+
+def docker_compose_tmpl(network, geth_tag, geth_http_port, geth_ws_port, op_node_tag, op_node_http_port,
+                        op_node_pprof_port,
+                        op_node_metrics_port):
+    return f"""
+version: '3.4'
+
+volumes:
+  {network}_l2_data:
+  {network}_op_log:
+
+services:
+  l2:
+    image: ethereumoptimism/reference-optimistic-geth:{geth_tag}
+    ports:
+      - "{geth_http_port}:8545"
+      - "{geth_ws_port}:8546"
+    volumes:
+      - "{network}_l2_data:/db"
+      - ./genesis.json:/genesis.json
+      - ./jwt-secret.txt:/jwt-secret.txt
+      - ./entrypoint.sh:/entrypoint.sh
+    entrypoint:
+      - "/bin/sh"
+      - "/entrypoint.sh"
+      - "--authrpc.jwtsecret=/jwt-secret.txt"
+
+  op-node:
+    depends_on:
+      - l2
+    image: us-central1-docker.pkg.dev/bedrock-goerli-development/images/op-node:{op_node_tag}
+    command: op-node
+    ports:
+      - "{op_node_http_port}:8545"
+      - "{op_node_pprof_port}:6666"
+      - "{op_node_metrics_port}:7300"
+    env_file:
+      - ./op-node.env
+    volumes:
+      - ./jwt-secret.txt:/config/jwt-secret.txt
+      - ./rollup.json:/config/rollup.json
+      - ./p2p-node-key.txt:/config/p2p-node-key.txt
+      - {network}_op_log:/op_log
+    """
+
+
+ENTRYPOINT = """
+#!/bin/sh
+set -exu
+
+apk add jq
+
+VERBOSITY=${GETH_VERBOSITY:-3}
+GETH_DATA_DIR=/db
+GETH_CHAINDATA_DIR="$GETH_DATA_DIR/geth/chaindata"
+GETH_KEYSTORE_DIR="$GETH_DATA_DIR/keystore"
+GENESIS_FILE_PATH="${GENESIS_FILE_PATH:-/genesis.json}"
+CHAIN_ID=$(cat "$GENESIS_FILE_PATH" | jq -r .config.chainId)
+BLOCK_SIGNER_PRIVATE_KEY="3e4bde571b86929bf08e2aaad9a6a1882664cd5e65b96fff7d03e1c4e6dfa15c"
+BLOCK_SIGNER_ADDRESS="0xca062b0fd91172d89bcd4bb084ac4e21972cc467"
+RPC_PORT="${RPC_PORT:-8545}"
+WS_PORT="${WS_PORT:-8546}"
+
+if [ ! -d "$GETH_KEYSTORE_DIR" ]; then
+    echo "$GETH_KEYSTORE_DIR missing, running account import"
+    echo -n "pwd" > "$GETH_DATA_DIR"/password
+    echo -n "$BLOCK_SIGNER_PRIVATE_KEY" | sed 's/0x//' > "$GETH_DATA_DIR"/block-signer-key
+    geth account import \\
+        --datadir="$GETH_DATA_DIR" \\
+        --password="$GETH_DATA_DIR"/password \\
+        "$GETH_DATA_DIR"/block-signer-key
+else
+    echo "$GETH_KEYSTORE_DIR exists."
+fi
+
+if [ ! -d "$GETH_CHAINDATA_DIR" ]; then
+    echo "$GETH_CHAINDATA_DIR missing, running init"
+    echo "Initializing genesis."
+    geth --verbosity="$VERBOSITY" init \\
+        --datadir="$GETH_DATA_DIR" \\
+        "$GENESIS_FILE_PATH"
+else
+    echo "$GETH_CHAINDATA_DIR exists."
+fi
+
+# Warning: Archive mode is required, otherwise old trie nodes will be
+# pruned within minutes of starting the devnet.
+
+exec geth \\
+    --datadir="$GETH_DATA_DIR" \\
+    --verbosity="$VERBOSITY" \\
+    --http \\
+    --http.corsdomain="*" \\
+    --http.vhosts="*" \\
+    --http.addr=0.0.0.0 \\
+    --http.port="$RPC_PORT" \\
+    --http.api=web3,debug,eth,txpool,net,engine \\
+    --ws \\
+    --ws.addr=0.0.0.0 \\
+    --ws.port="$WS_PORT" \\
+    --ws.origins="*" \\
+    --ws.api=debug,eth,txpool,net,engine \\
+    --syncmode=full \\
+    --nodiscover \\
+    --maxpeers=1 \\
+    --networkid=$CHAIN_ID \\
+    --unlock=$BLOCK_SIGNER_ADDRESS \\
+    --mine \\
+    --miner.etherbase=$BLOCK_SIGNER_ADDRESS \\
+    --password="$GETH_DATA_DIR"/password \\
+    --allow-insecure-unlock \\
+    --gcmode=archive \\
+    "$@"
+"""
+
+if __name__ == '__main__':
+    main()