Merge pull request #7788 from ethereum-optimism/devnet-ci-speed

devnet: devnet-test and CI improvements

.circleci/config.yml

@@ -960,6 +960,7 @@ jobs:
       image: ubuntu-2204:2022.10.2
     environment:
       DOCKER_BUILDKIT: 1
+      DEVNET_NO_BUILD: 'true'
     steps:
       - checkout
       - check-changed:

bedrock-devnet/devnet/__init__.py

@@ -10,6 +10,9 @@ import time
 import shutil
 import http.client
 from multiprocessing import Process, Queue
+import concurrent.futures
+from collections import namedtuple
+
 
 import devnet.log_setup
 
@@ -97,14 +100,18 @@ def main():
     git_commit = subprocess.run(['git', 'rev-parse', 'HEAD'], capture_output=True, text=True).stdout.strip()
     git_date = subprocess.run(['git', 'show', '-s', "--format=%ct"], capture_output=True, text=True).stdout.strip()
 
-    log.info(f'Building docker images for git commit {git_commit} ({git_date})')
-    run_command(['docker', 'compose', 'build', '--progress', 'plain',
-                 '--build-arg', f'GIT_COMMIT={git_commit}', '--build-arg', f'GIT_DATE={git_date}'],
-                cwd=paths.ops_bedrock_dir, env={
-        'PWD': paths.ops_bedrock_dir,
-        'DOCKER_BUILDKIT': '1', # (should be available by default in later versions, but explicitly enable it anyway)
-        'COMPOSE_DOCKER_CLI_BUILD': '1'  # use the docker cache
-    })
+    # CI loads the images from workspace, and does not otherwise know the images are good as-is
+    if os.getenv('DEVNET_NO_BUILD') == "true":
+        log.info('Skipping docker images build')
+    else:
+        log.info(f'Building docker images for git commit {git_commit} ({git_date})')
+        run_command(['docker', 'compose', 'build', '--progress', 'plain',
+                     '--build-arg', f'GIT_COMMIT={git_commit}', '--build-arg', f'GIT_DATE={git_date}'],
+                    cwd=paths.ops_bedrock_dir, env={
+            'PWD': paths.ops_bedrock_dir,
+            'DOCKER_BUILDKIT': '1', # (should be available by default in later versions, but explicitly enable it anyway)
+            'COMPOSE_DOCKER_CLI_BUILD': '1'  # use the docker cache
+        })
 
     log.info('Devnet starting')
     devnet_deploy(paths)
@@ -297,6 +304,10 @@ def wait_for_rpc_server(url):
             log.info(f'Waiting for RPC server at {url}')
             time.sleep(1)
 
+
+CommandPreset = namedtuple('Command', ['name', 'args', 'cwd', 'timeout'])
+
+
 def devnet_test(paths):
     # Check the L2 config
     run_command(
@@ -304,17 +315,57 @@ def devnet_test(paths):
         cwd=paths.ops_chain_ops,
     )
 
-    run_command(
-         ['npx', 'hardhat',  'deposit-erc20', '--network',  'devnetL1', '--l1-contracts-json-path', paths.addresses_json_path],
-         cwd=paths.sdk_dir,
-         timeout=8*60,
-    )
+    # Run the two commands with different signers, so the ethereum nonce management does not conflict
+    # And do not use devnet system addresses, to avoid breaking fee-estimation or nonce values.
+    run_commands([
+        CommandPreset('erc20-test',
+          ['npx', 'hardhat',  'deposit-erc20', '--network',  'devnetL1',
+           '--l1-contracts-json-path', paths.addresses_json_path, '--signer-index', '14'],
+          cwd=paths.sdk_dir, timeout=8*60),
+        CommandPreset('eth-test',
+          ['npx', 'hardhat',  'deposit-eth', '--network',  'devnetL1',
+           '--l1-contracts-json-path', paths.addresses_json_path, '--signer-index', '15'],
+          cwd=paths.sdk_dir, timeout=8*60)
+    ], max_workers=2)
+
+
+def run_commands(commands: list[CommandPreset], max_workers=2):
+    with concurrent.futures.ThreadPoolExecutor(max_workers=max_workers) as executor:
+        futures = [executor.submit(run_command_preset, cmd) for cmd in commands]
+
+        for future in concurrent.futures.as_completed(futures):
+            result = future.result()
+            if result:
+                print(result.stdout)
+
+
+def run_command_preset(command: CommandPreset):
+    with subprocess.Popen(command.args, cwd=command.cwd,
+                          stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) as proc:
+        try:
+            # Live output processing
+            for line in proc.stdout:
+                # Annotate and print the line with timestamp and command name
+                timestamp = datetime.datetime.utcnow().strftime('%H:%M:%S.%f')
+                # Annotate and print the line with the timestamp
+                print(f"[{timestamp}][{command.name}] {line}", end='')
+
+            stdout, stderr = proc.communicate(timeout=command.timeout)
+
+            if proc.returncode != 0:
+                raise RuntimeError(f"Command '{' '.join(command.args)}' failed with return code {proc.returncode}: {stderr}")
+
+        except subprocess.TimeoutExpired:
+            raise RuntimeError(f"Command '{' '.join(command.args)}' timed out!")
+
+        except Exception as e:
+            raise RuntimeError(f"Error executing '{' '.join(command.args)}': {e}")
+
+        finally:
+            # Ensure process is terminated
+            proc.kill()
+    return proc.returncode
 
-    run_command(
-         ['npx', 'hardhat',  'deposit-eth', '--network',  'devnetL1', '--l1-contracts-json-path', paths.addresses_json_path],
-         cwd=paths.sdk_dir,
-         timeout=8*60,
-    )
 
 def run_command(args, check=True, shell=False, cwd=None, env=None, timeout=None):
     env = env if env else {}

ops-bedrock/docker-compose.yml

@@ -33,6 +33,8 @@ services:
       - "l1_data:/db"
       - "${PWD}/../.devnet/genesis-l1.json:/genesis.json"
       - "${PWD}/test-jwt-secret.txt:/config/test-jwt-secret.txt"
+    environment:
+      GETH_MINER_RECOMMIT: 100ms
 
   l2:
     build:
@@ -49,6 +51,8 @@ services:
       - "/bin/sh"
       - "/entrypoint.sh"
       - "--authrpc.jwtsecret=/config/test-jwt-secret.txt"
+    environment:
+      GETH_MINER_RECOMMIT: 100ms
 
   op-node:
     depends_on:

packages/contracts-bedrock/deploy-config/devnetL1-template.json

@@ -11,7 +11,7 @@
   "cliqueSignerAddress": "0xf39Fd6e51aad88F6F4ce6aB8827279cffFb92266",
   "l1UseClique": true,
   "l1StartingBlockTag": "earliest",
-  "l2OutputOracleSubmissionInterval": 6,
+  "l2OutputOracleSubmissionInterval": 10,
   "l2OutputOracleStartingTimestamp": 0,
   "l2OutputOracleStartingBlockNumber": 0,
   "l2OutputOracleProposer": "0x70997970C51812dc3A010C7d01b50e0d17dc79C8",

packages/sdk/hardhat.config.ts

@@ -24,7 +24,27 @@ const config: HardhatUserConfig = {
     devnetL1: {
       url: 'http://localhost:8545',
       accounts: [
-        'ac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80',
+        // warning: keys 0 - 12 (incl) are used by the system
+        'ac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80', // 0
+        '59c6995e998f97a5a0044966f0945389dc9e86dae88c7a8412f4603b6b78690d', // 1
+        '5de4111afa1a4b94908f83103eb1f1706367c2e68ca870fc3fb9a804cdab365a', // 2
+        '7c852118294e51e653712a81e05800f419141751be58f605c371e15141b007a6', // 3
+        '47e179ec197488593b187f80a00eb0da91f1b9d0b13f8733639f19c30a34926a', // 4
+        '8b3a350cf5c34c9194ca85829a2df0ec3153be0318b5e2d3348e872092edffba', // 5
+        '92db14e403b83dfe3df233f83dfa3a0d7096f21ca9b0d6d6b8d88b2b4ec1564e', // 6
+        '4bbbf85ce3377467afe5d46f804f221813b2bb87f24d81f60f1fcdbf7cbf4356', // 7
+        'dbda1821b80551c9d65939329250298aa3472ba22feea921c0cf5d620ea67b97', // 8
+        '2a871d0798f97d79848a013d4936a73bf4cc922c825d33c1cf7073dff6d409c6', // 9
+        'f214f2b2cd398c806f84e317254e0f0b801d0643303237d97a22a48e01628897', // 10
+        '701b615bbdfb9de65240bc28bd21bbc0d996645a3dd57e7b12bc2bdf6f192c82', // 11
+        'a267530f49f8280200edf313ee7af6b827f2a8bce2897751d06a843f644967b1', // 12
+        '47c99abed3324a2707c28affff1267e45918ec8c3f20b8aa892e8b065d2942dd', // 13
+        'c526ee95bf44d8fc405a158bb884d9d1238d99f0612e9f33d006bb0789009aaa', // 14
+        '8166f546bab6da521a8369cab06c5d2b9e46670292d85c875ee9ec20e84ffb61', // 15
+        'ea6c44ac03bff858b476bba40716402b03e41b8e97e276d1baec7c37d42484a0', // 16
+        '689af8efa8c651a91ad287602527f3af2fe9f6501a7ac4b061667b5a93e037fd', // 17
+        'de9be858da4a475276426320d5e9262ecfc3ba460bfac56360bfa6c4c28b4ee0', // 18
+        'df57089febbacf7ba0bc227dafbffa9fc08a93fdc68e1e42411a14efcf23656e', // 19
       ],
     },
     hivenet: {

packages/sdk/tasks/deposit-erc20.ts

@@ -2,6 +2,7 @@ import { promises as fs } from 'fs'
 
 import { task, types } from 'hardhat/config'
 import { HardhatRuntimeEnvironment } from 'hardhat/types'
+import { SignerWithAddress } from '@nomiclabs/hardhat-ethers/signers'
 import '@nomiclabs/hardhat-ethers'
 import 'hardhat-deploy'
 import { Event, Contract, Wallet, providers, utils, ethers } from 'ethers'
@@ -27,11 +28,9 @@ import {
 
 const deployWETH9 = async (
   hre: HardhatRuntimeEnvironment,
+  signer: SignerWithAddress,
   wrap: boolean
 ): Promise<Contract> => {
-  const signers = await hre.ethers.getSigners()
-  const signer = signers[0]
-
   const Factory__WETH9 = new hre.ethers.ContractFactory(
     Artifact__WETH9.abi,
     Artifact__WETH9.bytecode.object,
@@ -117,13 +116,16 @@ task('deposit-erc20', 'Deposits WETH9 onto L2.')
     '',
     types.string
   )
+  .addOptionalParam('signerIndex', 'Index of signer to use', 0, types.int)
   .setAction(async (args, hre) => {
     const signers = await hre.ethers.getSigners()
     if (signers.length === 0) {
       throw new Error('No configured signers')
     }
-    // Use the first configured signer for simplicity
-    const signer = signers[0]
+    if (args.signerIndex < 0 || signers.length <= args.signerIndex) {
+      throw new Error('Invalid signer index')
+    }
+    const signer = signers[args.signerIndex]
     const address = await signer.getAddress()
     console.log(`Using signer ${address}`)
 
@@ -137,7 +139,7 @@ task('deposit-erc20', 'Deposits WETH9 onto L2.')
     const l2Provider = new providers.StaticJsonRpcProvider(args.l2ProviderUrl)
 
     const l2Signer = new hre.ethers.Wallet(
-      hre.network.config.accounts[0],
+      hre.network.config.accounts[args.signerIndex],
       l2Provider
     )
 
@@ -219,7 +221,7 @@ task('deposit-erc20', 'Deposits WETH9 onto L2.')
     console.log(params)
 
     console.log('Deploying WETH9 to L1')
-    const WETH9 = await deployWETH9(hre, true)
+    const WETH9 = await deployWETH9(hre, signer, true)
     console.log(`Deployed to ${WETH9.address}`)
 
     console.log('Creating L2 WETH9')

packages/sdk/tasks/deposit-eth.ts

@@ -50,14 +50,17 @@ task('deposit-eth', 'Deposits ether to L2.')
     '',
     types.string
   )
+  .addOptionalParam('signerIndex', 'Index of signer to use', 0, types.int)
   .addOptionalParam('withdrawAmount', 'Amount to withdraw', '', types.string)
   .setAction(async (args, hre) => {
     const signers = await hre.ethers.getSigners()
     if (signers.length === 0) {
       throw new Error('No configured signers')
     }
-    // Use the first configured signer for simplicity
-    const signer = signers[0]
+    if (args.signerIndex < 0 || signers.length <= args.signerIndex) {
+      throw new Error('Invalid signer index')
+    }
+    const signer = signers[args.signerIndex]
     const address = await signer.getAddress()
     console.log(`Using signer ${address}`)
 
@@ -81,7 +84,7 @@ task('deposit-eth', 'Deposits ether to L2.')
       : amount.div(2)
 
     const l2Signer = new hre.ethers.Wallet(
-      hre.network.config.accounts[0],
+      hre.network.config.accounts[args.signerIndex],
       l2Provider
     )