Merge pull request #1817 from mslipper/feat/actor-tests-framework

integration-tests: Add actor testing framework, ACD actor tests

.changeset/sour-oranges-pull.md

+---
+'@eth-optimism/integration-tests': minor
+---
+
+Add actor tests

.gitignore

@@ -23,4 +23,5 @@ packages/data-transport-layer/db
 *.swp
 
 .env
+.env*
 *.log

integration-tests/README.md

@@ -9,7 +9,7 @@ Then, run:
 yarn build
 ```
 
-## Running tests
+## Running integration tests
 
 ### Testing a live network
 
@@ -40,3 +40,12 @@ To run the Uniswap integration tests against a deployed set of Uniswap contracts
 UNISWAP_POSITION_MANAGER_ADDRESS=<non fungible position manager address>
 UNISWAP_ROUTER_ADDRESS=<router address>
 ```
+
+
+## Running actor tests
+
+Actor tests use the same environment variables as the integration tests, so set up your `.env` file if you haven't
+already. Then, run `yarn test:actor <args>` to run the tests. Note that it will be **very expensive** to run the actor
+tests against mainnet, and that the tests can take a while to complete.
+
+See [actor-tests/README.md](actor-tests/README.md) for information on actor tests. 
\ No newline at end of file

integration-tests/actor-tests/README.md

+# Actor Tests
+
+This README describes how to use the actor testing library to write new tests. If you're just looking for how to run test cases, check out the README [in the root of the repo](../README.md).
+
+## Introduction
+
+An "actor test" is a test case that simulates how a user might interact with Optimism. For example, an actor test might automate a user minting NFTs or swapping on Uniswap. Multiple actor tests are composed together in order to simulate real-world usage and help us optimize network performance under realistic load.
+
+Actor tests are designed to catch race conditions, resource leaks, and performance regressions. They aren't a replacement for standard unit/integration tests, and aren't executed on every pull request since they take time to run.
+
+This directory contains the actor testing framework as well as the tests themselves. The framework lives in `lib` and the tests live in this directory with `.test.ts` prefixes. Read on to find out more about how to use the framework to write actor tests of your own.
+
+## CLI
+
+Use the following command to run actor tests from the CLI:
+
+```
+ts-node actor-tests/lib/runner.ts -f <path-to-test-file> -c <concurrency> -r <num-runs> -t <time-to-run> --think-time <think time?>
+```
+
+You can also run `ts-node actor-tests/lib/runner.ts --help` for a full list of options with documentation.
+
+**Arguments:**
+
+- `path-to-test-file`: A path to the TS file containing the actor test.
+- `concurrency`: How many workers to spawn.
+- `run-for`: How long, in milliseconds, the worker should run.
+- `think-time`: How long the runner should pause between test runs. Defaults to zero, meaning runs execute as fast as possible.
+
+## Usage
+
+### Test DSL
+
+Actor tests are defined using a Mocha-like DSL. Follow along using the example below:
+
+```typescript
+import { actor, setupRun, setupActor, run } from './lib/convenience'
+
+interface Context {
+  wallet: Wallet
+}
+
+actor('Value sender', () => {
+  let env: OptimismEnv
+
+  setupActor(async () => {
+    env = await OptimismEnv.new()
+  })
+
+  setupRun(async () => {
+    const wallet = Wallet.createRandom()
+    await env.l2Wallet.sendTransaction({
+      to: wallet.address,
+      value: utils.parseEther('0.01'),
+    })
+    return {
+      wallet: wallet.connect(env.l2Wallet.provider),
+    }
+  })
+
+  run(async (b, ctx: Context) => {
+    const randWallet = Wallet.createRandom().connect(env.l2Wallet.provider)
+    await b.bench('send funds', async () => {
+      await ctx.wallet.sendTransaction({
+        to: randWallet.address,
+        value: 0x42,
+      })
+    })
+    expect(await randWallet.getBalance()).to.deep.equal(BigNumber.from(0x42))
+  })
+})
+```
+
+#### `actor(name: string, cb: () => void)`
+
+Defines a new actor.
+
+**Arguments:**
+
+- `name`: Sets the actor's name. Used in logs and in outputted metrics.
+- `cb`: The body of the actor. Cannot be async. All the other DSL methods (i.e. `setup*`, `run`) must be called within this callback.
+
+#### `setupActor(cb: () => Promise<void>)`
+
+Defines a setup method that gets called after the actor is instantiated but before any workers are spawned. Useful to set variables that need to be shared across all worker instances.
+
+**Note:** Any variables set using `setupActor` must be thread-safe. Don't use `setupActor` to define a shared `Provider` instance, for example, since this will introduce nonce errors. Use `setupRun` to define a test context instead.
+
+#### `setupRun(cb: () => Promise<T>)`
+
+Defines a setup method that gets called inside each worker after it is instantiated but before any runs have executed. The value returned by the `setupRun` method becomes the worker's test context, which will be described in more detail below.
+
+**Note:** While `setupRun` is called once in each worker, invocations of the `setupRun` callback are executed serially. This makes `setupRun` useful for nonce-dependent setup tasks like funding worker wallets.
+
+#### `run<T>(cb: (b: Benchmarker, ctx: T) => Promise<void>)`
+
+Defines what the actor actually does. The test runner will execute the `run` method multiple times depending on its configuration.
+
+**Benchmarker**
+
+Sections of the `run` method can be benchmarked using the `Benchmarker` argument to the `run` callback. Use the `Benchmarker` like this:
+
+```typescript
+b.bench('bench name', async () => {
+ // benchmarked code here
+})
+```
+
+A summary of the benchmark's runtime and a count of how many times it succeeded/failed across each worker will be recorded in the run's metrics.
+
+**Context**
+
+The value returned by `setupRun` will be passed into the `ctx` argument to the `run` callback. Use the test context for values that need to be local to a particular worker. In the example, we use it to pass around the worker's wallet.
+
+### Error Handling
+
+Errors in setup methods cause the test process to crash. Errors in the `run` method are recorded in the test's metrics, and cause the run to be retried. The runtime of failed runs are not recorded.
+
+It's useful to use `expect`/`assert` to make sure that actors are executing properly.
+
+### Test Runner
+
+The test runner is responsible for executing actor tests and managing their lifecycle. It can run in one of two modes: 
+
+1. Fixed run mode, which will execute the `run` method a fixed number of times.
+2. Timed mode, which will will execute the `run` method as many times as possible until a period of time has elapsed.
+
+Test lifecycle is as follows:
+
+1. The runner collects all the actors it needs to run.
+	
+	> Actors automatically register themselves with the default instance of the runner upon being `require()`d.
+2. The runner executes each actor's `setupActor` method.
+3. The runner spawns `n` workers.
+4. The runner executes the `setupRun` method in each worker. The runner will wait for all `setupRun` methods to complete before continuing.
+5. The runner executes the `run` method according to the mode described above.
+
+## Metrics
+
+The test runner prints metrics about each run to `stdout` on exit. This output can then be piped into Prometheus for visualization in Grafana or similar tools. Example metrics output might looks like:
+
+```
+# HELP actor_successful_bench_runs_total Count of total successful bench runs.
+# TYPE actor_successful_bench_runs_total counter
+actor_successful_bench_runs_total{actor_name="value_sender",bench_name="send_funds",worker_id="0"} 20
+actor_successful_bench_runs_total{actor_name="value_sender",bench_name="send_funds",worker_id="1"} 20
+
+# HELP actor_failed_bench_runs_total Count of total failed bench runs.
+# TYPE actor_failed_bench_runs_total counter
+
+# HELP actor_step_durations_ms_summary Summary of successful bench durations.
+# TYPE actor_step_durations_ms_summary summary
+actor_step_durations_ms_summary{quantile="0.5",actor_name="value_sender",bench_name="send_funds"} 1278.0819790065289
+actor_step_durations_ms_summary{quantile="0.9",actor_name="value_sender",bench_name="send_funds"} 1318.4640210270882
+actor_step_durations_ms_summary{quantile="0.95",actor_name="value_sender",bench_name="send_funds"} 1329.5195834636688
+actor_step_durations_ms_summary{quantile="0.99",actor_name="value_sender",bench_name="send_funds"} 1338.0024159550667
+actor_step_durations_ms_summary_sum{actor_name="value_sender",bench_name="send_funds"} 51318.10741400719
+actor_step_durations_ms_summary_count{actor_name="value_sender",bench_name="send_funds"} 40
+
+# HELP actor_successful_actor_runs_total Count of total successful actor runs.
+# TYPE actor_successful_actor_runs_total counter
+actor_successful_actor_runs_total{actor_name="value_sender"} 40
+
+# HELP actor_failed_actor_runs_total Count of total failed actor runs.
+# TYPE actor_failed_actor_runs_total counter
+```
\ No newline at end of file

integration-tests/actor-tests/chain-reader.test.ts

+import { actor, run, setupActor } from './lib/convenience'
+import { OptimismEnv } from '../test/shared/env'
+
+actor('Chain reader', () => {
+  let env: OptimismEnv
+
+  setupActor(async () => {
+    env = await OptimismEnv.new()
+  })
+
+  run(async (b) => {
+    const blockNumber = await b.bench('get block number', () =>
+      env.l2Provider.getBlockNumber()
+    )
+    await b.bench('get random block', () =>
+      env.l2Provider.getBlock(Math.floor(blockNumber * Math.random()))
+    )
+  })
+})

integration-tests/actor-tests/deposits.test.ts

+import { utils, Wallet, BigNumber } from 'ethers'
+import { setupActor, setupRun, actor, run } from './lib/convenience'
+import { OptimismEnv } from '../test/shared/env'
+import { Direction } from '../test/shared/watcher-utils'
+import { expect } from 'chai'
+
+interface BenchContext {
+  l1Wallet: Wallet
+  l2Wallet: Wallet
+}
+
+const DEFAULT_TEST_GAS_L1 = 330_000
+const DEFAULT_TEST_GAS_L2 = 1_300_000
+
+actor('Funds depositor', () => {
+  let env: OptimismEnv
+
+  setupActor(async () => {
+    env = await OptimismEnv.new()
+  })
+
+  setupRun(async () => {
+    const wallet = Wallet.createRandom()
+    await env.l1Wallet.sendTransaction({
+      to: wallet.address,
+      value: utils.parseEther('0.01'),
+    })
+    return {
+      l1Wallet: wallet.connect(env.l1Wallet.provider),
+      l2Wallet: wallet.connect(env.l2Wallet.provider),
+    }
+  })
+
+  run(async (b, ctx: BenchContext) => {
+    const { l1Wallet, l2Wallet } = ctx
+    const balBefore = await l2Wallet.getBalance()
+    await b.bench('deposit', async () => {
+      await env.waitForXDomainTransaction(
+        env.l1Bridge
+          .connect(l1Wallet)
+          .depositETH(DEFAULT_TEST_GAS_L2, '0xFFFF', {
+            value: 0x42,
+            gasLimit: DEFAULT_TEST_GAS_L1,
+          }),
+        Direction.L1ToL2
+      )
+    })
+    expect((await l2Wallet.getBalance()).sub(balBefore)).to.deep.equal(
+      BigNumber.from(0x42)
+    )
+  })
+})

integration-tests/actor-tests/lib/actor.ts

+import { Mutex } from 'async-mutex'
+import { sleep } from '../../test/shared/utils'
+import {
+  sanitizeForMetrics,
+  benchDurationsSummary,
+  successfulBenchRunsTotal,
+  failedActorRunsTotal,
+  successfulActorRunsTotal,
+  failedBenchRunsTotal,
+} from './metrics'
+import { ActorLogger, WorkerLogger } from './logger'
+import { performance } from 'perf_hooks'
+
+// eslint-disable-next-line @typescript-eslint/no-empty-function
+const asyncNoop = async () => {}
+
+export type AsyncCB = () => Promise<void>
+
+export interface Bencher {
+  bench: (name: string, cb: () => Promise<any>) => Promise<any>
+}
+
+export type RunCB<C> = (b: Bencher, ctx: C) => Promise<void>
+
+export interface RunOpts {
+  runs: number | null
+  runFor: number | null
+  concurrency: number
+  thinkTime: number
+}
+
+class Latch {
+  private n: number
+
+  private p: Promise<void>
+
+  private resolver: () => void
+
+  constructor(n: number) {
+    this.n = n
+    this.p = new Promise((resolve) => {
+      this.resolver = resolve
+    })
+  }
+
+  countDown() {
+    this.n--
+    if (this.n === 0) {
+      this.resolver()
+    }
+  }
+
+  wait() {
+    return this.p
+  }
+}
+
+export class Runner {
+  private readonly workerId: number
+
+  private readonly actor: Actor
+
+  private readonly mtx: Mutex
+
+  private readonly readyLatch: Latch
+
+  private readonly stepper: Bencher
+
+  private readonly logger: WorkerLogger
+
+  constructor(workerId: number, actor: Actor, mtx: Mutex, readyLatch: Latch) {
+    this.workerId = workerId
+    this.actor = actor
+    this.mtx = mtx
+    this.readyLatch = readyLatch
+    this.stepper = {
+      bench: this.bench,
+    }
+    this.logger = new WorkerLogger(this.actor.name, workerId)
+  }
+
+  bench = async (name: string, cb: () => Promise<void>) => {
+    const metricLabels = {
+      actor_name: sanitizeForMetrics(this.actor.name),
+      bench_name: sanitizeForMetrics(name),
+    }
+    const start = performance.now()
+    let res
+    try {
+      res = await cb()
+    } catch (e) {
+      failedBenchRunsTotal.inc({
+        ...metricLabels,
+        worker_id: this.workerId,
+      })
+      throw e
+    }
+    benchDurationsSummary.observe(metricLabels, performance.now() - start)
+    successfulBenchRunsTotal.inc({
+      ...metricLabels,
+      worker_id: this.workerId,
+    })
+    return res
+  }
+
+  async run(opts: RunOpts) {
+    const actor = this.actor
+
+    this.logger.log('Setting up.')
+    let ctx
+    try {
+      ctx = await this.mtx.runExclusive(this.actor.setupRun)
+    } finally {
+      this.readyLatch.countDown()
+    }
+    this.logger.log('Waiting for other workers to finish setup.')
+    await this.readyLatch.wait()
+
+    this.logger.log('Executing.')
+    const benchStart = performance.now()
+    let lastDurPrint = benchStart
+    let i = 0
+    const metricLabels = {
+      actor_name: sanitizeForMetrics(this.actor.name),
+    }
+
+    while (true) {
+      const now = performance.now()
+
+      if (
+        (opts.runs && i === opts.runs) ||
+        (opts.runFor && now - benchStart >= opts.runFor)
+      ) {
+        this.logger.log(`Worker exited.`)
+        break
+      }
+
+      try {
+        await this.actor.run(this.stepper, ctx)
+      } catch (e) {
+        console.error('Error in actor run:')
+        console.error(`Benchmark name: ${actor.name}`)
+        console.error(`Worker ID:      ${this.workerId}`)
+        console.error(`Run index:      ${i}`)
+        console.error('Stack trace:')
+        console.error(e)
+        failedActorRunsTotal.inc(metricLabels)
+        continue
+      }
+
+      successfulActorRunsTotal.inc(metricLabels)
+
+      i++
+
+      if (opts.runs && (i % 10 === 0 || i === opts.runs)) {
+        this.logger.log(`Completed run ${i} of ${opts.runs}.`)
+      }
+
+      if (opts.runFor && now - lastDurPrint > 10000) {
+        const runningFor = Math.floor(now - benchStart)
+        this.logger.log(`Running for ${runningFor} of ${opts.runFor} ms.`)
+        lastDurPrint = now
+      }
+
+      if (opts.thinkTime > 0) {
+        await sleep(opts.thinkTime)
+      }
+    }
+
+    await this.mtx.runExclusive(() => this.actor.tearDownRun(ctx))
+  }
+}
+
+export class Actor {
+  public readonly name: string
+
+  private _setupEnv: AsyncCB = asyncNoop
+
+  private _tearDownEnv: AsyncCB = asyncNoop
+
+  private _setupRun: <C>() => Promise<C> = asyncNoop as any
+
+  private _tearDownRun: <C>(ctx: C) => Promise<void> = asyncNoop as any
+
+  // eslint-disable-next-line @typescript-eslint/no-empty-function
+  private _run: <C>(b: Bencher, ctx: C) => Promise<void> = asyncNoop
+
+  private logger: ActorLogger
+
+  constructor(name: string) {
+    this.name = name
+    this.logger = new ActorLogger(this.name)
+  }
+
+  get setupEnv(): AsyncCB {
+    return this._setupEnv
+  }
+
+  set setupEnv(value: AsyncCB) {
+    this._setupEnv = value
+  }
+
+  get tearDownEnv(): AsyncCB {
+    return this._tearDownEnv
+  }
+
+  set tearDownEnv(value: AsyncCB) {
+    this._tearDownEnv = value
+  }
+
+  get setupRun(): <C>() => Promise<C> {
+    return this._setupRun
+  }
+
+  set setupRun(value: () => Promise<any>) {
+    this._setupRun = value
+  }
+
+  get tearDownRun(): <C>(ctx: C) => Promise<void> {
+    return this._tearDownRun
+  }
+
+  set tearDownRun(value: (ctx: any) => Promise<any>) {
+    this._tearDownRun = value
+  }
+
+  get run(): RunCB<any> {
+    return this._run
+  }
+
+  set run(cb: RunCB<any>) {
+    this._run = cb
+  }
+
+  async exec(opts: RunOpts) {
+    this.logger.log('Setting up.')
+
+    try {
+      await this.setupEnv()
+    } catch (e) {
+      console.error(`Error in setupEnv hook for actor "${this.name}":`)
+      console.error(e)
+      return
+    }
+
+    this.logger.log('Starting.')
+
+    const parallelRuns = []
+    const mtx = new Mutex()
+    const latch = new Latch(opts.concurrency)
+    for (let i = 0; i < opts.concurrency; i++) {
+      const runner = new Runner(i, this, mtx, latch)
+      parallelRuns.push(runner.run(opts))
+    }
+    await Promise.all(parallelRuns)
+
+    this.logger.log('Tearing down.')
+
+    try {
+      await this.tearDownEnv()
+    } catch (e) {
+      console.error(`Error in after hook for benchmark "${this.name}":`)
+      console.error(e)
+      return
+    }
+
+    this.logger.log('Teardown complete.')
+  }
+}
+
+export class Runtime {
+  private actors: Actor[] = []
+
+  addActor(actor: Actor) {
+    this.actors.push(actor)
+  }
+
+  async run(opts: Partial<RunOpts>) {
+    opts = {
+      runs: 1,
+      concurrency: 1,
+      runFor: null,
+      thinkTime: 0,
+      ...(opts || {}),
+    }
+    if (opts.runFor) {
+      opts.runs = null
+    }
+
+    for (const actor of this.actors) {
+      await actor.exec(opts as RunOpts)
+    }
+  }
+}

integration-tests/actor-tests/lib/convenience.ts

+import { AsyncCB, Actor, RunCB, Runtime } from './actor'
+
+export const defaultRuntime = new Runtime()
+
+let currBenchmark: Actor | null = null
+
+export const actor = (name: string, cb: () => void) => {
+  if (currBenchmark) {
+    throw new Error('Cannot call actor within actor.')
+  }
+
+  currBenchmark = new Actor(name)
+  cb()
+  defaultRuntime.addActor(currBenchmark)
+  currBenchmark = null
+}
+
+export const setupActor = (cb: AsyncCB) => {
+  if (!currBenchmark) {
+    throw new Error('Cannot call setupEnv outside of actor.')
+  }
+  currBenchmark.setupEnv = cb
+}
+
+export const setupRun = (cb: () => Promise<any>) => {
+  if (!currBenchmark) {
+    throw new Error('Cannot call setupRun outside of actor.')
+  }
+  currBenchmark.setupRun = cb
+}
+
+export const tearDownRun = (cb: AsyncCB) => {
+  if (!currBenchmark) {
+    throw new Error('Cannot call tearDownRun outside of actor.')
+  }
+  currBenchmark.tearDownRun = cb
+}
+
+export const run = (cb: RunCB<any>) => {
+  if (!currBenchmark) {
+    throw new Error('Cannot call run outside of actor.')
+  }
+  currBenchmark.run = cb
+}
+
+export const retryOnGasTooLow = async (cb: () => Promise<any>) => {
+  while (true) {
+    try {
+      return await cb()
+    } catch (e) {
+      if (e.toString().includes('gas price too low')) {
+        await new Promise((resolve) => setTimeout(resolve, 100))
+        continue
+      }
+
+      throw e
+    }
+  }
+}

integration-tests/actor-tests/lib/logger.ts

+import { sanitizeForMetrics } from './metrics'
+
+abstract class Logger {
+  log(msg: string) {
+    const date = new Date()
+    process.stderr.write(`[${date.toISOString()}] ${msg}\n`)
+  }
+}
+
+export class ActorLogger extends Logger {
+  private readonly name: string
+
+  constructor(name: string) {
+    super()
+    this.name = name
+  }
+
+  log(msg: string) {
+    super.log(`[actor:${sanitizeForMetrics(this.name)}] ${msg}`)
+  }
+}
+
+export class WorkerLogger extends Logger {
+  private readonly name: string
+
+  private readonly workerId: number
+
+  constructor(name: string, workerId: number) {
+    super()
+    this.name = name
+    this.workerId = workerId
+  }
+
+  log(msg: string) {
+    super.log(
+      `[bench:${sanitizeForMetrics(this.name)}] [wid:${this.workerId}] ${msg}`
+    )
+  }
+}

integration-tests/actor-tests/lib/metrics.ts

+import fs from 'fs'
+import client from 'prom-client'
+
+export const metricsRegistry = new client.Registry()
+
+const metricName = (name: string) => {
+  return `actor_${name}`
+}
+
+export const successfulBenchRunsTotal = new client.Counter({
+  name: metricName('successful_bench_runs_total'),
+  help: 'Count of total successful bench runs.',
+  labelNames: ['actor_name', 'bench_name', 'worker_id'] as const,
+  registers: [metricsRegistry],
+})
+
+export const failedBenchRunsTotal = new client.Counter({
+  name: metricName('failed_bench_runs_total'),
+  help: 'Count of total failed bench runs.',
+  labelNames: ['actor_name', 'bench_name', 'worker_id'] as const,
+  registers: [metricsRegistry],
+})
+
+export const benchDurationsSummary = new client.Summary({
+  name: metricName('step_durations_ms_summary'),
+  help: 'Summary of successful bench durations.',
+  percentiles: [0.5, 0.9, 0.95, 0.99],
+  labelNames: ['actor_name', 'bench_name'] as const,
+  registers: [metricsRegistry],
+})
+
+export const successfulActorRunsTotal = new client.Counter({
+  name: metricName('successful_actor_runs_total'),
+  help: 'Count of total successful actor runs.',
+  labelNames: ['actor_name'] as const,
+  registers: [metricsRegistry],
+})
+
+export const failedActorRunsTotal = new client.Counter({
+  name: metricName('failed_actor_runs_total'),
+  help: 'Count of total failed actor runs.',
+  labelNames: ['actor_name'] as const,
+  registers: [metricsRegistry],
+})
+
+export const sanitizeForMetrics = (input: string) => {
+  return input.toLowerCase().replace(/ /gi, '_')
+}
+
+export const dumpMetrics = async (filename: string) => {
+  const metrics = await metricsRegistry.metrics()
+  await fs.promises.writeFile(filename, metrics, {
+    flag: 'w+',
+  })
+}

integration-tests/actor-tests/lib/runner.ts

+import * as path from 'path'
+import { defaultRuntime } from './convenience'
+import { RunOpts } from './actor'
+import { Command } from 'commander'
+import pkg from '../../package.json'
+import { metricsRegistry } from './metrics'
+
+const program = new Command()
+program.version(pkg.version)
+program.name('actor-tests')
+
+program
+  .requiredOption('-f, --file <file>', 'test file to run')
+  .option('-r, --runs <n>', 'number of runs. cannot be use with -t/--time')
+  .option(
+    '-t, --time <ms>',
+    'how long to run in milliseconds. cannot be used with -r/--runs',
+  )
+  .option('-c, --concurrency <n>', 'number of concurrent workers to spawn', '1')
+  .option('--think-time <n>', 'how long to wait between each run', '0')
+
+program.parse(process.argv)
+
+const options = program.opts()
+const testFile = options.file
+const runsNum = Number(options.runs)
+const timeNum = Number(options.time)
+const concNum = Number(options.concurrency)
+const thinkNum = Number(options.thinkTime)
+
+if (isNaN(runsNum) && isNaN(timeNum)) {
+  console.error('Must define either a number of runs or how long to run.')
+  process.exit(1)
+}
+
+if (isNaN(concNum) || concNum <= 0) {
+  console.error('Invalid concurrency value.')
+  process.exit(1)
+}
+
+if (isNaN(thinkNum) || thinkNum < 0) {
+  console.error('Invalid think time value.')
+  process.exit(1)
+}
+
+try {
+  require(path.resolve(path.join(process.cwd(), testFile)))
+} catch (e) {
+  console.error(`Invalid test file ${testFile}:`)
+  console.error(e)
+  process.exit(1)
+}
+
+const opts: Partial<RunOpts> = {
+  runFor: timeNum,
+  concurrency: concNum,
+  thinkTime: thinkNum,
+  runs: runsNum,
+}
+
+defaultRuntime
+  .run(opts)
+  .then(() => metricsRegistry.metrics())
+  .then((metrics) => {
+    process.stderr.write('Run complete. Metrics:\n')
+    console.log(metrics)
+  })
+  .catch((err) => {
+    console.error('Error running:')
+    console.error(err)
+    process.exit(1)
+  })

integration-tests/actor-tests/nft.test.ts

+import { utils, Wallet, Contract, ContractFactory } from 'ethers'
+import { actor, run, setupActor, setupRun } from './lib/convenience'
+import { OptimismEnv } from '../test/shared/env'
+import ERC721 from '../artifacts/contracts/NFT.sol/NFT.json'
+import { expect } from 'chai'
+
+interface Context {
+  wallet: Wallet
+  contract: Contract
+}
+
+actor('NFT claimer', () => {
+  let env: OptimismEnv
+
+  let contract: Contract
+
+  setupActor(async () => {
+    env = await OptimismEnv.new()
+
+    const factory = new ContractFactory(
+      ERC721.abi,
+      ERC721.bytecode,
+      env.l2Wallet
+    )
+    contract = await factory.deploy()
+    await contract.deployed()
+  })
+
+  setupRun(async () => {
+    const wallet = Wallet.createRandom().connect(env.l2Wallet.provider)
+    await env.l2Wallet.sendTransaction({
+      to: wallet.address,
+      value: utils.parseEther('0.01'),
+    })
+    return {
+      wallet,
+      contract: contract.connect(wallet),
+    }
+  })
+
+  run(async (b, ctx: Context) => {
+    let receipt: any
+    await b.bench('mint', async () => {
+      const tx = await ctx.contract.give()
+      receipt = await tx.wait()
+    })
+    expect(receipt.events[0].event).to.equal('Transfer')
+    expect(receipt.events[0].args[1]).to.equal(ctx.wallet.address)
+  })
+})

integration-tests/actor-tests/sends.test.ts

+import { utils, Wallet, BigNumber } from 'ethers'
+import { expect } from 'chai'
+import { actor, setupRun, setupActor, run } from './lib/convenience'
+import { OptimismEnv } from '../test/shared/env'
+
+interface Context {
+  wallet: Wallet
+}
+
+actor('Value sender', () => {
+  let env: OptimismEnv
+
+  setupActor(async () => {
+    env = await OptimismEnv.new()
+  })
+
+  setupRun(async () => {
+    const wallet = Wallet.createRandom()
+    await env.l2Wallet.sendTransaction({
+      to: wallet.address,
+      value: utils.parseEther('0.01'),
+    })
+    return {
+      wallet: wallet.connect(env.l2Wallet.provider),
+    }
+  })
+
+  run(async (b, ctx: Context) => {
+    const randWallet = Wallet.createRandom().connect(env.l2Wallet.provider)
+    await b.bench('send funds', async () => {
+      await ctx.wallet.sendTransaction({
+        to: randWallet.address,
+        value: 0x42,
+      })
+    })
+    expect(await randWallet.getBalance()).to.deep.equal(BigNumber.from(0x42))
+  })
+})

integration-tests/actor-tests/state-dos.test.ts

+import { utils, Wallet, Contract, ContractFactory } from 'ethers'
+import { actor, setupActor, run, setupRun } from './lib/convenience'
+import { OptimismEnv } from '../test/shared/env'
+import StateDOS from '../artifacts/contracts/StateDOS.sol/StateDOS.json'
+import { expect } from 'chai'
+
+interface Context {
+  wallet: Wallet
+}
+
+actor('Trie DoS accounts', () => {
+  let env: OptimismEnv
+
+  let contract: Contract
+
+  setupActor(async () => {
+    env = await OptimismEnv.new()
+
+    const factory = new ContractFactory(
+      StateDOS.abi,
+      StateDOS.bytecode,
+      env.l2Wallet
+    )
+    contract = await factory.deploy()
+    await contract.deployed()
+  })
+
+  setupRun(async () => {
+    const wallet = Wallet.createRandom()
+    await env.l2Wallet.sendTransaction({
+      to: wallet.address,
+      value: utils.parseEther('1'),
+    })
+    return {
+      wallet: wallet.connect(env.l2Wallet.provider),
+    }
+  })
+
+  run(async (b, ctx: Context) => {
+    await b.bench('DOS transactions', async () => {
+      const tx = await contract.connect(ctx.wallet).attack({
+        gasLimit: 9000000 + Math.floor(1000000 * Math.random()),
+      })
+      const receipt = await tx.wait()
+      // make sure that this was an actual transaction in a block
+      expect(receipt.blockNumber).to.be.gt(1)
+      expect(receipt.gasUsed.gte(8000000)).to.be.true
+    })
+  })
+})

integration-tests/actor-tests/uniswap.test.ts

+import { BigNumber, Contract, utils, Wallet, ContractFactory } from 'ethers'
+import { actor, run, setupActor, setupRun } from './lib/convenience'
+import { OptimismEnv } from '../test/shared/env'
+import { UniswapV3Deployer } from 'uniswap-v3-deploy-plugin/dist/deployer/UniswapV3Deployer'
+import { FeeAmount, TICK_SPACINGS } from '@uniswap/v3-sdk'
+import ERC20 from '../artifacts/contracts/ERC20.sol/ERC20.json'
+
+interface Context {
+  contracts: { [name: string]: Contract }
+  wallet: Wallet
+}
+
+// Below methods taken from the Uniswap test suite, see
+// https://github.com/Uniswap/v3-periphery/blob/main/test/shared/ticks.ts
+export const getMinTick = (tickSpacing: number) =>
+  Math.ceil(-887272 / tickSpacing) * tickSpacing
+export const getMaxTick = (tickSpacing: number) =>
+  Math.floor(887272 / tickSpacing) * tickSpacing
+
+actor('Uniswap swapper', () => {
+  let env: OptimismEnv
+
+  let tokens: [Contract, Contract]
+
+  let contracts: { [name: string]: Contract }
+
+  setupActor(async () => {
+    env = await OptimismEnv.new()
+
+    const factory = new ContractFactory(ERC20.abi, ERC20.bytecode, env.l2Wallet)
+    const tokenA = await factory.deploy(1000000000, 'OVM1', 8, 'OVM1')
+    await tokenA.deployed()
+    const tokenB = await factory.deploy(1000000000, 'OVM2', 8, 'OVM2')
+    await tokenB.deployed()
+
+    tokens =
+      tokenA.address < tokenB.address ? [tokenA, tokenB] : [tokenB, tokenA]
+    contracts = await UniswapV3Deployer.deploy(env.l2Wallet)
+
+    let tx
+    for (const token of tokens) {
+      tx = await token.approve(contracts.positionManager.address, 1000000000)
+      await tx.wait()
+      tx = await token.approve(contracts.router.address, 1000000000)
+      await tx.wait()
+    }
+
+    tx = await contracts.positionManager.createAndInitializePoolIfNecessary(
+      tokens[0].address,
+      tokens[1].address,
+      FeeAmount.MEDIUM,
+      // initial ratio of 1/1
+      BigNumber.from('79228162514264337593543950336')
+    )
+    await tx.wait()
+
+    tx = await contracts.positionManager.mint(
+      {
+        token0: tokens[0].address,
+        token1: tokens[1].address,
+        tickLower: getMinTick(TICK_SPACINGS[FeeAmount.MEDIUM]),
+        tickUpper: getMaxTick(TICK_SPACINGS[FeeAmount.MEDIUM]),
+        fee: FeeAmount.MEDIUM,
+        recipient: env.l2Wallet.address,
+        amount0Desired: 100000000,
+        amount1Desired: 100000000,
+        amount0Min: 0,
+        amount1Min: 0,
+        deadline: Date.now() * 2,
+      },
+      {
+        gasLimit: 10000000,
+      }
+    )
+    await tx.wait()
+  })
+
+  setupRun(async () => {
+    const wallet = Wallet.createRandom().connect(env.l2Provider)
+
+    await env.l2Wallet.sendTransaction({
+      to: wallet.address,
+      value: utils.parseEther('0.1'),
+    })
+
+    for (const token of tokens) {
+      let tx = await token.transfer(wallet.address, 1000000)
+      await tx.wait()
+      const boundToken = token.connect(wallet)
+      tx = await boundToken.approve(contracts.positionManager.address, 1000000000)
+      await tx.wait()
+      tx = await boundToken.approve(contracts.router.address, 1000000000)
+      await tx.wait()
+    }
+
+    return {
+      contracts: Object.entries(contracts).reduce((acc, [name, value]) => {
+        acc[name] = value.connect(wallet)
+        return acc
+      }, {}),
+      wallet,
+    }
+  })
+
+  run(async (b, ctx: Context) => {
+    await b.bench('swap', async () => {
+      const tx = await ctx.contracts.router.exactInputSingle(
+        {
+          tokenIn: tokens[0].address,
+          tokenOut: tokens[1].address,
+          fee: FeeAmount.MEDIUM,
+          recipient: ctx.wallet.address,
+          deadline: Date.now() * 2,
+          amountIn: Math.max(Math.floor(1000 * Math.random()), 100),
+          amountOutMinimum: 0,
+          sqrtPriceLimitX96: 0,
+        },
+        {
+          gasLimit: 10000000,
+        }
+      )
+      await tx.wait()
+    })
+  })
+})

integration-tests/contracts/NFT.sol

+// SPDX-License-Identifier: MIT
+pragma solidity ^0.8.9;
+
+import "@openzeppelin/contracts/token/ERC721/ERC721.sol";
+import "@openzeppelin/contracts/utils/Counters.sol";
+
+contract NFT is ERC721 {
+    using Counters for Counters.Counter;
+    Counters.Counter private _tokenIds;
+
+    constructor() ERC721("OVM NFT", "ONFT") {
+    }
+
+    function give() public {
+        _tokenIds.increment();
+        uint256 newItemId = _tokenIds.current();
+        _mint(msg.sender, newItemId);
+    }
+}
\ No newline at end of file

integration-tests/contracts/StateDOS.sol

+// SPDX-License-Identifier: MIT
+pragma solidity ^0.8.9;
+
+contract StateDOS {
+    bool public hasRun = false;
+
+    function attack() public {
+        //  jumpdest     ; jump label, start of loop
+        // 	gas          ; get a 'random' value on the stack
+        // 	extcodesize  ; trigger trie lookup
+        // 	pop          ; ignore the extcodesize result
+        // 	push1 0x00   ; jump label dest
+        // 	jump         ; jump back to start
+
+        assembly {
+            let thegas := gas()
+
+        // While greater than 23000 gas. This will let us SSTORE at the end.
+            for { } gt(thegas, 0x59D8) { } {
+                thegas := gas()
+                let ignoredext := extcodesize(thegas)
+            }
+        }
+        hasRun = true; // Sanity check
+    }
+}
\ No newline at end of file

integration-tests/package.json

@@ -9,6 +9,7 @@
     "lint:check": "eslint .",
     "build": "hardhat compile",
     "test:integration": "hardhat --network optimism test",
+    "test:actor": "IS_LIVE_NETWORK=true ts-node actor-tests/lib/runner.ts",
     "test:integration:live": "NO_NETWORK=true IS_LIVE_NETWORK=true hardhat --network optimism test",
     "test:sync": "hardhat --network optimism test sync-tests/*.spec.ts --no-compile",
     "clean": "rimraf cache artifacts",
@@ -36,6 +37,7 @@
     "@ethersproject/transactions": "^5.4.0",
     "@nomiclabs/hardhat-ethers": "^2.0.2",
     "@nomiclabs/hardhat-waffle": "^2.0.1",
+    "@openzeppelin/contracts": "^4.4.0",
     "@types/chai": "^4.2.18",
     "@types/chai-as-promised": "^7.1.4",
     "@types/mocha": "^8.2.2",
@@ -46,9 +48,11 @@
     "@uniswap/v3-core": "1.0.0",
     "@uniswap/v3-periphery": "^1.0.1",
     "@uniswap/v3-sdk": "^3.6.2",
+    "async-mutex": "^0.3.2",
     "babel-eslint": "^10.1.0",
     "chai": "^4.3.4",
     "chai-as-promised": "^7.1.1",
+    "commander": "^8.3.0",
     "docker-compose": "^0.23.8",
     "dotenv": "^10.0.0",
     "envalid": "^7.1.0",
@@ -67,6 +71,7 @@
     "hardhat-gas-reporter": "^1.0.4",
     "lint-staged": "11.0.0",
     "mocha": "^8.4.0",
+    "prom-client": "^14.0.1",
     "rimraf": "^3.0.2",
     "shelljs": "^0.8.4",
     "typescript": "^4.3.5",

integration-tests/test/shared/env.ts

@@ -17,7 +17,6 @@ import {
   getOvmEth,
   getL1Bridge,
   getL2Bridge,
-  IS_LIVE_NETWORK,
   sleep,
 } from './utils'
 import {
@@ -211,34 +210,3 @@ export class OptimismEnv {
     }
   }
 }
-
-/**
- * Sets the timeout of a test based on the challenge period of the current network. If the
- * challenge period is greater than 60s (e.g., on Mainnet) then we skip this test entirely.
- *
- * @param testctx Function context of the test to modify (i.e. `this` when inside a test).
- * @param env Optimism environment used to resolve the StateCommitmentChain.
- */
-export const useDynamicTimeoutForWithdrawals = async (
-  testctx: any,
-  env: OptimismEnv
-) => {
-  if (!IS_LIVE_NETWORK) {
-    return
-  }
-
-  const challengePeriod = await env.scc.FRAUD_PROOF_WINDOW()
-  if (challengePeriod.gt(60)) {
-    console.log(
-      `WARNING: challenge period is greater than 60s (${challengePeriod.toString()}s), skipping test`
-    )
-    testctx.skip()
-  }
-
-  // 60s for state root batch to be published + (challenge period x 4)
-  const timeoutMs = 60000 + challengePeriod.toNumber() * 1000 * 4
-  console.log(
-    `NOTICE: inside a withdrawal test on a prod network, dynamically setting timeout to ${timeoutMs}ms`
-  )
-  testctx.timeout(timeoutMs)
-}

integration-tests/tsconfig.json

@@ -3,6 +3,12 @@
   "compilerOptions": {
       "resolveJsonModule": true
   },
-  "include": ["./test", "sync-tests/*.ts", "./artifacts/**/*.json"],
+  "include": [
+    "./test",
+    "sync-tests/*.ts",
+    "./actor-tests/**/*.ts",
+    "./artifacts/**/*.json",
+    "./package.json"
+  ],
   "files": ["./hardhat.config.ts"]
 }

yarn.lock

@@ -2404,6 +2404,11 @@
   resolved "https://registry.yarnpkg.com/@openzeppelin/contracts/-/contracts-4.3.3.tgz#ff6ee919fc2a1abaf72b22814bfb72ed129ec137"
   integrity sha512-tDBopO1c98Yk7Cv/PZlHqrvtVjlgK5R4J6jxLwoO7qxK4xqOiZG+zSkIvGFpPZ0ikc3QOED3plgdqjgNTnBc7g==
 
+"@openzeppelin/contracts@^4.4.0":
+  version "4.4.0"
+  resolved "https://registry.yarnpkg.com/@openzeppelin/contracts/-/contracts-4.4.0.tgz#4a1df71f736c31230bbbd634dfb006a756b51e6b"
+  integrity sha512-dlKiZmDvJnGRLHojrDoFZJmsQVeltVeoiRN7RK+cf2FmkhASDEblE0RiaYdxPNsUZa6mRG8393b9bfyp+V5IAw==
+
 "@resolver-engine/core@^0.3.3":
   version "0.3.3"
   resolved "https://registry.yarnpkg.com/@resolver-engine/core/-/core-0.3.3.tgz#590f77d85d45bc7ecc4e06c654f41345db6ca967"
@@ -3733,6 +3738,13 @@ async-limiter@~1.0.0:
   resolved "https://registry.yarnpkg.com/async-limiter/-/async-limiter-1.0.1.tgz#dd379e94f0db8310b08291f9d64c3209766617fd"
   integrity sha512-csOlWGAcRFJaI6m+F2WKdnMKr4HhdhFVBk0H/QbJFMCr+uO2kwohwXQPxw/9OCxp05r5ghVBFSyioixx3gfkNQ==
 
+async-mutex@^0.3.2:
+  version "0.3.2"
+  resolved "https://registry.yarnpkg.com/async-mutex/-/async-mutex-0.3.2.tgz#1485eda5bda1b0ec7c8df1ac2e815757ad1831df"
+  integrity sha512-HuTK7E7MT7jZEh1P9GtRW9+aTWiDWWi9InbZ5hjxrnRa39KS4BW04+xLBhYNS2aXhHUIKZSw3gj4Pn1pj+qGAA==
+  dependencies:
+    tslib "^2.3.1"
+
 async@1.x, async@^1.4.2:
   version "1.5.2"
   resolved "https://registry.yarnpkg.com/async/-/async-1.5.2.tgz#ec6a61ae56480c0c3cb241c95618e20892f9672a"
@@ -5333,6 +5345,11 @@ commander@^7.2.0:
   resolved "https://registry.yarnpkg.com/commander/-/commander-7.2.0.tgz#a36cb57d0b501ce108e4d20559a150a391d97ab7"
   integrity sha512-QrWXB+ZQSVPmIWIhtEO9H+gwHaMGYiF5ChvoJ+K9ZGHG/sVsa6yiesAD1GC/x46sET00Xlwo1u49RVVVzvcSkw==
 
+commander@^8.3.0:
+  version "8.3.0"
+  resolved "https://registry.yarnpkg.com/commander/-/commander-8.3.0.tgz#4837ea1b2da67b9c616a67afbb0fafee567bca66"
+  integrity sha512-OkTL9umf+He2DZkUq8f8J9of7yL6RJKI24dVITBmNfZBmri9zYZQrKkuXiKhyfPSu8tUhnVBB1iKXevvnlR4Ww==
+
 comment-parser@1.1.6-beta.0:
   version "1.1.6-beta.0"
   resolved "https://registry.yarnpkg.com/comment-parser/-/comment-parser-1.1.6-beta.0.tgz#57e503b18d0a5bd008632dcc54b1f95c2fffe8f6"
@@ -12571,6 +12588,13 @@ prom-client@^13.1.0:
   dependencies:
     tdigest "^0.1.1"
 
+prom-client@^14.0.1:
+  version "14.0.1"
+  resolved "https://registry.yarnpkg.com/prom-client/-/prom-client-14.0.1.tgz#bdd9583e02ec95429677c0e013712d42ef1f86a8"
+  integrity sha512-HxTArb6fkOntQHoRGvv4qd/BkorjliiuO2uSWC2KC17MUTKYttWdDoXX/vxOhQdkoECEM9BBH0pj2l8G8kev6w==
+  dependencies:
+    tdigest "^0.1.1"
+
 promise-inflight@^1.0.1:
   version "1.0.1"
   resolved "https://registry.yarnpkg.com/promise-inflight/-/promise-inflight-1.0.1.tgz#98472870bf228132fcbdd868129bad12c3c029e3"
@@ -15032,7 +15056,7 @@ tsconfig-paths@^3.10.1, tsconfig-paths@^3.5.0:
     minimist "^1.2.0"
     strip-bom "^3.0.0"
 
-tslib@2.3.1:
+tslib@2.3.1, tslib@^2.3.1:
   version "2.3.1"
   resolved "https://registry.yarnpkg.com/tslib/-/tslib-2.3.1.tgz#e8a335add5ceae51aa261d32a490158ef042ef01"
   integrity sha512-77EbyPPpMz+FRFRuAFlWMtmgUWGe9UOG2Z25NqCwiIjRhOf5iKGuzSe5P2w1laq+FkRy4p+PCuVkJSGkzTEKVw==