Started porting tests

packages/contracts/contracts/optimistic-ethereum/OVM/execution/OVM_ExecutionManager.sol

@@ -14,6 +14,9 @@ import { iOVM_SafetyChecker } from "../../iOVM/execution/iOVM_SafetyChecker.sol"
 /* Contract Imports */
 import { OVM_ECDSAContractAccount } from "../accounts/OVM_ECDSAContractAccount.sol";
 
+/* Logging */
+import { console } from "@nomiclabs/buidler/console.sol";
+
 /**
  * @title OVM_ExecutionManager
  */
@@ -724,7 +727,7 @@ contract OVM_ExecutionManager is iOVM_ExecutionManager {
         address ethAddress = Lib_EthUtils.createContract(_bytecode);
 
         // Now reset this flag so we go back to normal revert behavior.
-        messageContext.isCreation = true;
+        messageContext.isCreation = false;
 
         // Contract creation returns the zero address when it fails, which should only be possible
         // if the user intentionally runs out of gas. However, we might still have a bit of gas
@@ -1252,6 +1255,16 @@ contract OVM_ExecutionManager is iOVM_ExecutionManager {
             return bytes('');
         }
 
+        // INVALID_STATE_ACCESS doesn't need to return any data other than the flag.
+        if (_flag == RevertFlag.INVALID_STATE_ACCESS) {
+            return abi.encode(
+                _flag,
+                0,
+                0,
+                bytes('')
+            );
+        }
+
         // Just ABI encode the rest of the parameters.
         return abi.encode(
             _flag,

packages/contracts/contracts/test-helpers/Helper_CodeContracts.sol

-pragma solidity >=0.7.0;
-pragma experimental ABIEncoderV2;
-
-import {console} from "@nomiclabs/buidler/console.sol";
-
-interface Helper_CodeContractDataTypes {
-    struct CALLResponse {
-        bool success;
-        bytes data;
-    }
-}
-
-
-contract Helper_CodeContractForCalls is Helper_CodeContractDataTypes {
-    bytes constant sampleCREATEData = abi.encodeWithSignature("ovmCREATE(bytes)", hex"");
-    bytes constant sampleCREATE2Data = abi.encodeWithSignature("ovmCREATE2(bytes,bytes32)", hex"", 0x4242424242424242424242424242424242424242424242424242424242424242);
-
-    function runSteps(
-        bytes[] memory callsToEM,
-        bool _shouldRevert,
-        address _createEMResponsesStorer
-    ) public returns(CALLResponse[] memory) {
-        console.log("in runSteps()");
-        uint numSteps = callsToEM.length;
-        CALLResponse[] memory EMResponses = new CALLResponse[](numSteps);
-        for (uint i = 0; i < numSteps; i++) {
-            bytes memory dataToSend = callsToEM[i];
-            console.log("calling EM with data:");
-            console.logBytes(dataToSend);
-            (bool success, bytes memory responseData) = address(msg.sender).call(dataToSend);
-            console.log("step to EM had result:");
-            console.logBool(success);
-            EMResponses[i].success = success;
-            if (_isOVMCreateCall(dataToSend)) {
-                console.log("step to EM returned data:");
-                console.logBytes(responseData);
-                EMResponses[i].data = abi.encode(
-                    responseData,
-                    _getStoredEMREsponsesInCreate(_createEMResponsesStorer)
-                );
-                console.log("since this is create step, stored concatenation:");
-                console.logBytes(EMResponses[i].data);
-            } else {
-                console.log("step to EM returned data:");
-                console.logBytes(responseData);
-                EMResponses[i].data = responseData;
-            }
-        }
-        return EMResponses;
-    }
-
-    function _getStoredEMREsponsesInCreate(address _createEMResponsesStorer) internal returns(bytes memory) {
-        (bool success, bytes memory data) = _createEMResponsesStorer.call(abi.encodeWithSignature("getLastResponses()"));
-        return data;
-    }
-
-    function _isOVMCreateCall(bytes memory _calldata) public returns(bool) {
-        return (
-            _doMethodIdsMatch(_calldata, sampleCREATEData) || _doMethodIdsMatch(_calldata, sampleCREATE2Data)
-        );
-    }
-
-    function _doMethodIdsMatch(bytes memory _calldata1, bytes memory _calldata2) internal returns(bool) {
-        return (
-            _calldata1[0] == _calldata2[0] &&
-            _calldata1[1] == _calldata2[1] &&
-            _calldata1[2] == _calldata2[2] &&
-            _calldata1[3] == _calldata2[3]            
-        );
-    }
-}
-
-contract Helper_CodeContractForCreates is Helper_CodeContractForCalls {
-    constructor(
-        bytes[] memory callsToEM,
-        bool _shouldRevert,
-        bytes memory _codeToDeploy,
-        address _createEMResponsesStorer
-    ) {  
-        console.log("In CREATE helper (deployment)");
-        CALLResponse[] memory responses = runSteps(callsToEM, _shouldRevert, _createEMResponsesStorer);
-        Helper_CreateEMResponsesStorer(_createEMResponsesStorer).store(responses);
-        uint lengthToDeploy = _codeToDeploy.length;
-        // todo  revert if _shouldrevert
-        assembly {
-            return(add(_codeToDeploy, 0x20), lengthToDeploy)
-        }
-    }
-}
-
-contract Helper_CreateEMResponsesStorer is Helper_CodeContractDataTypes {
-    CALLResponse[] responses;
-
-    function store(
-        CALLResponse[] memory _responses
-    ) public {
-        console.log("create storer helper is storing responses...");
-        for (uint i = 0; i < _responses.length; i++) {
-            responses.push();
-            responses[i] = _responses[i];
-        }
-        console.log("helper successfully stored this many responses:");
-        console.logUint(responses.length);
-    }
-
-    function getLastResponses() public returns(CALLResponse[] memory) {
-        console.log("helper is retreiving last stored responses.  It has this many responses stored:");
-        console.logUint(responses.length);
-
-        CALLResponse[] memory toReturn = responses;
-        delete responses;
-        return toReturn;
-    }
-}
-
-contract Helper_CodeContractForReverts {
-    function doRevert(
-        bytes memory _revertdata
-    ) public {
-        uint revertLength = _revertdata.length;
-        assembly {
-            revert(add(_revertdata, 0x20), revertLength)
-        }
-    }
-}
-
-// note: behavior of this contract covers all 3 cases of EVM message exceptions:
-// - out of gas
-// - INVALID opcode
-// - invalid JUMPDEST
-contract Helper_CodeContractForInvalid {
-    function doInvalid() public {
-        assembly {
-            invalid()
-        }
-    }
-}
-
-// note: behavior of this contract covers all 3 cases of EVM message exceptions:
-// - out of gas
-// - INVALID opcode
-// - invalid JUMPDEST
-contract Helper_CodeContractForInvalidInCreation {
-    constructor() public {
-        assembly {
-            invalid()
-        }
-    }
-}

packages/contracts/contracts/test-helpers/Helper_TestRunner.sol

+// SPDX-License-Identifier: UNLICENSED
+pragma solidity ^0.7.0;
+pragma experimental ABIEncoderV2;
+
+/* Logging */
+import { console } from "@nomiclabs/buidler/console.sol";
+
+/**
+ * @title Helper_TestRunner
+ */
+contract Helper_TestRunner {
+    struct TestStep {
+        string functionName;
+        bytes functionData;
+        bool expectedReturnStatus;
+        bytes expectedReturnData;
+    }
+
+    function runSingleTestStep(
+        TestStep memory _step
+    )
+        public
+    {
+        bytes32 namehash = keccak256(abi.encodePacked(_step.functionName));
+        if (namehash == keccak256("evmRETURN")) {
+            bytes memory returndata = _step.functionData;
+            assembly {
+                return(add(returndata, 0x20), mload(returndata))
+            }
+        }
+        if (namehash == keccak256("evmREVERT")) {
+            bytes memory returndata = _step.functionData;
+            assembly {
+                revert(add(returndata, 0x20), mload(returndata))
+            }
+        }
+        if (namehash == keccak256("evmINVALID")) {
+            assembly {
+                invalid()
+            }
+        }
+
+        (bool success, bytes memory returndata) = address(msg.sender).call(_step.functionData);
+
+        if (success != _step.expectedReturnStatus) {
+            if (success == true) {
+                console.log("ERROR: Expected function to revert, but function returned successfully");
+                console.log("Offending Step: %s", _step.functionName);
+                console.log("Return Data:");
+                console.logBytes(returndata);
+                console.log("");
+            } else {
+                (
+                    uint256 _flag,
+                    uint256 _nuisanceGasLeft,
+                    uint256 _ovmGasRefund,
+                    bytes memory _data
+                ) = _decodeRevertData(returndata);
+
+                console.log("ERROR: Expected function to return successfully, but function reverted");
+                console.log("Offending Step: %s", _step.functionName);
+                console.log("Flag: %s", _flag);
+                console.log("Nuisance Gas Left: %s", _nuisanceGasLeft);
+                console.log("OVM Gas Refund: %s", _ovmGasRefund);
+                console.log("Extra Data:");
+                console.logBytes(_data);
+                console.log("");
+            }
+
+            revert("Test step failed.");
+        }
+
+        if (keccak256(returndata) != keccak256(_step.expectedReturnData)) {
+            if (success == true) {
+                console.log("ERROR: Actual return data does not match expected return data");
+                console.log("Offending Step: %s", _step.functionName);
+                console.log("Expected:");
+                console.logBytes(_step.expectedReturnData);
+                console.log("Actual:");
+                console.logBytes(returndata);
+                console.log("");
+            } else {
+                (
+                    uint256 _expectedFlag,
+                    uint256 _expectedNuisanceGasLeft,
+                    uint256 _expectedOvmGasRefund,
+                    bytes memory _expectedData
+                ) = _decodeRevertData(_step.expectedReturnData);
+
+                (
+                    uint256 _flag,
+                    uint256 _nuisanceGasLeft,
+                    uint256 _ovmGasRefund,
+                    bytes memory _data
+                ) = _decodeRevertData(returndata);
+
+                console.log("ERROR: Actual revert flag data does not match expected revert flag data");
+                console.log("Offending Step: %s", _step.functionName);
+                console.log("Expected Flag: %s", _expectedFlag);
+                console.log("Actual Flag: %s", _flag);
+                console.log("Expected Nuisance Gas Left: %s", _expectedNuisanceGasLeft);
+                console.log("Actual Nuisance Gas Left: %s", _nuisanceGasLeft);
+                console.log("Expected OVM Gas Refund: %s", _expectedOvmGasRefund);
+                console.log("Actual OVM Gas Refund: %s", _ovmGasRefund);
+                console.log("Expected Extra Data:");
+                console.logBytes(_expectedData);
+                console.log("Actual Extra Data:");
+                console.logBytes(_data);
+                console.log("");
+            }
+
+            revert("Test step failed.");
+        }
+
+        if (success == false || (success == true && returndata.length == 1)) {
+            assembly {
+                if eq(extcodesize(address()), 0) {
+                    return(0, 1)
+                }
+
+                revert(add(returndata, 0x20), mload(returndata))
+            }
+        }
+    }
+
+    function runMultipleTestSteps(
+        TestStep[] memory _steps
+    )
+        public
+    {
+        for (uint256 i = 0; i < _steps.length; i++) {
+            runSingleTestStep(_steps[i]);
+        }
+    }
+
+    function _decodeRevertData(
+        bytes memory _revertdata
+    )
+        internal
+        returns (
+            uint256 _flag,
+            uint256 _nuisanceGasLeft,
+            uint256 _ovmGasRefund,
+            bytes memory _data
+        )
+    {
+        if (_revertdata.length == 0) {
+            return (
+                0,
+                0,
+                0,
+                bytes('')
+            );
+        }
+
+        return abi.decode(_revertdata, (uint256, uint256, uint256, bytes));
+    }
+}
+
+contract Helper_TestRunner_CREATE is Helper_TestRunner {
+    constructor(
+        bytes memory _bytecode,
+        TestStep[] memory _steps
+    ) {
+        if (_steps.length > 0) {
+            runMultipleTestSteps(_steps);
+        } else {
+            assembly {
+                return(add(_bytecode, 0x20), mload(_bytecode))
+            }
+        }
+    }
+}

packages/contracts/package.json

@@ -7,7 +7,7 @@
     "build": "yarn run build:contracts",
     "build:contracts": "buidler compile",
     "test": "yarn run test:contracts",
-    "test:contracts": "buidler test \"test/contracts/OVM/execution/OVM_ExecutionManager/ovmSTATICCALL.spec.ts\"",
+    "test:contracts": "buidler test \"test/contracts/OVM/execution/OVM_ExecutionManager/ovmCREATE.spec.ts\"",
     "lint": "tslint --format stylish --project .",
     "fix": "prettier --config prettier-config.json --write \"buidler.config.ts\" \"{src,test}/**/*.ts\""
   },

packages/contracts/test/contracts/OVM/execution/OVM_test.spec.ts

+import { ExecutionManagerTestRunner } from '../../../helpers/test-utils/test-runner'
+import { TestDefinition } from '../../../helpers/test-utils/test.types2'
+import {
+  GAS_LIMIT,
+  NULL_BYTES32,
+  NON_NULL_BYTES32,
+  REVERT_FLAGS,
+  DUMMY_BYTECODE,
+  VERIFIED_EMPTY_CONTRACT_HASH,
+} from '../../../helpers'
+
+const CREATED_CONTRACT_1 = '0x2bda4a99d5be88609d23b1e4ab5d1d34fb1c2feb'
+const DUMMY_REVERT_DATA =
+  '0xdeadbeef1e5420deadbeef1e5420deadbeef1e5420deadbeef1e5420deadbeef1e5420'
+
+const test: TestDefinition = {
+  name: 'An Example Test',
+  preState: {
+    ExecutionManager: {
+      ovmStateManager: '$OVM_STATE_MANAGER',
+      ovmSafetyChecker: '$OVM_SAFETY_CHECKER',
+      messageRecord: {
+        nuisanceGasLeft: GAS_LIMIT,
+      },
+    },
+    StateManager: {
+      owner: '$OVM_EXECUTION_MANAGER',
+      accounts: {
+        $DUMMY_OVM_ADDRESS_1: {
+          codeHash: NON_NULL_BYTES32,
+          ethAddress: '$OVM_CALL_HELPER',
+        },
+        $DUMMY_OVM_ADDRESS_2: {
+          codeHash: NON_NULL_BYTES32,
+          ethAddress: '$OVM_CALL_HELPER',
+        },
+        [CREATED_CONTRACT_1]: {
+          codeHash: VERIFIED_EMPTY_CONTRACT_HASH,
+        },
+      },
+    },
+  },
+  subTests: [
+    {
+      name: 'An Example Subtest',
+      parameters: [
+        {
+          name: 'An Example CALL revert test',
+          steps: [
+            {
+              functionName: 'ovmCALL',
+              functionParams: {
+                gasLimit: GAS_LIMIT,
+                target: '$DUMMY_OVM_ADDRESS_1',
+                subSteps: [
+                  {
+                    functionName: 'ovmCALL',
+                    functionParams: {
+                      gasLimit: GAS_LIMIT,
+                      target: '$DUMMY_OVM_ADDRESS_2',
+                      subSteps: [
+                        {
+                          functionName: 'evmREVERT',
+                          returnData: {
+                            flag: REVERT_FLAGS.INTENTIONAL_REVERT,
+                            data: DUMMY_REVERT_DATA,
+                          },
+                        },
+                      ],
+                    },
+                    expectedReturnStatus: false,
+                    expectedReturnValue: DUMMY_REVERT_DATA,
+                  },
+                ],
+              },
+              expectedReturnStatus: true,
+            },
+          ],
+        },
+        {
+          name: 'An Example CREATE test',
+          steps: [
+            {
+              functionName: 'ovmCALL',
+              functionParams: {
+                gasLimit: GAS_LIMIT,
+                target: '$DUMMY_OVM_ADDRESS_1',
+                subSteps: [
+                  {
+                    functionName: 'ovmCREATE',
+                    functionParams: {
+                      subSteps: [
+                        {
+                          functionName: 'ovmCALLER',
+                          expectedReturnStatus: true,
+                          expectedReturnValue: '$DUMMY_OVM_ADDRESS_1',
+                        },
+                      ],
+                    },
+                    expectedReturnStatus: true,
+                    expectedReturnValue: CREATED_CONTRACT_1,
+                  },
+                ],
+              },
+              expectedReturnStatus: true,
+            },
+          ],
+        },
+        {
+          name: 'An Example CALL test',
+          steps: [
+            {
+              functionName: 'ovmCALL',
+              functionParams: {
+                gasLimit: GAS_LIMIT,
+                target: '$DUMMY_OVM_ADDRESS_1',
+                subSteps: [
+                  {
+                    functionName: 'ovmADDRESS',
+                    expectedReturnStatus: true,
+                    expectedReturnValue: '$DUMMY_OVM_ADDRESS_1',
+                  },
+                  {
+                    functionName: 'ovmCALL',
+                    functionParams: {
+                      gasLimit: GAS_LIMIT,
+                      target: '$DUMMY_OVM_ADDRESS_2',
+                      subSteps: [
+                        {
+                          functionName: 'ovmADDRESS',
+                          expectedReturnStatus: true,
+                          expectedReturnValue: '$DUMMY_OVM_ADDRESS_2',
+                        },
+                        {
+                          functionName: 'ovmCALLER',
+                          expectedReturnStatus: true,
+                          expectedReturnValue: '$DUMMY_OVM_ADDRESS_1',
+                        },
+                      ],
+                    },
+                    expectedReturnStatus: true,
+                  },
+                ],
+              },
+              expectedReturnStatus: true,
+            },
+          ],
+        },
+      ],
+    },
+  ],
+}
+
+const runner = new ExecutionManagerTestRunner()
+runner.run(test)

packages/contracts/test/helpers/constants.ts

@@ -20,4 +20,5 @@ export const NON_NULL_BYTES32 = makeHexString('11', 32)
 export const ZERO_ADDRESS = makeAddress('00')
 export const NON_ZERO_ADDRESS = makeAddress('11')
 
-export const VERIFIED_EMPTY_CONTRACT_HASH = makeHexString('69', 32)
+export const VERIFIED_EMPTY_CONTRACT_HASH =
+  '0x00004B1DC0DE000000004B1DC0DE000000004B1DC0DE000000004B1DC0DE0000'

packages/contracts/test/helpers/mocks/mock-binding.ts

@@ -93,7 +93,7 @@ export const bindMockWatcherToVM = (): void => {
 
   // Modify the post-message handler to insert the correct return data.
   const originalAfterMessageHandler = vmTracer['_afterMessageHandler' as any]
-  function modifiedAfterMessageHandler(result: any, next: any) {
+  async function modifiedAfterMessageHandler(result: any, next: any) {
     // We don't need to do anything if we haven't stored any mock messages.
     if (messages.length > 0) {
       // We need to look at the messages backwards since the first result will

packages/contracts/test/helpers/test-utils/index.ts

-export * from './test-generation'
-export * from './test-parsing'
+export * from './test-runner'
 export * from './test.types'

packages/contracts/test/helpers/test-utils/test-parsing.ts

-import { expect } from '../../setup'
-
-/* External Imports */
-import { Contract } from 'ethers'
-import { cloneDeep } from 'lodash'
-import { ethers } from '@nomiclabs/buidler'
-
-/* Internal Imports */
-import { getModifiableStorageFactory } from '../storage/contract-storage'
-import { GAS_LIMIT, NON_NULL_BYTES32 } from '../constants'
-import { getTestGenerator, getInitcode, getBytecode } from './test-generation'
-import { TestParameters, TestDefinition, isTestDefinition } from './test.types'
-import { int } from '@nomiclabs/buidler/internal/core/params/argumentTypes'
-
-const getDummyOVMAddress = (kv: string): string => {
-  return '0x' + (kv.split('$DUMMY_OVM_ADDRESS_')[1] + '0').repeat(20)
-}
-
-const setPlaceholderStrings = (
-  test: any,
-  ovmExecutionManager: Contract,
-  ovmStateManager: Contract,
-  ovmSafetyChecker: Contract,
-  ovmCallHelper: Contract,
-  ovmRevertHelper: Contract
-): any => {
-  const setPlaceholder = (kv: string): string => {
-    if (kv === '$OVM_EXECUTION_MANAGER') {
-      return ovmExecutionManager.address
-    } else if (kv === '$OVM_STATE_MANAGER') {
-      return ovmStateManager.address
-    } else if (kv === '$OVM_SAFETY_CHECKER') {
-      return ovmSafetyChecker.address
-    } else if (kv === '$OVM_CALL_HELPER_CODE') {
-      return getBytecode('Helper_CodeContractForCalls')
-    } else if (kv === '$OVM_CALL_HELPER') {
-      return ovmCallHelper.address
-    } else if (kv === '$OVM_REVERT_HELPER') {
-      return ovmRevertHelper.address
-    } else if (kv.startsWith('$DUMMY_OVM_ADDRESS_')) {
-      return getDummyOVMAddress(kv)
-    } else {
-      return kv
-    }
-  }
-
-  if (Array.isArray(test)) {
-    test = test.map((element) => {
-      return setPlaceholderStrings(
-        element,
-        ovmExecutionManager,
-        ovmStateManager,
-        ovmSafetyChecker,
-        ovmCallHelper,
-        ovmRevertHelper
-      )
-    })
-  } else if (typeof test === 'object' && test !== null) {
-    for (const key of Object.keys(test)) {
-      const replacedKey = setPlaceholder(key)
-
-      if (replacedKey !== key) {
-        test[replacedKey] = test[key]
-        delete test[key]
-      }
-
-      test[replacedKey] = setPlaceholderStrings(
-        test[replacedKey],
-        ovmExecutionManager,
-        ovmStateManager,
-        ovmSafetyChecker,
-        ovmCallHelper,
-        ovmRevertHelper
-      )
-    }
-  } else if (typeof test === 'string') {
-    test = setPlaceholder(test)
-  }
-
-  return test
-}
-
-const fixtureDeployContracts = async (): Promise<{
-  OVM_SafetyChecker: Contract
-  OVM_StateManager: Contract
-  OVM_ExecutionManager: Contract
-  OVM_CallHelper: Contract
-  OVM_RevertHelper: Contract
-  OVM_CreateStorer: Contract
-  OVM_InvalidHelper: Contract
-}> => {
-  const Factory__OVM_SafetyChecker = await ethers.getContractFactory(
-    'OVM_SafetyChecker'
-  )
-  const Factory__OVM_StateManager = await getModifiableStorageFactory(
-    'OVM_StateManager'
-  )
-  const Factory__OVM_ExecutionManager = await getModifiableStorageFactory(
-    'OVM_ExecutionManager'
-  )
-  const Factory__Helper_CodeContractForCalls = await getModifiableStorageFactory(
-    'Helper_CodeContractForCalls'
-  )
-  const Factory__Helper_CodeContractForReverts = await ethers.getContractFactory(
-    'Helper_CodeContractForReverts'
-  )
-  const Factory__Helper_CreateEMResponsesStorer = await ethers.getContractFactory(
-    'Helper_CreateEMResponsesStorer'
-  )
-
-  const Helper_CodeContractForInvalid = await ethers.getContractFactory(
-    'Helper_CodeContractForInvalid'
-  )
-
-  const OVM_SafetyChecker = await Factory__OVM_SafetyChecker.deploy()
-  const OVM_ExecutionManager = await Factory__OVM_ExecutionManager.deploy(
-    OVM_SafetyChecker.address
-  )
-  const OVM_StateManager = await Factory__OVM_StateManager.deploy(
-    OVM_ExecutionManager.address
-  )
-  const OVM_CallHelper = await Factory__Helper_CodeContractForCalls.deploy()
-  const OVM_RevertHelper = await Factory__Helper_CodeContractForReverts.deploy()
-  const OVM_CreateStorer = await Factory__Helper_CreateEMResponsesStorer.deploy()
-  const OVM_InvalidHelper = await Helper_CodeContractForInvalid.deploy()
-
-  return {
-    OVM_SafetyChecker,
-    OVM_StateManager,
-    OVM_ExecutionManager,
-    OVM_CallHelper,
-    OVM_RevertHelper,
-    OVM_CreateStorer,
-    OVM_InvalidHelper,
-  }
-}
-
-export const runExecutionManagerTest = (test: TestDefinition): void => {
-  test.preState = test.preState || {}
-  test.postState = test.postState || {}
-
-  describe(`Standard test: ${test.name}`, () => {
-    test.parameters.map((parameters) => {
-      if (isTestDefinition(parameters)) {
-        runExecutionManagerTest({
-          ...parameters,
-          preState: {
-            ...test.preState,
-            ...parameters.preState,
-          },
-          postState: {
-            ...test.postState,
-            ...parameters.postState,
-          },
-        })
-      } else {
-        let OVM_SafetyChecker: Contract
-        let OVM_StateManager: Contract
-        let OVM_ExecutionManager: Contract
-        let OVM_CallHelper: Contract
-        let OVM_RevertHelper: Contract
-        let OVM_CreateStorer: Contract
-        let OVM_InvalidHelper: Contract
-        beforeEach(async () => {
-          const contracts = await fixtureDeployContracts()
-          OVM_SafetyChecker = contracts.OVM_SafetyChecker
-          OVM_StateManager = contracts.OVM_StateManager
-          OVM_ExecutionManager = contracts.OVM_ExecutionManager
-          OVM_CallHelper = contracts.OVM_CallHelper
-          OVM_RevertHelper = contracts.OVM_RevertHelper
-          OVM_CreateStorer = contracts.OVM_CreateStorer
-          OVM_InvalidHelper = contracts.OVM_InvalidHelper
-        })
-
-        let replacedParams: TestParameters
-        let replacedTest: TestDefinition
-        beforeEach(async () => {
-          replacedParams = setPlaceholderStrings(
-            cloneDeep(parameters),
-            OVM_ExecutionManager,
-            OVM_StateManager,
-            OVM_SafetyChecker,
-            OVM_CallHelper,
-            OVM_RevertHelper
-          )
-          replacedTest = setPlaceholderStrings(
-            cloneDeep(test),
-            OVM_ExecutionManager,
-            OVM_StateManager,
-            OVM_SafetyChecker,
-            OVM_CallHelper,
-            OVM_RevertHelper
-          )
-        })
-
-        beforeEach(async () => {
-          await OVM_ExecutionManager.__setContractStorage(
-            replacedTest.preState.ExecutionManager
-          )
-          await OVM_StateManager.__setContractStorage(
-            replacedTest.preState.StateManager
-          )
-        })
-
-        afterEach(async () => {
-          await OVM_ExecutionManager.__checkContractStorage({
-            ...replacedTest.postState.ExecutionManager,
-          })
-          await OVM_StateManager.__checkContractStorage({
-            ...replacedTest.postState.StateManager,
-          })
-        })
-
-        parameters.steps.map((step, idx) => {
-          const scopedFunction = !!test.focus ? it.only : it
-          scopedFunction(`should run test: ${test.name} ${idx}`, async () => {
-            const testGenerator = getTestGenerator(
-              replacedParams.steps[idx],
-              OVM_ExecutionManager,
-              OVM_CallHelper,
-              OVM_CreateStorer,
-              (await ethers.getContractFactory('Helper_CodeContractForCreates'))
-                .interface,
-              OVM_RevertHelper,
-              OVM_InvalidHelper
-            )
-
-            const callResult = await OVM_ExecutionManager.provider.call({
-              to: OVM_ExecutionManager.address,
-              data: testGenerator.getCalldata(),
-              gasLimit: GAS_LIMIT,
-            })
-
-            await OVM_ExecutionManager.signer.sendTransaction({
-              to: OVM_ExecutionManager.address,
-              data: testGenerator.getCalldata(),
-              gasLimit: GAS_LIMIT,
-            })
-
-            const interpretation = testGenerator.interpretActualReturnData(
-              callResult,
-              true
-            )
-            console.log('interpretation of actual results:\n' + interpretation) // in future we can add conditional here but for now always assume succeed
-
-            const interpretationOfExpected = testGenerator.interpretActualReturnData(
-              testGenerator.getReturnData(),
-              true
-            )
-            console.log(
-              'interpretation of expected: \n' + interpretationOfExpected
-            )
-
-            expect(callResult).to.equal(testGenerator.getReturnData()) //, 'got bad response, looks like it did:\n' + testGenerator.interpretActualReturnData(callResult))
-          })
-        })
-      }
-    })
-  })
-}

packages/contracts/test/helpers/test-utils/test-runner.ts

+import { expect } from '../../setup'
+
+/* External Imports */
+import { ethers } from '@nomiclabs/buidler'
+import { Contract, BigNumber, ContractFactory } from 'ethers'
+import { cloneDeep } from 'lodash'
+
+/* Internal Imports */
+import {
+  TestDefinition,
+  TestStep,
+  isTestStep_SSTORE,
+  isTestStep_SLOAD,
+  isTestStep_CALL,
+  isTestStep_CREATE,
+  isTestStep_CREATE2,
+  isTestStep_Context,
+  ParsedTestStep,
+  isRevertFlagError,
+  TestParameter,
+  isTestStep_evm,
+  isTestStep_EXTCODESIZE,
+  isTestStep_EXTCODEHASH,
+  isTestStep_EXTCODECOPY,
+  isTestStep_REVERT,
+} from './test.types'
+import { encodeRevertData } from '../codec'
+import { getModifiableStorageFactory } from '../storage/contract-storage'
+import { GAS_LIMIT, NON_NULL_BYTES32 } from '../constants'
+
+export class ExecutionManagerTestRunner {
+  private snapshot: string
+  private contracts: {
+    OVM_SafetyChecker: Contract
+    OVM_StateManager: Contract
+    OVM_ExecutionManager: Contract
+    Helper_TestRunner: Contract
+    Factory__Helper_TestRunner_CREATE: ContractFactory
+  } = {
+    OVM_SafetyChecker: undefined,
+    OVM_StateManager: undefined,
+    OVM_ExecutionManager: undefined,
+    Helper_TestRunner: undefined,
+    Factory__Helper_TestRunner_CREATE: undefined,
+  }
+
+  public run(test: TestDefinition) {
+    test.preState = test.preState || {}
+    test.postState = test.postState || {}
+
+    describe(`OVM_ExecutionManager Test: ${test.name}`, () => {
+      test.subTests?.map((subTest) => {
+        this.run({
+          ...subTest,
+          preState: {
+            ...test.preState,
+            ...subTest.preState,
+          },
+          postState: {
+            ...test.postState,
+            ...subTest.postState,
+          },
+        })
+      })
+
+      test.parameters?.map((parameter) => {
+        beforeEach(async () => {
+          await this.initContracts()
+        })
+
+        let replacedTest: TestDefinition
+        let replacedParameter: TestParameter
+        beforeEach(async () => {
+          replacedTest = this.setPlaceholderStrings(test)
+          replacedParameter = this.setPlaceholderStrings(parameter)
+        })
+
+        beforeEach(async () => {
+          await this.contracts.OVM_StateManager.__setContractStorage({
+            accounts: {
+              [this.contracts.Helper_TestRunner.address]: {
+                nonce: 0,
+                codeHash: NON_NULL_BYTES32,
+                ethAddress: this.contracts.Helper_TestRunner.address,
+              },
+            },
+          })
+        })
+
+        beforeEach(async () => {
+          await this.contracts.OVM_ExecutionManager.__setContractStorage(
+            replacedTest.preState.ExecutionManager
+          )
+          await this.contracts.OVM_StateManager.__setContractStorage(
+            replacedTest.preState.StateManager
+          )
+        })
+
+        afterEach(async () => {
+          await this.contracts.OVM_ExecutionManager.__checkContractStorage(
+            replacedTest.postState.ExecutionManager
+          )
+          await this.contracts.OVM_StateManager.__checkContractStorage(
+            replacedTest.postState.StateManager
+          )
+        })
+
+        if (parameter.focus) {
+          it.only(`should execute: ${parameter.name}`, async () => {
+            for (const step of replacedParameter.steps) {
+              await this.runTestStep(step)
+            }
+          })
+        } else {
+          it(`should execute: ${parameter.name}`, async () => {
+            for (const step of replacedParameter.steps) {
+              await this.runTestStep(step)
+            }
+          })
+        }
+      })
+    })
+  }
+
+  private async initContracts() {
+    if (this.snapshot) {
+      await ethers.provider.send('evm_revert', [this.snapshot])
+      return
+    }
+
+    this.contracts.OVM_SafetyChecker = await (
+      await ethers.getContractFactory('OVM_SafetyChecker')
+    ).deploy()
+    this.contracts.OVM_ExecutionManager = await (
+      await getModifiableStorageFactory('OVM_ExecutionManager')
+    ).deploy(this.contracts.OVM_SafetyChecker.address)
+    this.contracts.OVM_StateManager = await (
+      await getModifiableStorageFactory('OVM_StateManager')
+    ).deploy(this.contracts.OVM_ExecutionManager.address)
+    this.contracts.Helper_TestRunner = await (
+      await ethers.getContractFactory('Helper_TestRunner')
+    ).deploy()
+    this.contracts.Factory__Helper_TestRunner_CREATE = await ethers.getContractFactory(
+      'Helper_TestRunner_CREATE'
+    )
+
+    this.snapshot = await ethers.provider.send('evm_snapshot', [])
+  }
+
+  private setPlaceholderStrings(obj: any) {
+    const getReplacementString = (kv: string): string => {
+      if (kv === '$OVM_EXECUTION_MANAGER') {
+        return this.contracts.OVM_ExecutionManager.address
+      } else if (kv === '$OVM_STATE_MANAGER') {
+        return this.contracts.OVM_StateManager.address
+      } else if (kv === '$OVM_SAFETY_CHECKER') {
+        return this.contracts.OVM_SafetyChecker.address
+      } else if (kv === '$OVM_CALL_HELPER') {
+        return this.contracts.Helper_TestRunner.address
+      } else if (kv.startsWith('$DUMMY_OVM_ADDRESS_')) {
+        return '0x' + (kv.split('$DUMMY_OVM_ADDRESS_')[1] + '0').repeat(20)
+      } else {
+        return kv
+      }
+    }
+
+    let ret: any = cloneDeep(obj)
+    if (Array.isArray(ret)) {
+      ret = ret.map((element: any) => {
+        return this.setPlaceholderStrings(element)
+      })
+    } else if (typeof ret === 'object' && ret !== null) {
+      for (const key of Object.keys(ret)) {
+        const replacedKey = getReplacementString(key)
+
+        if (replacedKey !== key) {
+          ret[replacedKey] = ret[key]
+          delete ret[key]
+        }
+
+        ret[replacedKey] = this.setPlaceholderStrings(ret[replacedKey])
+      }
+    } else if (typeof ret === 'string') {
+      ret = getReplacementString(ret)
+    }
+
+    return ret
+  }
+
+  private async runTestStep(step: TestStep) {
+    await this.contracts.OVM_ExecutionManager.ovmCALL(
+      GAS_LIMIT / 2,
+      this.contracts.Helper_TestRunner.address,
+      this.contracts.Helper_TestRunner.interface.encodeFunctionData(
+        'runSingleTestStep',
+        [this.parseTestStep(step)]
+      )
+    )
+  }
+
+  private parseTestStep(step: TestStep): ParsedTestStep {
+    return {
+      functionName: step.functionName,
+      functionData: this.encodeFunctionData(step),
+      expectedReturnStatus: this.getReturnStatus(step),
+      expectedReturnData: this.encodeExpectedReturnData(step),
+    }
+  }
+
+  private getReturnStatus(step: TestStep): boolean {
+    if (isTestStep_evm(step)) {
+      return false
+    } else if (isTestStep_Context(step)) {
+      return true
+    } else {
+      return step.expectedReturnStatus
+    }
+  }
+
+  private encodeFunctionData(step: TestStep): string {
+    if (isTestStep_evm(step)) {
+      if (isRevertFlagError(step.returnData)) {
+        return encodeRevertData(
+          step.returnData.flag,
+          step.returnData.data,
+          step.returnData.nuisanceGasLeft,
+          step.returnData.ovmGasRefund
+        )
+      } else {
+        return step.returnData || '0x'
+      }
+    }
+
+    let functionParams: any[] = []
+    if (
+      isTestStep_SSTORE(step) ||
+      isTestStep_SLOAD(step) ||
+      isTestStep_EXTCODESIZE(step) ||
+      isTestStep_EXTCODEHASH(step) ||
+      isTestStep_EXTCODECOPY(step)
+    ) {
+      functionParams = Object.values(step.functionParams)
+    } else if (isTestStep_CALL(step)) {
+      functionParams = [
+        step.functionParams.gasLimit,
+        step.functionParams.target,
+        step.functionParams.calldata ||
+          this.contracts.Helper_TestRunner.interface.encodeFunctionData(
+            'runMultipleTestSteps',
+            [
+              step.functionParams.subSteps.map((subStep) => {
+                return this.parseTestStep(subStep)
+              }),
+            ]
+          ),
+      ]
+    } else if (isTestStep_CREATE(step)) {
+      functionParams = [
+        this.contracts.Factory__Helper_TestRunner_CREATE.getDeployTransaction(
+          step.functionParams.bytecode || '0x',
+          step.functionParams.subSteps?.map((subStep) => {
+            return this.parseTestStep(subStep)
+          }) || []
+        ).data,
+      ]
+    } else if (isTestStep_REVERT(step)) {
+      functionParams = [step.revertData || '0x']
+    }
+
+    return this.contracts.OVM_ExecutionManager.interface.encodeFunctionData(
+      step.functionName,
+      functionParams
+    )
+  }
+
+  private encodeExpectedReturnData(step: TestStep): string {
+    if (isTestStep_evm(step)) {
+      return '0x'
+    }
+
+    if (isTestStep_REVERT(step)) {
+      return step.expectedReturnValue || '0x'
+    }
+
+    if (isRevertFlagError(step.expectedReturnValue)) {
+      return encodeRevertData(
+        step.expectedReturnValue.flag,
+        step.expectedReturnValue.data,
+        step.expectedReturnValue.nuisanceGasLeft,
+        step.expectedReturnValue.ovmGasRefund
+      )
+    }
+
+    let returnData: any[] = []
+    if (isTestStep_CALL(step)) {
+      if (step.expectedReturnValue === '0x00') {
+        return step.expectedReturnValue
+      } else {
+        returnData = [
+          step.expectedReturnStatus,
+          step.expectedReturnValue || '0x',
+        ]
+      }
+    } else if (BigNumber.isBigNumber(step.expectedReturnValue)) {
+      returnData = [step.expectedReturnValue.toHexString()]
+    } else if (step.expectedReturnValue !== undefined) {
+      if (step.expectedReturnValue === '0x00') {
+        return step.expectedReturnValue
+      } else {
+        returnData = [step.expectedReturnValue]
+      }
+    }
+
+    return this.contracts.OVM_ExecutionManager.interface.encodeFunctionResult(
+      step.functionName,
+      returnData
+    )
+  }
+}

packages/contracts/test/helpers/test-utils/test.types.ts

 /* External Imports */
 import { BigNumber } from 'ethers'
 
-export type SolidityFunctionParameter = string | number | BigNumber
+export type ContextOpcode =
+  | 'ovmCALLER'
+  | 'ovmADDRESS'
+  | 'ovmORIGIN'
+  | 'ovmTIMESTAMP'
+  | 'ovmGASLIMIT'
+  | 'ovmCHAINID'
 
-export interface TestStep {
+type CallOpcode = 'ovmCALL' | 'ovmSTATICCALL' | 'ovmDELEGATECALL'
+
+type RevertFlagError = {
+  flag: number
+  nuisanceGasLeft?: number
+  ovmGasRefund?: number
+  data?: string
+}
+
+interface TestStep_evm {
+  functionName: 'evmRETURN' | 'evmREVERT' | 'evmINVALID'
+  returnData?: string | RevertFlagError
+}
+
+interface TestStep_Context {
+  functionName: ContextOpcode
+  expectedReturnValue: string | number | BigNumber
+}
+
+interface TestStep_REVERT {
+  functionName: 'ovmREVERT'
+  revertData?: string
+  expectedReturnStatus: boolean
+  expectedReturnValue?: string
+}
+
+interface TestStep_EXTCODESIZE {
+  functionName: 'ovmEXTCODESIZE'
+  functionParams: {
+    address: string
+  }
+  expectedReturnStatus: boolean
+  expectedReturnValue: number | RevertFlagError
+}
+
+interface TestStep_EXTCODEHASH {
+  functionName: 'ovmEXTCODEHASH'
+  functionParams: {
+    address: string
+  }
+  expectedReturnStatus: boolean
+  expectedReturnValue: string | RevertFlagError
+}
+
+interface TestStep_EXTCODECOPY {
+  functionName: 'ovmEXTCODECOPY'
+  functionParams: {
+    address: string
+    offset: number
+    length: number
+  }
+  expectedReturnStatus: boolean
+  expectedReturnValue: string | RevertFlagError
+}
+
+interface TestStep_SSTORE {
+  functionName: 'ovmSSTORE'
+  functionParams: {
+    key: string
+    value: string
+  }
+  expectedReturnStatus: boolean
+  expectedReturnValue?: RevertFlagError
+}
+
+interface TestStep_SLOAD {
+  functionName: 'ovmSLOAD'
+  functionParams: {
+    key: string
+  }
+  expectedReturnStatus: boolean
+  expectedReturnValue: string | RevertFlagError
+}
+
+interface TestStep_CALL {
+  functionName: CallOpcode
+  functionParams: {
+    gasLimit: number | BigNumber
+    target: string
+    calldata?: string
+    subSteps?: TestStep[]
+  }
+  expectedReturnStatus: boolean
+  expectedReturnValue?: string | RevertFlagError
+}
+
+interface TestStep_CREATE {
+  functionName: 'ovmCREATE'
+  functionParams: {
+    bytecode?: string
+    subSteps?: TestStep[]
+  }
+  expectedReturnStatus: boolean
+  expectedReturnValue: string | RevertFlagError
+}
+
+interface TestStep_CREATE2 {
+  functionName: 'ovmCREATE2'
+  functionParams: {
+    salt: string
+    bytecode?: string
+    subSteps?: TestStep[]
+  }
+  expectedReturnStatus: boolean
+  expectedReturnValue: string | RevertFlagError
+}
+
+export type TestStep =
+  | TestStep_Context
+  | TestStep_SSTORE
+  | TestStep_SLOAD
+  | TestStep_CALL
+  | TestStep_CREATE
+  | TestStep_CREATE2
+  | TestStep_EXTCODESIZE
+  | TestStep_EXTCODEHASH
+  | TestStep_EXTCODECOPY
+  | TestStep_REVERT
+  | TestStep_evm
+
+export interface ParsedTestStep {
   functionName: string
-  functionParams: Array<
-    | SolidityFunctionParameter
-    | SolidityFunctionParameter[]
-    | TestStep[]
-    | boolean
-  >
+  functionData: string
   expectedReturnStatus: boolean
-  expectedReturnValues: any[]
+  expectedReturnData: string
 }
 
-export interface TestParameters {
-  steps: TestStep[]
+export const isRevertFlagError = (
+  expectedReturnValue: any
+): expectedReturnValue is RevertFlagError => {
+  return (
+    typeof expectedReturnValue === 'object' &&
+    expectedReturnValue !== null &&
+    expectedReturnValue.flag !== undefined
+  )
 }
 
-export interface TestDefinition {
+export const isTestStep_evm = (step: TestStep): step is TestStep_evm => {
+  return ['evmRETURN', 'evmREVERT', 'evmINVALID'].includes(step.functionName)
+}
+
+export const isTestStep_Context = (
+  step: TestStep
+): step is TestStep_Context => {
+  return [
+    'ovmCALLER',
+    'ovmADDRESS',
+    'ovmORIGIN',
+    'ovmTIMESTAMP',
+    'ovmGASLIMIT',
+    'ovmCHAINID',
+  ].includes(step.functionName)
+}
+
+export const isTestStep_SSTORE = (step: TestStep): step is TestStep_SSTORE => {
+  return step.functionName == 'ovmSSTORE'
+}
+
+export const isTestStep_SLOAD = (step: TestStep): step is TestStep_SLOAD => {
+  return step.functionName == 'ovmSLOAD'
+}
+
+export const isTestStep_EXTCODESIZE = (
+  step: TestStep
+): step is TestStep_EXTCODESIZE => {
+  return step.functionName == 'ovmEXTCODESIZE'
+}
+
+export const isTestStep_EXTCODEHASH = (
+  step: TestStep
+): step is TestStep_EXTCODEHASH => {
+  return step.functionName == 'ovmEXTCODEHASH'
+}
+
+export const isTestStep_EXTCODECOPY = (
+  step: TestStep
+): step is TestStep_EXTCODECOPY => {
+  return step.functionName == 'ovmEXTCODECOPY'
+}
+
+export const isTestStep_REVERT = (step: TestStep): step is TestStep_REVERT => {
+  return step.functionName == 'ovmREVERT'
+}
+
+export const isTestStep_CALL = (step: TestStep): step is TestStep_CALL => {
+  return ['ovmCALL', 'ovmSTATICCALL', 'ovmDELEGATECALL'].includes(
+    step.functionName
+  )
+}
+
+export const isTestStep_CREATE = (step: TestStep): step is TestStep_CREATE => {
+  return step.functionName == 'ovmCREATE'
+}
+
+export const isTestStep_CREATE2 = (
+  step: TestStep
+): step is TestStep_CREATE2 => {
+  return step.functionName == 'ovmCREATE2'
+}
+
+interface TestState {
+  ExecutionManager: any
+  StateManager: any
+}
+
+export interface TestParameter {
   name: string
   focus?: boolean
-  preState?: {
-    ExecutionManager?: any
-    StateManager?: any
-  }
-  parameters: Array<TestParameters | TestDefinition>
-  postState?: {
-    ExecutionManager?: any
-    StateManager?: any
-  }
+  steps: TestStep[]
 }
 
-export const isTestDefinition = (
-  parameters: TestParameters | TestDefinition
-): parameters is TestDefinition => {
-  return (parameters as TestDefinition).name !== undefined
+export interface TestDefinition {
+  name: string
+  focus?: boolean
+  preState?: Partial<TestState>
+  postState?: Partial<TestState>
+  parameters?: TestParameter[]
+  subTests?: TestDefinition[]
 }