state-surgery: hardhat utils (#3206)

* state-surgery: hardhat utils

Add hardhat utils for reading hardhat artifacts
from disk. This can read hardhat artifacts, buildinfo
and hardhat deploy artifacts. This is necessary for
implementing the state surgery in go, because these
files must be consumed as part of the state surgery process.

* ci: run state-surgery tests in ci
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>

.circleci/config.yml

@@ -413,6 +413,24 @@ jobs:
                 command: make <<parameters.binary_name>>
                 working_directory: <<parameters.working_directory>>
 
+  state-surgery-tests:
+    docker:
+      - image: ethereumoptimism/ci-builder:latest
+    steps:
+      - checkout
+      - run:
+          name: Check if we should run
+          command: |
+            CHANGED=$(bash ./ops/docker/ci-builder/check-changed.sh "state-surgery")
+            if [[ "$CHANGED" = "FALSE" ]]; then
+              circleci step halt
+            fi
+      - run:
+          name: Test
+          command: |
+            gotestsum --junitfile /test-results/state-surgery.xml -- -coverpkg=github.com/ethereum-optimism/optimism/... -coverprofile=coverage.out -covermode=atomic ./...
+          working_directory: state-surgery
+
   geth-tests:
     docker:
       - image: ethereumoptimism/ci-builder:latest
@@ -692,6 +710,7 @@ workflows:
       - integration-tests
       - semgrep-scan
       - go-mod-tidy
+      - state-surgery-tests
 
   nightly:
     triggers:

go.work.sum

@@ -59,6 +59,7 @@ github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9G
 github.com/multiformats/go-multihash v0.1.0/go.mod h1:RJlXsxt6vHGaia+S8We0ErjhojtKzPP2AH4+kYM7k84=
 github.com/multiformats/go-multistream v0.3.1/go.mod h1:ODRoqamLUsETKS9BNcII4gcRsJBU5VAwRIv7O39cEXg=
 github.com/prometheus/common v0.35.0/go.mod h1:phzohg0JFMnBEFGxTDbfu3QyL5GI8gTQJFhYO5B3mfA=
+github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE=
 github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
 github.com/supranational/blst v0.3.8-0.20220526154634-513d2456b344/go.mod h1:jZJtfjgudtNl4en1tzwPIV3KjUnQUvG3/j+w+fVonLw=
 github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=

state-surgery/hardhat/hardhat.go

+package hardhat
+
+import (
+	"encoding/json"
+	"fmt"
+	"io/fs"
+	"os"
+	"path/filepath"
+	"strings"
+	"sync"
+)
+
+// `Hardhat` encapsulates all of the functionality required to interact
+// with hardhat style artifacts.
+type Hardhat struct {
+	ArtifactPaths   []string
+	DeploymentPaths []string
+
+	network string
+
+	amu sync.Mutex
+	dmu sync.Mutex
+	bmu sync.Mutex
+
+	artifacts   []*Artifact
+	deployments []*Deployment
+	buildInfos  []*BuildInfo
+}
+
+// New creates a new `Hardhat` struct and reads all of the files from
+// disk so that they are cached for the end user. A network is passed
+// that corresponds to the network that they deployments are associated
+// with. A slice of artifact paths and deployment paths are passed
+// so that a single `Hardhat` instance can operate on multiple sets
+// of artifacts and deployments. The deployments paths should be
+// the root of the deployments directory that contains additional
+// directories for each particular network.
+func New(network string, artifacts, deployments []string) (*Hardhat, error) {
+	hh := &Hardhat{
+		network:         network,
+		ArtifactPaths:   artifacts,
+		DeploymentPaths: deployments,
+	}
+
+	if err := hh.init(); err != nil {
+		return nil, err
+	}
+
+	return hh, nil
+}
+
+// init is called in the constructor and will cache required files to disk.
+func (h *Hardhat) init() error {
+	if err := h.initArtifacts(); err != nil {
+		return err
+	}
+	if err := h.initDeployments(); err != nil {
+		return err
+	}
+	return nil
+}
+
+// initDeployments reads all of the deployment json files from disk and then
+// caches the deserialized `Deployment` structs.
+func (h *Hardhat) initDeployments() error {
+	for _, deploymentPath := range h.DeploymentPaths {
+		fileSystem := os.DirFS(filepath.Join(deploymentPath, h.network))
+		fs.WalkDir(fileSystem, ".", func(path string, d fs.DirEntry, err error) error {
+			if err != nil {
+				return err
+			}
+			if d.IsDir() {
+				return nil
+			}
+			if strings.Contains(path, "solcInputs") {
+				return nil
+			}
+
+			name := filepath.Join(deploymentPath, h.network, path)
+			if !strings.HasSuffix(name, ".json") {
+				return nil
+			}
+			file, err := os.ReadFile(name)
+			if err != nil {
+				return err
+			}
+			var deployment Deployment
+			if err := json.Unmarshal(file, &deployment); err != nil {
+				return err
+			}
+
+			deployment.Name = filepath.Base(strings.TrimRight(name, ".json"))
+			h.deployments = append(h.deployments, &deployment)
+			return nil
+		})
+	}
+	return nil
+}
+
+// initArtifacts reads all of the artifact json files from disk and then caches
+// the deserialized `Artifact` structs.
+func (h *Hardhat) initArtifacts() error {
+	for _, artifactPath := range h.ArtifactPaths {
+		fileSystem := os.DirFS(artifactPath)
+		fs.WalkDir(fileSystem, ".", func(path string, d fs.DirEntry, err error) error {
+			if err != nil {
+				return err
+			}
+			if d.IsDir() {
+				return nil
+			}
+			name := filepath.Join(artifactPath, path)
+
+			if strings.Contains(name, "build-info") {
+				return nil
+			}
+			if strings.HasSuffix(name, ".dbg.json") {
+				return nil
+			}
+			file, err := os.ReadFile(name)
+			if err != nil {
+				return err
+			}
+			var artifact Artifact
+			if err := json.Unmarshal(file, &artifact); err != nil {
+				return err
+			}
+			h.artifacts = append(h.artifacts, &artifact)
+			return nil
+		})
+	}
+	return nil
+}
+
+// GetArtifact returns the artifact that corresponds to the contract.
+// This method supports just the contract name and the fully qualified
+// contract name.
+func (h *Hardhat) GetArtifact(name string) (*Artifact, error) {
+	h.amu.Lock()
+	defer h.amu.Unlock()
+
+	if IsFullyQualifiedName(name) {
+		fqn := ParseFullyQualifiedName(name)
+		for _, artifact := range h.artifacts {
+			contractNameMatches := artifact.ContractName == fqn.ContractName
+			sourceNameMatches := artifact.SourceName == fqn.SourceName
+
+			if contractNameMatches && sourceNameMatches {
+				return artifact, nil
+			}
+		}
+		return nil, fmt.Errorf("cannot find artifact %s", name)
+	}
+
+	for _, artifact := range h.artifacts {
+		if name == artifact.ContractName {
+			return artifact, nil
+		}
+	}
+
+	return nil, fmt.Errorf("cannot find artifact %s", name)
+}
+
+// GetDeployment returns the deployment that corresponds to the contract.
+// It does not support fully qualified contract names.
+func (h *Hardhat) GetDeployment(name string) (*Deployment, error) {
+	h.dmu.Lock()
+	defer h.dmu.Unlock()
+
+	fqn := ParseFullyQualifiedName(name)
+	for _, deployment := range h.deployments {
+		if deployment.Name == fqn.ContractName {
+			return deployment, nil
+		}
+	}
+
+	return nil, fmt.Errorf("cannot find deployment %s", name)
+}
+
+// GetBuildInfo returns the build info that corresponds to the contract.
+// It does not support fully qualified contract names.
+func (h *Hardhat) GetBuildInfo(name string) (*BuildInfo, error) {
+	h.bmu.Lock()
+	defer h.bmu.Unlock()
+
+	fqn := ParseFullyQualifiedName(name)
+	buildInfos := make([]*BuildInfo, 0)
+
+	for _, artifactPath := range h.ArtifactPaths {
+		fileSystem := os.DirFS(artifactPath)
+		fs.WalkDir(fileSystem, ".", func(path string, d fs.DirEntry, err error) error {
+			if err != nil {
+				return err
+			}
+			if d.IsDir() {
+				return nil
+			}
+			name := filepath.Join(artifactPath, path)
+
+			if !strings.HasSuffix(name, ".dbg.json") {
+				return nil
+			}
+
+			// Remove ".dbg.json"
+			target := filepath.Base(name[:len(name)-9])
+			if fqn.ContractName != target {
+				return nil
+			}
+
+			file, err := os.ReadFile(name)
+			if err != nil {
+				return err
+			}
+			var debugFile DebugFile
+			if err := json.Unmarshal(file, &debugFile); err != nil {
+				return err
+			}
+			relPath := filepath.Join(filepath.Dir(name), debugFile.BuildInfo)
+			if err != nil {
+				return err
+			}
+			debugPath, _ := filepath.Abs(relPath)
+
+			buildInfoFile, err := os.ReadFile(debugPath)
+			if err != nil {
+				return err
+			}
+
+			var buildInfo BuildInfo
+			if err := json.Unmarshal(buildInfoFile, &buildInfo); err != nil {
+				return err
+			}
+
+			buildInfos = append(buildInfos, &buildInfo)
+
+			return nil
+		})
+	}
+
+	// TODO(tynes): handle multiple contracts with same name when required
+	if len(buildInfos) > 1 {
+		return nil, fmt.Errorf("Multiple contracts with name %s", name)
+	}
+	if len(buildInfos) == 0 {
+		return nil, fmt.Errorf("Cannot find BuildInfo for %s", name)
+	}
+
+	return buildInfos[0], nil
+}

state-surgery/hardhat/hardhat_test.go

+package hardhat_test
+
+import (
+	"testing"
+
+	"github.com/ethereum-optimism/optimism/state-surgery/hardhat"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestGetFullyQualifiedName(t *testing.T) {
+	t.Parallel()
+
+	cases := []struct {
+		fqn    hardhat.QualifiedName
+		expect string
+	}{
+		{
+			fqn:    hardhat.QualifiedName{"contract.sol", "C"},
+			expect: "contract.sol:C",
+		},
+		{
+			fqn:    hardhat.QualifiedName{"folder/contract.sol", "C"},
+			expect: "folder/contract.sol:C",
+		},
+		{
+			fqn:    hardhat.QualifiedName{"folder/a:b/contract.sol", "C"},
+			expect: "folder/a:b/contract.sol:C",
+		},
+	}
+
+	for _, test := range cases {
+		got := hardhat.GetFullyQualifiedName(test.fqn.SourceName, test.fqn.ContractName)
+		require.Equal(t, got, test.expect)
+	}
+}
+
+func TestParseFullyQualifiedName(t *testing.T) {
+	t.Parallel()
+
+	cases := []struct {
+		fqn    string
+		expect hardhat.QualifiedName
+	}{
+		{
+			fqn:    "contract.sol:C",
+			expect: hardhat.QualifiedName{"contract.sol", "C"},
+		},
+		{
+			fqn:    "folder/contract.sol:C",
+			expect: hardhat.QualifiedName{"folder/contract.sol", "C"},
+		},
+		{
+			fqn:    "folder/a:b/contract.sol:C",
+			expect: hardhat.QualifiedName{"folder/a:b/contract.sol", "C"},
+		},
+	}
+
+	for _, test := range cases {
+		got := hardhat.ParseFullyQualifiedName(test.fqn)
+		require.Equal(t, got, test.expect)
+	}
+}
+
+func TestIsFullyQualifiedName(t *testing.T) {
+	t.Parallel()
+
+	cases := []struct {
+		fqn    string
+		expect bool
+	}{
+		{
+			fqn:    "contract.sol:C",
+			expect: true,
+		},
+		{
+			fqn:    "folder/contract.sol:C",
+			expect: true,
+		},
+		{
+			fqn:    "folder/a:b/contract.sol:C",
+			expect: true,
+		},
+		{
+			fqn:    "C",
+			expect: false,
+		},
+		{
+			fqn:    "contract.sol",
+			expect: false,
+		},
+		{
+			fqn:    "folder/contract.sol",
+			expect: false,
+		},
+	}
+
+	for _, test := range cases {
+		got := hardhat.IsFullyQualifiedName(test.fqn)
+		require.Equal(t, got, test.expect)
+	}
+}
+
+func TestHardhatGetArtifact(t *testing.T) {
+	t.Parallel()
+
+	hh, err := hardhat.New(
+		"goerli",
+		[]string{"testdata/artifacts"},
+		[]string{"testdata/deployments"},
+	)
+	require.Nil(t, err)
+
+	artifact, err := hh.GetArtifact("HelloWorld")
+	require.Nil(t, err)
+	require.NotNil(t, artifact)
+}
+
+func TestHardhatGetBuildInfo(t *testing.T) {
+	t.Parallel()
+
+	hh, err := hardhat.New(
+		"goerli",
+		[]string{"testdata/artifacts"},
+		[]string{"testdata/deployments"},
+	)
+	require.Nil(t, err)
+
+	buildInfo, err := hh.GetBuildInfo("HelloWorld")
+	require.Nil(t, err)
+	require.NotNil(t, buildInfo)
+}
+
+func TestHardhatGetDeployments(t *testing.T) {
+	hh, err := hardhat.New(
+		"goerli",
+		[]string{"testdata/artifacts"},
+		[]string{"testdata/deployments"},
+	)
+	require.Nil(t, err)
+
+	deployment, err := hh.GetDeployment("OptimismPortal")
+	require.Nil(t, err)
+	require.NotNil(t, deployment)
+}

state-surgery/hardhat/testdata/artifacts/contracts/HelloWorld.sol/HelloWorld.dbg.json

+{
+  "_format": "hh-sol-dbg-1",
+  "buildInfo": "../../build-info/c5729209e616d57e62ada8bf0034436e.json"
+}

state-surgery/hardhat/testdata/artifacts/contracts/HelloWorld.sol/HelloWorld.json

+{
+  "_format": "hh-sol-artifact-1",
+  "contractName": "HelloWorld",
+  "sourceName": "contracts/HelloWorld.sol",
+  "abi": [
+    {
+      "inputs": [],
+      "stateMutability": "nonpayable",
+      "type": "constructor"
+    },
+    {
+      "inputs": [],
+      "name": "gm",
+      "outputs": [
+        {
+          "internalType": "uint256",
+          "name": "",
+          "type": "uint256"
+        }
+      ],
+      "stateMutability": "nonpayable",
+      "type": "function"
+    },
+    {
+      "inputs": [],
+      "name": "time",
+      "outputs": [
+        {
+          "internalType": "uint256",
+          "name": "",
+          "type": "uint256"
+        }
+      ],
+      "stateMutability": "view",
+      "type": "function"
+    }
+  ],
+  "bytecode": "0x608060405234801561001057600080fd5b504260008190555060ed806100266000396000f3fe6080604052348015600f57600080fd5b506004361060325760003560e01c806316ada547146037578063c0129d43146051575b600080fd5b603d606b565b60405160489190609e565b60405180910390f35b60576071565b60405160629190609e565b60405180910390f35b60005481565b6000806000549050426000819055508091505090565b6000819050919050565b6098816087565b82525050565b600060208201905060b160008301846091565b9291505056fea2646970667358221220880ac624623135a410271b0c719fe0f86b85ff1eb258d2f766c58f2e87907b8864736f6c634300080f0033",
+  "deployedBytecode": "0x6080604052348015600f57600080fd5b506004361060325760003560e01c806316ada547146037578063c0129d43146051575b600080fd5b603d606b565b60405160489190609e565b60405180910390f35b60576071565b60405160629190609e565b60405180910390f35b60005481565b6000806000549050426000819055508091505090565b6000819050919050565b6098816087565b82525050565b600060208201905060b160008301846091565b9291505056fea2646970667358221220880ac624623135a410271b0c719fe0f86b85ff1eb258d2f766c58f2e87907b8864736f6c634300080f0033",
+  "linkReferences": {},
+  "deployedLinkReferences": {}
+}

state-surgery/hardhat/types.go

+package hardhat
+
+import (
+	"encoding/json"
+
+	"github.com/ethereum/go-ethereum/accounts/abi"
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/common/hexutil"
+)
+
+// Deployment represents a hardhat-deploy artifact file
+type Deployment struct {
+	Name             string
+	Abi              abi.ABI         `json:"abi"`
+	Address          common.Address  `json:"address"`
+	Args             []any           `json:"args"`
+	Bytecode         hexutil.Bytes   `json:"bytecode"`
+	DeployedBytecode hexutil.Bytes   `json:"deployedBytecode"`
+	Devdoc           json.RawMessage `json:"devdoc"`
+	Metadata         string          `json:"metadata"`
+	Receipt          Receipt         `json:"receipt"`
+	SolcInputHash    string          `json:"solcInputHash"`
+	StorageLayout    StorageLayout   `json:"storageLayout"`
+	TransactionHash  common.Hash     `json:"transactionHash"`
+	Userdoc          json.RawMessage `json:"userdoc"`
+}
+
+// Receipt represents the receipt held in a hardhat-deploy
+// artifact file
+type Receipt struct {
+	To                *common.Address `json:"to"`
+	From              common.Address  `json:"from"`
+	ContractAddress   *common.Address `json:"contractAddress"`
+	TransactionIndex  uint            `json:"transactionIndex"`
+	GasUsed           uint            `json:"gasUsed,string"`
+	LogsBloom         hexutil.Bytes   `json:"logsBloom"`
+	BlockHash         common.Hash     `json:"blockHash"`
+	TransactionHash   common.Hash     `json:"transactionHash"`
+	Logs              []Log           `json:"logs"`
+	BlockNumber       uint            `json:"blockNumber"`
+	CumulativeGasUsed uint            `json:"cumulativeGasUsed,string"`
+	Status            uint            `json:"status"`
+	Byzantium         bool            `json:"byzantium"`
+}
+
+// Log represents the logs in the hardhat deploy artifact receipt
+type Log struct {
+	TransactionIndex uint           `json:"transactionIndex"`
+	BlockNumber      uint           `json:"blockNumber"`
+	TransactionHash  common.Hash    `json:"transactionHash"`
+	Address          common.Address `json:"address"`
+	Topics           []common.Hash  `json:"topics"`
+	Data             hexutil.Bytes  `json:"data"`
+	LogIndex         uint           `json:"logIndex"`
+	Blockhash        common.Hash    `json:"blockHash"`
+}
+
+// StorageLayout represents the storage layout of a contract
+type StorageLayout struct {
+	Storage []StorageLayoutEntry         `json:"storage"`
+	Types   map[string]StorageLayoutType `json:"types"`
+}
+
+// StorageLayoutEntry represents a single entry in the StorageLayout
+type StorageLayoutEntry struct {
+	AstId    uint   `json:"astId"`
+	Contract string `json:"contract"`
+	Label    string `json:"label"`
+	Offset   uint   `json:"offset"`
+	Slot     uint   `json:"slot,string"`
+	Type     string `json"type"`
+}
+
+// StorageLayoutType represents the type of storage layout
+type StorageLayoutType struct {
+	Encoding      string `json:"encoding"`
+	Label         string `json:"label"`
+	NumberOfBytes string `json:"numberOfBytes"`
+	Key           string `json:"key,omitempty"`
+	Value         string `json:"value,omitempty"`
+}
+
+// Artifact represents a hardhat compilation artifact
+type Artifact struct {
+	Format                 string         `json:"_format"`
+	ContractName           string         `json:"contractName"`
+	SourceName             string         `json:"sourceName"`
+	Abi                    abi.ABI        `json:"abi"`
+	Bytecode               hexutil.Bytes  `json:"bytecode"`
+	DeployedBytecode       hexutil.Bytes  `json:"deployedBytecode"`
+	LinkReferences         LinkReferences `json:"linkReferences"`
+	DeployedLinkReferences LinkReferences `json:"deployedLinkReferences"`
+}
+
+// LinkReferences represents the linked contracts
+type LinkReferences map[string]LinkReference
+
+// LinkReference represents a single linked contract
+type LinkReference map[string][]LinkReferenceOffset
+
+// LinkReferenceOffset represents the offsets in a link reference
+type LinkReferenceOffset struct {
+	Length uint `json:"length"`
+	Start  uint `json:"start"`
+}
+
+// DebugFile represents the debug file that contains the path
+// to the build info file
+type DebugFile struct {
+	Format    string `json:"_format"`
+	BuildInfo string `json:"buildInfo"`
+}
+
+// BuildInfo represents a hardhat build info artifact that is created
+// after compilation
+type BuildInfo struct {
+	Format          string          `json:"_format"`
+	Id              string          `json:"id"`
+	SolcVersion     string          `json:"solcVersion"`
+	SolcLongVersion string          `json:"solcLongVersion"`
+	Input           json.RawMessage `json:"input"`
+	Output          json.RawMessage `json:"output"`
+}

state-surgery/hardhat/utils.go

+package hardhat
+
+import "strings"
+
+type QualifiedName struct {
+	SourceName   string
+	ContractName string
+}
+
+func ParseFullyQualifiedName(name string) QualifiedName {
+	names := strings.Split(name, ":")
+	if len(names) == 1 {
+		return QualifiedName{
+			SourceName:   "",
+			ContractName: names[0],
+		}
+	}
+
+	contractName := names[len(names)-1]
+	sourceName := strings.Join(names[0:len(names)-1], ":")
+
+	return QualifiedName{
+		ContractName: contractName,
+		SourceName:   sourceName,
+	}
+}
+
+func GetFullyQualifiedName(sourceName, contractName string) string {
+	return sourceName + ":" + contractName
+}
+
+func IsFullyQualifiedName(name string) bool {
+	return strings.Contains(name, ":")
+}