transition snapshots script to common framework (#13398)

op-chain-ops/solc/types.go

 package solc
 
 import (
+	"encoding/json"
 	"fmt"
 
 	"github.com/ethereum/go-ethereum/accounts/abi"
 )
 
+type AbiType struct {
+	Parsed abi.ABI
+	Raw    interface{}
+}
+
+func (a *AbiType) UnmarshalJSON(data []byte) error {
+	if err := json.Unmarshal(data, &a.Raw); err != nil {
+		return err
+	}
+	return json.Unmarshal(data, &a.Parsed)
+}
+
 type CompilerInput struct {
 	Language string                       `json:"language"`
 	Sources  map[string]map[string]string `json:"sources"`
@@ -39,7 +52,7 @@ type CompilerOutputContracts map[string]CompilerOutputContract
 // CompilerOutputContract represents the solc compiler output for a contract.
 // Ignoring some fields such as devdoc and userdoc.
 type CompilerOutputContract struct {
-	Abi           abi.ABI           `json:"abi"`
+	Abi           AbiType           `json:"abi"`
 	Evm           CompilerOutputEvm `json:"evm"`
 	Metadata      string            `json:"metadata"`
 	StorageLayout StorageLayout     `json:"storageLayout"`
@@ -72,6 +85,14 @@ func (s *StorageLayout) GetStorageLayoutType(name string) (StorageLayoutType, er
 	return StorageLayoutType{}, fmt.Errorf("%s not found", name)
 }
 
+type AbiSpecStorageLayoutEntry struct {
+	Bytes  uint   `json:"bytes,string"`
+	Label  string `json:"label"`
+	Offset uint   `json:"offset"`
+	Slot   uint   `json:"slot,string"`
+	Type   string `json:"type"`
+}
+
 type StorageLayoutEntry struct {
 	AstId    uint   `json:"astId"`
 	Contract string `json:"contract"`
@@ -241,7 +262,7 @@ type Expression struct {
 }
 
 type ForgeArtifact struct {
-	Abi               abi.ABI                `json:"abi"`
+	Abi               AbiType                `json:"abi"`
 	Bytecode          CompilerOutputBytecode `json:"bytecode"`
 	DeployedBytecode  CompilerOutputBytecode `json:"deployedBytecode"`
 	MethodIdentifiers map[string]string      `json:"methodIdentifiers"`
@@ -266,7 +287,7 @@ type ForgeCompilerInfo struct {
 }
 
 type ForgeMetadataOutput struct {
-	Abi     abi.ABI        `json:"abi"`
+	Abi     AbiType        `json:"abi"`
 	DevDoc  ForgeDocObject `json:"devdoc"`
 	UserDoc ForgeDocObject `json:"userdoc"`
 }

packages/contracts-bedrock/scripts/autogen/generate-snapshots/main.go

 package main
 
 import (
-	"bytes"
-	"encoding/json"
-	"flag"
 	"fmt"
 	"os"
 	"path/filepath"
 	"regexp"
-	"sort"
+	"strings"
+
+	"github.com/ethereum-optimism/optimism/op-chain-ops/solc"
+	"github.com/ethereum-optimism/optimism/packages/contracts-bedrock/scripts/checks/common"
 )
 
-type ForgeArtifact struct {
-	// ABI is a nested JSON data structure, including some objects/maps.
-	// We declare it as interface, and not raw-message, such that Go decodes into map[string]interface{}
-	// where possible. The JSON-encoder will then sort the keys (default Go JSON behavior on maps),
-	// to reproduce the sortKeys(abi) result of the legacy Typescript version of the snapshort-generator.
-	ABI interface{} `json:"abi"`
-	Ast *struct {
-		NodeType string `json:"nodeType"`
-		Nodes    []struct {
-			NodeType     string `json:"nodeType"`
-			Name         string `json:"name"`
-			ContractKind string `json:"contractKind"`
-			Abstract     bool   `json:"abstract"`
-		} `json:"nodes"`
-	} `json:"ast"`
-	StorageLayout struct {
-		Storage []struct {
-			Type   string          `json:"type"`
-			Label  json.RawMessage `json:"label"`
-			Offset json.RawMessage `json:"offset"`
-			Slot   json.RawMessage `json:"slot"`
-		} `json:"storage"`
-		Types map[string]struct {
-			Label         string          `json:"label"`
-			NumberOfBytes json.RawMessage `json:"numberOfBytes"`
-		} `json:"types"`
-	} `json:"storageLayout"`
-	Bytecode struct {
-		Object string `json:"object"`
-	} `json:"bytecode"`
-}
+const (
+	storageLayoutDir = "snapshots/storageLayout"
+	abiDir           = "snapshots/abi"
+)
 
-type AbiSpecStorageLayoutEntry struct {
-	Bytes  json.RawMessage `json:"bytes"`
-	Label  json.RawMessage `json:"label"`
-	Offset json.RawMessage `json:"offset"`
-	Slot   json.RawMessage `json:"slot"`
-	Type   string          `json:"type"`
+type SnapshotResult struct {
+	ContractName  string
+	Abi           interface{}
+	StorageLayout []solc.AbiSpecStorageLayoutEntry
 }
 
 func main() {
-	flag.Parse()
-	if flag.NArg() != 1 {
-		fmt.Println("Expected path of contracts-bedrock as CLI argument")
+	if err := resetDirectory(storageLayoutDir); err != nil {
+		fmt.Printf("failed to reset storage layout directory: %v\n", err)
 		os.Exit(1)
 	}
-	rootDir := flag.Arg(0)
-	err := generateSnapshots(rootDir)
+	if err := resetDirectory(abiDir); err != nil {
+		fmt.Printf("failed to reset abi directory: %v\n", err)
+		os.Exit(1)
+	}
+
+	results, err := common.ProcessFilesGlob(
+		[]string{"forge-artifacts/**/*.json"},
+		[]string{},
+		processFile,
+	)
 	if err != nil {
 		fmt.Printf("Failed to generate snapshots: %v\n", err)
 		os.Exit(1)
 	}
-}
 
-func generateSnapshots(rootDir string) error {
-
-	forgeArtifactsDir := filepath.Join(rootDir, "forge-artifacts")
-	srcDir := filepath.Join(rootDir, "src")
-	outDir := filepath.Join(rootDir, "snapshots")
-
-	storageLayoutDir := filepath.Join(outDir, "storageLayout")
-	abiDir := filepath.Join(outDir, "abi")
-
-	fmt.Printf("writing abi and storage layout snapshots to %s\n", outDir)
-
-	// Clean and recreate directories
-	if err := os.RemoveAll(storageLayoutDir); err != nil {
-		return fmt.Errorf("failed to remove storage layout dir: %w", err)
-	}
-	if err := os.RemoveAll(abiDir); err != nil {
-		return fmt.Errorf("failed to remove ABI dir: %w", err)
-	}
-	if err := os.MkdirAll(storageLayoutDir, os.ModePerm); err != nil {
-		return fmt.Errorf("failed to create storage layout dir: %w", err)
-	}
-	if err := os.MkdirAll(abiDir, os.ModePerm); err != nil {
-		return fmt.Errorf("failed to create ABI dir: %w", err)
+	for _, result := range results {
+		if result == nil {
+			continue
 		}
 
-	contractSources, err := getAllContractsSources(srcDir)
+		err := common.WriteJSON(result.Abi, filepath.Join(abiDir, fmt.Sprintf("%s.json", result.ContractName)))
 		if err != nil {
-		return fmt.Errorf("failed to retrieve contract sources: %w", err)
+			fmt.Printf("failed to write abi: %v\n", err)
+			os.Exit(1)
 		}
 
-	knownAbis := make(map[string]interface{})
-
-	for _, contractFile := range contractSources {
-		contractArtifacts := filepath.Join(forgeArtifactsDir, contractFile)
-		files, err := os.ReadDir(contractArtifacts)
+		err = common.WriteJSON(result.StorageLayout, filepath.Join(storageLayoutDir, fmt.Sprintf("%s.json", result.ContractName)))
 		if err != nil {
-			return fmt.Errorf("failed to scan contract artifacts of %q: %w", contractFile, err)
+			fmt.Printf("failed to write storage layout: %v\n", err)
+			os.Exit(1)
+		}
 	}
+}
 
-		for _, file := range files {
-			artifactPath := filepath.Join(contractArtifacts, file.Name())
-			data, err := os.ReadFile(artifactPath)
+func processFile(file string) (*SnapshotResult, []error) {
+	artifact, err := common.ReadForgeArtifact(file)
 	if err != nil {
-				return fmt.Errorf("failed to read artifact %q: %w", artifactPath, err)
-			}
-			var artifact ForgeArtifact
-			if err := json.Unmarshal(data, &artifact); err != nil {
-				return fmt.Errorf("failed to decode artifact %q: %w", artifactPath, err)
+		return nil, []error{err}
 	}
 
-			contractName, err := parseArtifactName(file.Name())
+	contractName, err := parseArtifactName(file)
 	if err != nil {
-				return fmt.Errorf("failed to parse artifact name %q: %w", file.Name(), err)
+		return nil, []error{fmt.Errorf("failed to parse artifact name %q: %w", file, err)}
 	}
 
-			// HACK: This is a hack to ignore libraries and abstract contracts. Not robust against changes to solc's internal ast repr
-			if artifact.Ast == nil {
-				return fmt.Errorf("ast isn't present in forge-artifacts. Did you run forge build with `--ast`? Artifact: %s", artifactPath)
+	// Skip anything that isn't in the src directory.
+	if !strings.HasPrefix(artifact.Ast.AbsolutePath, "src/") {
+		return nil, nil
 	}
-			// Check if the artifact is a contract
+
+	// Skip anything that isn't a proper contract.
 	isContract := false
 	for _, node := range artifact.Ast.Nodes {
 		if node.NodeType == "ContractDefinition" &&
@@ -136,18 +89,19 @@ func generateSnapshots(rootDir string) error {
 		}
 	}
 	if !isContract {
-				fmt.Printf("ignoring library/interface %s\n", contractName)
-				continue
+		return nil, nil
 	}
 
-			storageLayout := make([]AbiSpecStorageLayoutEntry, 0, len(artifact.StorageLayout.Storage))
+	storageLayout := make([]solc.AbiSpecStorageLayoutEntry, 0, len(artifact.StorageLayout.Storage))
 	for _, storageEntry := range artifact.StorageLayout.Storage {
-				// convert ast-based type to solidity type
+		// Convert ast-based type to Solidity type.
 		typ, ok := artifact.StorageLayout.Types[storageEntry.Type]
 		if !ok {
-					return fmt.Errorf("undefined type for %s:%s", contractName, storageEntry.Label)
+			return nil, []error{fmt.Errorf("undefined type for %s:%s", contractName, storageEntry.Label)}
 		}
-				storageLayout = append(storageLayout, AbiSpecStorageLayoutEntry{
+
+		// Convert to Solidity storage layout entry.
+		storageLayout = append(storageLayout, solc.AbiSpecStorageLayoutEntry{
 			Label:  storageEntry.Label,
 			Bytes:  typ.NumberOfBytes,
 			Offset: storageEntry.Offset,
@@ -156,96 +110,31 @@ func generateSnapshots(rootDir string) error {
 		})
 	}
 
-			if existingAbi, exists := knownAbis[contractName]; exists {
-				if !jsonEqual(existingAbi, artifact.ABI) {
-					return fmt.Errorf("detected multiple artifact versions with different ABIs for %s", contractFile)
-				} else {
-					fmt.Printf("detected multiple artifacts for %s\n", contractName)
-				}
-			} else {
-				knownAbis[contractName] = artifact.ABI
-			}
-
-			// Sort and write snapshots
-			if err := writeJSON(filepath.Join(abiDir, contractName+".json"), artifact.ABI); err != nil {
-				return fmt.Errorf("failed to write ABI snapshot JSON of %q: %w", contractName, err)
-			}
-
-			if err := writeJSON(filepath.Join(storageLayoutDir, contractName+".json"), storageLayout); err != nil {
-				return fmt.Errorf("failed to write storage layout snapshot JSON of %q: %w", contractName, err)
-			}
-		}
-	}
-	return nil
-}
-
-func getAllContractsSources(srcDir string) ([]string, error) {
-	var paths []string
-	if err := readFilesRecursively(srcDir, &paths); err != nil {
-		return nil, fmt.Errorf("failed to retrieve files: %w", err)
-	}
-
-	var solFiles []string
-	for _, p := range paths {
-		if filepath.Ext(p) == ".sol" {
-			solFiles = append(solFiles, filepath.Base(p))
-		}
-	}
-	sort.Strings(solFiles)
-	return solFiles, nil
-}
-
-func readFilesRecursively(dir string, paths *[]string) error {
-	files, err := os.ReadDir(dir)
-	if err != nil {
-		return err
-	}
-
-	for _, file := range files {
-		filePath := filepath.Join(dir, file.Name())
-		if file.IsDir() {
-			if err := readFilesRecursively(filePath, paths); err != nil {
-				return fmt.Errorf("failed to recurse into %q: %w", filePath, err)
-			}
-		} else {
-			*paths = append(*paths, filePath)
-		}
-	}
-	return nil
+	return &SnapshotResult{
+		ContractName:  contractName,
+		Abi:           artifact.Abi.Raw,
+		StorageLayout: storageLayout,
+	}, nil
 }
 
 // ContractName.0.9.8.json -> ContractName.sol
 // ContractName.json -> ContractName.sol
 func parseArtifactName(artifactVersionFile string) (string, error) {
 	re := regexp.MustCompile(`(.*?)\.([0-9]+\.[0-9]+\.[0-9]+)?`)
-	match := re.FindStringSubmatch(artifactVersionFile)
+	baseName := filepath.Base(artifactVersionFile)
+	match := re.FindStringSubmatch(baseName)
 	if len(match) < 2 {
 		return "", fmt.Errorf("invalid artifact file name: %q", artifactVersionFile)
 	}
 	return match[1], nil
 }
 
-func writeJSON(filename string, data interface{}) error {
-	var out bytes.Buffer
-	enc := json.NewEncoder(&out)
-	enc.SetEscapeHTML(false)
-	enc.SetIndent("", "  ")
-	err := enc.Encode(data)
-	if err != nil {
-		return fmt.Errorf("failed to encode data: %w", err)
+func resetDirectory(dir string) error {
+	if err := os.RemoveAll(dir); err != nil {
+		return fmt.Errorf("failed to remove directory %q: %w", dir, err)
 	}
-	jsonData := out.Bytes()
-	if len(jsonData) > 0 && jsonData[len(jsonData)-1] == '\n' { // strip newline
-		jsonData = jsonData[:len(jsonData)-1]
-	}
-	if err := os.WriteFile(filename, jsonData, 0644); err != nil {
-		return fmt.Errorf("failed to write file: %w", err)
+	if err := os.MkdirAll(dir, os.ModePerm); err != nil {
+		return fmt.Errorf("failed to create directory %q: %w", dir, err)
 	}
 	return nil
 }
-
-func jsonEqual(a, b interface{}) bool {
-	jsonA, errA := json.Marshal(a)
-	jsonB, errB := json.Marshal(b)
-	return errA == nil && errB == nil && string(jsonA) == string(jsonB)
-}

packages/contracts-bedrock/scripts/checks/common/util.go

 package common
 
 import (
+	"bytes"
 	"encoding/json"
 	"fmt"
 	"os"
-	"path/filepath"
 	"runtime"
 	"sync"
 	"sync/atomic"
@@ -100,8 +100,7 @@ func FindFiles(includes, excludes []string) (map[string]string, error) {
 			return nil, fmt.Errorf("glob pattern error: %w", err)
 		}
 		for _, match := range matches {
-			name := filepath.Base(match)
-			included[name] = match
+			included[match] = match
 		}
 	}
 
@@ -112,7 +111,7 @@ func FindFiles(includes, excludes []string) (map[string]string, error) {
 			return nil, fmt.Errorf("glob pattern error: %w", err)
 		}
 		for _, match := range matches {
-			excluded[filepath.Base(match)] = struct{}{}
+			excluded[match] = struct{}{}
 		}
 	}
 
@@ -137,3 +136,22 @@ func ReadForgeArtifact(path string) (*solc.ForgeArtifact, error) {
 
 	return &artifact, nil
 }
+
+func WriteJSON(data interface{}, path string) error {
+	var out bytes.Buffer
+	enc := json.NewEncoder(&out)
+	enc.SetEscapeHTML(false)
+	enc.SetIndent("", "  ")
+	err := enc.Encode(data)
+	if err != nil {
+		return fmt.Errorf("failed to encode data: %w", err)
+	}
+	jsonData := out.Bytes()
+	if len(jsonData) > 0 && jsonData[len(jsonData)-1] == '\n' { // strip newline
+		jsonData = jsonData[:len(jsonData)-1]
+	}
+	if err := os.WriteFile(path, jsonData, 0644); err != nil {
+		return fmt.Errorf("failed to write file: %w", err)
+	}
+	return nil
+}

packages/contracts-bedrock/scripts/checks/test-names/main.go

@@ -41,7 +41,7 @@ func processFile(path string) (*common.Void, []error) {
 
 func extractTestNames(artifact *solc.ForgeArtifact) []string {
 	isTest := false
-	for _, entry := range artifact.Abi.Methods {
+	for _, entry := range artifact.Abi.Parsed.Methods {
 		if entry.Name == "IS_TEST" {
 			isTest = true
 			break
@@ -52,7 +52,7 @@ func extractTestNames(artifact *solc.ForgeArtifact) []string {
 	}
 
 	names := []string{}
-	for _, entry := range artifact.Abi.Methods {
+	for _, entry := range artifact.Abi.Parsed.Methods {
 		if !strings.HasPrefix(entry.Name, "test") {
 			continue
 		}

packages/contracts-bedrock/scripts/checks/test-names/main_test.go

@@ -218,7 +218,8 @@ func TestExtractTestNames(t *testing.T) {
 		{
 			name: "valid test contract",
 			artifact: &solc.ForgeArtifact{
-				Abi: abi.ABI{
+				Abi: solc.AbiType{
+					Parsed: abi.ABI{
 						Methods: map[string]abi.Method{
 							"IS_TEST":                  {Name: "IS_TEST"},
 							"test_something_succeeds":  {Name: "test_something_succeeds"},
@@ -228,6 +229,7 @@ func TestExtractTestNames(t *testing.T) {
 						},
 					},
 				},
+			},
 			want: []string{
 				"test_something_succeeds",
 				"test_other_fails",
@@ -237,28 +239,33 @@ func TestExtractTestNames(t *testing.T) {
 		{
 			name: "non-test contract",
 			artifact: &solc.ForgeArtifact{
-				Abi: abi.ABI{
+				Abi: solc.AbiType{
+					Parsed: abi.ABI{
 						Methods: map[string]abi.Method{
 							"test_something_succeeds": {Name: "test_something_succeeds"},
 							"not_a_test":              {Name: "not_a_test"},
 						},
 					},
 				},
+			},
 			want: nil,
 		},
 		{
 			name: "empty contract",
 			artifact: &solc.ForgeArtifact{
-				Abi: abi.ABI{
+				Abi: solc.AbiType{
+					Parsed: abi.ABI{
 						Methods: map[string]abi.Method{},
 					},
 				},
+			},
 			want: nil,
 		},
 		{
 			name: "test contract with no test methods",
 			artifact: &solc.ForgeArtifact{
-				Abi: abi.ABI{
+				Abi: solc.AbiType{
+					Parsed: abi.ABI{
 						Methods: map[string]abi.Method{
 							"IS_TEST":        {Name: "IS_TEST"},
 							"not_a_test":     {Name: "not_a_test"},
@@ -266,6 +273,7 @@ func TestExtractTestNames(t *testing.T) {
 						},
 					},
 				},
+			},
 			want: []string{},
 		},
 	}