op-challenger: Use gzip encoded proofs and snapshots

cannon/cmd/json.go

 package cmd
 
 import (
-	"compress/gzip"
 	"encoding/json"
 	"errors"
 	"fmt"
 	"io"
 	"os"
-	"strings"
+
+	"github.com/ethereum-optimism/optimism/op-service/ioutil"
 )
 
 func loadJSON[X any](inputPath string) (*X, error) {
@@ -15,18 +15,11 @@ func loadJSON[X any](inputPath string) (*X, error) {
 		return nil, errors.New("no path specified")
 	}
 	var f io.ReadCloser
-	f, err := os.OpenFile(inputPath, os.O_RDONLY, 0)
+	f, err := ioutil.OpenDecompressed(inputPath)
 	if err != nil {
 		return nil, fmt.Errorf("failed to open file %q: %w", inputPath, err)
 	}
 	defer f.Close()
-	if isGzip(inputPath) {
-		f, err = gzip.NewReader(f)
-		if err != nil {
-			return nil, fmt.Errorf("create gzip reader: %w", err)
-		}
-		defer f.Close()
-	}
 	var state X
 	if err := json.NewDecoder(f).Decode(&state); err != nil {
 		return nil, fmt.Errorf("failed to decode file %q: %w", inputPath, err)
@@ -37,17 +30,12 @@ func loadJSON[X any](inputPath string) (*X, error) {
 func writeJSON[X any](outputPath string, value X, outIfEmpty bool) error {
 	var out io.Writer
 	if outputPath != "" {
-		f, err := os.OpenFile(outputPath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0755)
+		f, err := ioutil.OpenCompressed(outputPath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0755)
 		if err != nil {
 			return fmt.Errorf("failed to open output file: %w", err)
 		}
 		defer f.Close()
 		out = f
-		if isGzip(outputPath) {
-			g := gzip.NewWriter(f)
-			defer g.Close()
-			out = g
-		}
 	} else if outIfEmpty {
 		out = os.Stdout
 	} else {
@@ -63,7 +51,3 @@ func writeJSON[X any](outputPath string, value X, outIfEmpty bool) error {
 	}
 	return nil
 }
-
-func isGzip(path string) bool {
-	return strings.HasSuffix(path, ".gz")
-}

op-challenger/game/fault/trace/cannon/cannon_state.go

@@ -3,13 +3,13 @@ package cannon
 import (
 	"encoding/json"
 	"fmt"
-	"os"
 
 	"github.com/ethereum-optimism/optimism/cannon/mipsevm"
+	"github.com/ethereum-optimism/optimism/op-service/ioutil"
 )
 
 func parseState(path string) (*mipsevm.State, error) {
-	file, err := os.Open(path)
+	file, err := ioutil.OpenDecompressed(path)
 	if err != nil {
 		return nil, fmt.Errorf("cannot open state file (%v): %w", path, err)
 	}

op-challenger/game/fault/trace/cannon/cannon_state_test.go

+package cannon
+
+import (
+	"compress/gzip"
+	_ "embed"
+	"encoding/json"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/ethereum-optimism/optimism/cannon/mipsevm"
+	"github.com/stretchr/testify/require"
+)
+
+//go:embed test_data/state.json
+var testState []byte
+
+func TestLoadState(t *testing.T) {
+	t.Run("Uncompressed", func(t *testing.T) {
+		dir := t.TempDir()
+		path := filepath.Join(dir, "state.json")
+		require.NoError(t, os.WriteFile(path, testState, 0644))
+
+		state, err := parseState(path)
+		require.NoError(t, err)
+
+		var expected mipsevm.State
+		require.NoError(t, json.Unmarshal(testState, &expected))
+		require.Equal(t, &expected, state)
+	})
+
+	t.Run("Gzipped", func(t *testing.T) {
+		dir := t.TempDir()
+		path := filepath.Join(dir, "state.json.gz")
+		f, err := os.OpenFile(path, os.O_WRONLY|os.O_TRUNC|os.O_CREATE, 0644)
+		require.NoError(t, err)
+		defer f.Close()
+		writer := gzip.NewWriter(f)
+		_, err = writer.Write(testState)
+		require.NoError(t, err)
+		require.NoError(t, writer.Close())
+
+		state, err := parseState(path)
+		require.NoError(t, err)
+
+		var expected mipsevm.State
+		require.NoError(t, json.Unmarshal(testState, &expected))
+		require.Equal(t, &expected, state)
+	})
+}

op-challenger/game/fault/trace/cannon/executor.go

@@ -20,10 +20,10 @@ import (
 const (
 	snapsDir     = "snapshots"
 	preimagesDir = "preimages"
-	finalState   = "final.json"
+	finalState   = "final.json.gz"
 )
 
-var snapshotNameRegexp = regexp.MustCompile(`^[0-9]+\.json$`)
+var snapshotNameRegexp = regexp.MustCompile(`^[0-9]+\.json.gz$`)
 
 type snapshotSelect func(logger log.Logger, dir string, absolutePreState string, i uint64) (string, error)
 type cmdExecutor func(ctx context.Context, l log.Logger, binary string, args ...string) error
@@ -77,9 +77,9 @@ func (e *Executor) GenerateProof(ctx context.Context, dir string, i uint64) erro
 		"--output", lastGeneratedState,
 		"--meta", "",
 		"--proof-at", "=" + strconv.FormatUint(i, 10),
-		"--proof-fmt", filepath.Join(proofDir, "%d.json"),
+		"--proof-fmt", filepath.Join(proofDir, "%d.json.gz"),
 		"--snapshot-at", "%" + strconv.FormatUint(uint64(e.snapshotFreq), 10),
-		"--snapshot-fmt", filepath.Join(snapshotDir, "%d.json"),
+		"--snapshot-fmt", filepath.Join(snapshotDir, "%d.json.gz"),
 	}
 	if i < math.MaxUint64 {
 		args = append(args, "--stop-at", "="+strconv.FormatUint(i+1, 10))
@@ -153,7 +153,7 @@ func findStartingSnapshot(logger log.Logger, snapDir string, absolutePreState st
 			logger.Warn("Unexpected file in snapshots dir", "parent", snapDir, "child", entry.Name())
 			continue
 		}
-		index, err := strconv.ParseUint(name[0:len(name)-len(".json")], 10, 64)
+		index, err := strconv.ParseUint(name[0:len(name)-len(".json.gz")], 10, 64)
 		if err != nil {
 			logger.Error("Unable to parse trace index of snapshot file", "parent", snapDir, "child", entry.Name())
 			continue
@@ -165,7 +165,7 @@ func findStartingSnapshot(logger log.Logger, snapDir string, absolutePreState st
 	if bestSnap == 0 {
 		return absolutePreState, nil
 	}
-	startFrom := fmt.Sprintf("%v/%v.json", snapDir, bestSnap)
+	startFrom := fmt.Sprintf("%v/%v.json.gz", snapDir, bestSnap)
 
 	return startFrom, nil
 }

op-challenger/game/fault/trace/cannon/executor_test.go

@@ -88,8 +88,8 @@ func TestGenerateProof(t *testing.T) {
 		require.Equal(t, cfg.L1EthRpc, args["--l1"])
 		require.Equal(t, cfg.CannonL2, args["--l2"])
 		require.Equal(t, filepath.Join(dir, preimagesDir), args["--datadir"])
-		require.Equal(t, filepath.Join(dir, proofsDir, "%d.json"), args["--proof-fmt"])
-		require.Equal(t, filepath.Join(dir, snapsDir, "%d.json"), args["--snapshot-fmt"])
+		require.Equal(t, filepath.Join(dir, proofsDir, "%d.json.gz"), args["--proof-fmt"])
+		require.Equal(t, filepath.Join(dir, snapsDir, "%d.json.gz"), args["--snapshot-fmt"])
 		require.Equal(t, cfg.CannonNetwork, args["--network"])
 		require.NotContains(t, args, "--rollup.config")
 		require.NotContains(t, args, "--l2.genesis")
@@ -174,37 +174,37 @@ func TestFindStartingSnapshot(t *testing.T) {
 	})
 
 	t.Run("UseClosestAvailableSnapshot", func(t *testing.T) {
-		dir := withSnapshots(t, "100.json", "123.json", "250.json")
+		dir := withSnapshots(t, "100.json.gz", "123.json.gz", "250.json.gz")
 
 		snapshot, err := findStartingSnapshot(logger, dir, execTestCannonPrestate, 101)
 		require.NoError(t, err)
-		require.Equal(t, filepath.Join(dir, "100.json"), snapshot)
+		require.Equal(t, filepath.Join(dir, "100.json.gz"), snapshot)
 
 		snapshot, err = findStartingSnapshot(logger, dir, execTestCannonPrestate, 123)
 		require.NoError(t, err)
-		require.Equal(t, filepath.Join(dir, "100.json"), snapshot)
+		require.Equal(t, filepath.Join(dir, "100.json.gz"), snapshot)
 
 		snapshot, err = findStartingSnapshot(logger, dir, execTestCannonPrestate, 124)
 		require.NoError(t, err)
-		require.Equal(t, filepath.Join(dir, "123.json"), snapshot)
+		require.Equal(t, filepath.Join(dir, "123.json.gz"), snapshot)
 
 		snapshot, err = findStartingSnapshot(logger, dir, execTestCannonPrestate, 256)
 		require.NoError(t, err)
-		require.Equal(t, filepath.Join(dir, "250.json"), snapshot)
+		require.Equal(t, filepath.Join(dir, "250.json.gz"), snapshot)
 	})
 
 	t.Run("IgnoreDirectories", func(t *testing.T) {
-		dir := withSnapshots(t, "100.json")
-		require.NoError(t, os.Mkdir(filepath.Join(dir, "120.json"), 0o777))
+		dir := withSnapshots(t, "100.json.gz")
+		require.NoError(t, os.Mkdir(filepath.Join(dir, "120.json.gz"), 0o777))
 		snapshot, err := findStartingSnapshot(logger, dir, execTestCannonPrestate, 150)
 		require.NoError(t, err)
-		require.Equal(t, filepath.Join(dir, "100.json"), snapshot)
+		require.Equal(t, filepath.Join(dir, "100.json.gz"), snapshot)
 	})
 
 	t.Run("IgnoreUnexpectedFiles", func(t *testing.T) {
-		dir := withSnapshots(t, ".file", "100.json", "foo", "bar.json")
+		dir := withSnapshots(t, ".file", "100.json.gz", "foo", "bar.json.gz")
 		snapshot, err := findStartingSnapshot(logger, dir, execTestCannonPrestate, 150)
 		require.NoError(t, err)
-		require.Equal(t, filepath.Join(dir, "100.json"), snapshot)
+		require.Equal(t, filepath.Join(dir, "100.json.gz"), snapshot)
 	})
 }

op-challenger/game/fault/trace/cannon/provider.go

@@ -11,6 +11,7 @@ import (
 	"github.com/ethereum-optimism/optimism/op-bindings/bindings"
 	"github.com/ethereum-optimism/optimism/op-challenger/config"
 	"github.com/ethereum-optimism/optimism/op-challenger/game/fault/types"
+	"github.com/ethereum-optimism/optimism/op-service/ioutil"
 	"github.com/ethereum/go-ethereum/accounts/abi/bind"
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/common/hexutil"
@@ -124,14 +125,14 @@ func (p *CannonTraceProvider) loadProof(ctx context.Context, i uint64) (*proofDa
 		// If the requested index is after the last step in the actual trace, extend the final no-op step
 		return p.lastProof, nil
 	}
-	path := filepath.Join(p.dir, proofsDir, fmt.Sprintf("%d.json", i))
-	file, err := os.Open(path)
+	path := filepath.Join(p.dir, proofsDir, fmt.Sprintf("%d.json.gz", i))
+	file, err := ioutil.OpenDecompressed(path)
 	if errors.Is(err, os.ErrNotExist) {
 		if err := p.generator.GenerateProof(ctx, p.dir, i); err != nil {
 			return nil, fmt.Errorf("generate cannon trace with proof at %v: %w", i, err)
 		}
 		// Try opening the file again now and it should exist.
-		file, err = os.Open(path)
+		file, err = ioutil.OpenDecompressed(path)
 		if errors.Is(err, os.ErrNotExist) {
 			// Expected proof wasn't generated, check if we reached the end of execution
 			state, err := parseState(filepath.Join(p.dir, finalState))

op-challenger/game/fault/trace/cannon/provider_test.go

@@ -13,6 +13,7 @@ import (
 	"github.com/ethereum-optimism/optimism/cannon/mipsevm"
 	"github.com/ethereum-optimism/optimism/op-challenger/game/fault/types"
 	"github.com/ethereum-optimism/optimism/op-node/testlog"
+	"github.com/ethereum-optimism/optimism/op-service/ioutil"
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/crypto"
 	"github.com/ethereum/go-ethereum/log"
@@ -207,7 +208,7 @@ func setupTestData(t *testing.T) (string, string) {
 		path := filepath.Join(srcDir, entry.Name())
 		file, err := testData.ReadFile(path)
 		require.NoErrorf(t, err, "reading %v", path)
-		err = os.WriteFile(filepath.Join(dataDir, proofsDir, entry.Name()), file, 0o644)
+		err = writeGzip(filepath.Join(dataDir, proofsDir, entry.Name()+".gz"), file)
 		require.NoErrorf(t, err, "writing %v", path)
 	}
 	return dataDir, "state.json"
@@ -237,15 +238,25 @@ func (e *stubGenerator) GenerateProof(ctx context.Context, dir string, i uint64)
 		if err != nil {
 			return err
 		}
-		return os.WriteFile(filepath.Join(dir, finalState), data, 0644)
+		return writeGzip(filepath.Join(dir, finalState), data)
 	}
 	if e.proof != nil {
-		proofFile := filepath.Join(dir, proofsDir, fmt.Sprintf("%d.json", i))
+		proofFile := filepath.Join(dir, proofsDir, fmt.Sprintf("%d.json.gz", i))
 		data, err := json.Marshal(e.proof)
 		if err != nil {
 			return err
 		}
-		return os.WriteFile(proofFile, data, 0644)
+		return writeGzip(proofFile, data)
 	}
 	return nil
 }
+
+func writeGzip(path string, data []byte) error {
+	writer, err := ioutil.OpenCompressed(path, os.O_WRONLY|os.O_TRUNC|os.O_CREATE, 0o644)
+	if err != nil {
+		return err
+	}
+	defer writer.Close()
+	_, err = writer.Write(data)
+	return err
+}

op-service/ioutil/gzip.go

+package ioutil
+
+import (
+	"compress/gzip"
+	"fmt"
+	"io"
+	"os"
+	"strings"
+)
+
+// OpenDecompressed opens a reader for the specified file and automatically gzip decompresses the content
+// if the filename ends with .gz
+func OpenDecompressed(path string) (io.ReadCloser, error) {
+	var r io.ReadCloser
+	r, err := os.Open(path)
+	if err != nil {
+		return nil, err
+	}
+	if IsGzip(path) {
+		r, err = gzip.NewReader(r)
+		if err != nil {
+			return nil, fmt.Errorf("failed to create gzip reader: %w", err)
+		}
+	}
+	return r, nil
+}
+
+// OpenCompressed opens a file for writing and automatically compresses the content if the filename ends with .gz
+func OpenCompressed(file string, flag int, perm os.FileMode) (io.WriteCloser, error) {
+	var out io.WriteCloser
+	out, err := os.OpenFile(file, flag, perm)
+	if err != nil {
+		return nil, err
+	}
+	if IsGzip(file) {
+		out = gzip.NewWriter(out)
+	}
+	return out, nil
+}
+
+// IsGzip determines if a path points to a gzip compressed file.
+// Returns true when the file has a .gz extension.
+func IsGzip(path string) bool {
+	return strings.HasSuffix(path, ".gz")
+}

op-service/ioutil/gzip_test.go

+package ioutil
+
+import (
+	"io"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestReadWriteWithOptionalCompression(t *testing.T) {
+	tests := []struct {
+		name       string
+		filename   string
+		compressed bool
+	}{
+		{"Uncompressed", "test.notgz", false},
+		{"Gzipped", "test.gz", true},
+	}
+	for _, test := range tests {
+		test := test
+		t.Run(test.name, func(t *testing.T) {
+			data := []byte{1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 0, 0, 0, 0, 0, 0, 0}
+			dir := t.TempDir()
+			path := filepath.Join(dir, test.filename)
+			out, err := OpenCompressed(path, os.O_WRONLY|os.O_TRUNC|os.O_CREATE, 0o644)
+			require.NoError(t, err)
+			defer out.Close()
+			_, err = out.Write(data)
+			require.NoError(t, err)
+			require.NoError(t, out.Close())
+
+			writtenData, err := os.ReadFile(path)
+			require.NoError(t, err)
+			if test.compressed {
+				require.NotEqual(t, data, writtenData, "should have compressed data on disk")
+			} else {
+				require.Equal(t, data, writtenData, "should not have compressed data on disk")
+			}
+
+			in, err := OpenDecompressed(path)
+			require.NoError(t, err)
+			readData, err := io.ReadAll(in)
+			require.NoError(t, err)
+			require.Equal(t, data, readData)
+		})
+	}
+}