feat: Add nested directory data format for op-program kvstore (#11795)

* feat: Add nested directory data format for op-program kvstore

* Review feedback: Update error message
Co-authored-by: Adrian Sutton <adrian@symphonious.net>

* op-program: Make directory preimage format the default.

---------
Co-authored-by: Adrian Sutton <adrian@symphonious.net>
Co-authored-by: Adrian Sutton <adrian@oplabs.co>

op-program/host/config/config.go

@@ -139,7 +139,7 @@ func NewConfig(
 		L2ClaimBlockNumber:  l2ClaimBlockNum,
 		L1RPCKind:           sources.RPCKindStandard,
 		IsCustomChainConfig: isCustomConfig,
-		DataFormat:          types.DataFormatFile,
+		DataFormat:          types.DataFormatDirectory,
 	}
 }
 

op-program/host/flags/flags.go

@@ -40,7 +40,7 @@ var (
 		Name:    "data.format",
 		Usage:   fmt.Sprintf("Format to use for preimage data storage. Available formats: %s", openum.EnumString(types.SupportedDataFormats)),
 		EnvVars: prefixEnvVars("DATA_FORMAT"),
-		Value:   string(types.DataFormatFile),
+		Value:   string(types.DataFormatDirectory),
 	}
 	L2NodeAddr = &cli.StringFlag{
 		Name:    "l2",

op-program/host/host.go

@@ -158,6 +158,8 @@ func PreimageServer(ctx context.Context, logger log.Logger, cfg *config.Config,
 		switch cfg.DataFormat {
 		case types.DataFormatFile:
 			kv = kvstore.NewFileKV(cfg.DataDir)
+		case types.DataFormatDirectory:
+			kv = kvstore.NewDirectoryKV(cfg.DataDir)
 		case types.DataFormatPebble:
 			kv = kvstore.NewPebbleKV(cfg.DataDir)
 		default:

op-program/host/kvstore/directory.go

+package kvstore
+
+import (
+	"encoding/hex"
+	"errors"
+	"fmt"
+	"io"
+	"os"
+	"path"
+	"sync"
+
+	"github.com/ethereum/go-ethereum/common"
+)
+
+// DirectoryKV is a disk-backed key-value store, every key-value pair is a hex-encoded .txt file, with the value as content.
+// DirectoryKV is safe for concurrent use with a single DirectoryKV instance.
+// DirectoryKV is safe for concurrent use between different DirectoryKV instances of the same disk directory as long as the
+// file system supports atomic renames.
+type DirectoryKV struct {
+	sync.RWMutex
+	path string
+}
+
+// NewDirectoryKV creates a DirectoryKV that puts/gets pre-images as files in the given directory path.
+// The path must exist, or subsequent Put/Get calls will error when it does not.
+func NewDirectoryKV(path string) *DirectoryKV {
+	return &DirectoryKV{path: path}
+}
+
+// pathKey returns the file path for the given key.
+// This is composed of the first characters of the non-0x-prefixed hex key as a directory, and the rest as the file name.
+func (d *DirectoryKV) pathKey(k common.Hash) string {
+	key := k.String()
+	dir, name := key[2:6], key[6:]
+	return path.Join(d.path, dir, name+".txt")
+}
+
+func (d *DirectoryKV) Put(k common.Hash, v []byte) error {
+	d.Lock()
+	defer d.Unlock()
+	f, err := openTempFile(d.path, k.String()+".txt.*")
+	if err != nil {
+		return fmt.Errorf("failed to open temp file for pre-image %s: %w", k, err)
+	}
+	defer os.Remove(f.Name()) // Clean up the temp file if it doesn't actually get moved into place
+	if _, err := f.Write([]byte(hex.EncodeToString(v))); err != nil {
+		_ = f.Close()
+		return fmt.Errorf("failed to write pre-image %s to disk: %w", k, err)
+	}
+	if err := f.Close(); err != nil {
+		return fmt.Errorf("failed to close temp pre-image %s file: %w", k, err)
+	}
+
+	targetFile := d.pathKey(k)
+	if err := os.MkdirAll(path.Dir(targetFile), 0777); err != nil {
+		return fmt.Errorf("failed to create parent directory for pre-image %s: %w", f.Name(), err)
+	}
+	if err := os.Rename(f.Name(), targetFile); err != nil {
+		return fmt.Errorf("failed to move temp file %v to final destination %v: %w", f.Name(), targetFile, err)
+	}
+	return nil
+}
+
+func (d *DirectoryKV) Get(k common.Hash) ([]byte, error) {
+	d.RLock()
+	defer d.RUnlock()
+	f, err := os.OpenFile(d.pathKey(k), os.O_RDONLY, filePermission)
+	if err != nil {
+		if errors.Is(err, os.ErrNotExist) {
+			return nil, ErrNotFound
+		}
+		return nil, fmt.Errorf("failed to open pre-image file %s: %w", k, err)
+	}
+	defer f.Close() // fine to ignore closing error here
+	dat, err := io.ReadAll(f)
+	if err != nil {
+		return nil, fmt.Errorf("failed to read pre-image from file %s: %w", k, err)
+	}
+	return hex.DecodeString(string(dat))
+}
+
+func (d *DirectoryKV) Close() error {
+	return nil
+}
+
+var _ KV = (*DirectoryKV)(nil)

op-program/host/kvstore/directory_test.go

+package kvstore
+
+import (
+	"path/filepath"
+	"testing"
+
+	"github.com/ethereum/go-ethereum/crypto"
+	"github.com/stretchr/testify/require"
+)
+
+func TestDirectoryKV(t *testing.T) {
+	tmp := t.TempDir() // automatically removed by testing cleanup
+	kv := NewDirectoryKV(tmp)
+	t.Cleanup(func() { // Can't use defer because kvTest runs tests in parallel.
+		require.NoError(t, kv.Close())
+	})
+	kvTest(t, kv)
+}
+
+func TestDirectoryKV_CreateMissingDirectory(t *testing.T) {
+	tmp := t.TempDir()
+	dir := filepath.Join(tmp, "data")
+	kv := NewDirectoryKV(dir)
+	defer kv.Close()
+	val := []byte{1, 2, 3, 4}
+	key := crypto.Keccak256Hash(val)
+	require.NoError(t, kv.Put(key, val))
+}

op-program/host/kvstore/file_test.go

@@ -8,7 +8,7 @@ import (
 	"github.com/stretchr/testify/require"
 )
 
-func TestDiskKV(t *testing.T) {
+func TestFileKV(t *testing.T) {
 	tmp := t.TempDir() // automatically removed by testing cleanup
 	kv := NewFileKV(tmp)
 	t.Cleanup(func() { // Can't use defer because kvTest runs tests in parallel.
@@ -17,7 +17,7 @@ func TestDiskKV(t *testing.T) {
 	kvTest(t, kv)
 }
 
-func TestCreateMissingDirectory(t *testing.T) {
+func TestFileKV_CreateMissingDirectory(t *testing.T) {
 	tmp := t.TempDir()
 	dir := filepath.Join(tmp, "data")
 	kv := NewFileKV(dir)

op-program/host/types/types.go

@@ -3,8 +3,9 @@ package types
 type DataFormat string
 
 const (
-	DataFormatFile   DataFormat = "file"
-	DataFormatPebble DataFormat = "pebble"
+	DataFormatFile      DataFormat = "file"
+	DataFormatDirectory DataFormat = "directory"
+	DataFormatPebble    DataFormat = "pebble"
 )
 
-var SupportedDataFormats = []DataFormat{DataFormatFile, DataFormatPebble}
+var SupportedDataFormats = []DataFormat{DataFormatFile, DataFormatDirectory, DataFormatPebble}