op-program: kv store for pre-images

op-program/host/kvstore/disk.go

+package kvstore
+
+import (
+	"encoding/hex"
+	"fmt"
+	"io"
+	"os"
+	"path"
+
+	"github.com/ethereum/go-ethereum/common"
+)
+
+// read/write mode for user/group/other, not executable.
+const diskPermission = 0666
+
+// DiskKV is a disk-backed key-value store, every key-value pair is a hex-encoded .txt file, with the value as content.
+// DiskKV is safe for concurrent use; Puts may conflict, but write the exact same data anyway.
+type DiskKV struct {
+	path string
+}
+
+// NewDiskKV creates a DiskKV that puts/gets pre-images as files in the given directory path.
+// The path must exist, or subsequent Put/Get calls will error when it does not.
+func NewDiskKV(path string) *DiskKV {
+	return &DiskKV{path: path}
+}
+
+func (d *DiskKV) pathKey(k common.Hash) string {
+	return path.Join(d.path, k.String()+".txt")
+}
+
+func (d *DiskKV) Put(k common.Hash, v []byte) error {
+	// no O_EXCL, the pre-image may already exist. It's fine to overwrite it.
+	f, err := os.OpenFile(d.pathKey(k), os.O_WRONLY|os.O_CREATE|os.O_TRUNC, diskPermission)
+	if err != nil {
+		return fmt.Errorf("failed to open new pre-image file %s: %w", k, err)
+	}
+	if _, err := f.Write([]byte(hex.EncodeToString(v))); err != nil {
+		_ = f.Close()
+		return fmt.Errorf("failed to write pre-image %s to disk: %w", k, err)
+	}
+	if err := f.Close(); err != nil {
+		return fmt.Errorf("failed to close pre-image %s file: %w", k, err)
+	}
+	return nil
+}
+
+func (d *DiskKV) Get(k common.Hash) ([]byte, error) {
+	f, err := os.OpenFile(d.pathKey(k), os.O_RDONLY, diskPermission)
+	if err != nil {
+		if os.IsNotExist(err) {
+			return nil, NotFoundErr
+		}
+		return nil, fmt.Errorf("failed to open pre-image file %s: %w", k, err)
+	}
+	defer f.Close() // fine to ignore closing error here
+	dat, err := io.ReadAll(f)
+	if err != nil {
+		return nil, fmt.Errorf("failed to read pre-image from file %s: %w", k, err)
+	}
+	return hex.DecodeString(string(dat))
+}
+
+var _ KV = (*DiskKV)(nil)

op-program/host/kvstore/disk_test.go

+package kvstore
+
+import "testing"
+
+func TestDiskKV(t *testing.T) {
+	tmp := t.TempDir() // automatically removed by testing cleanup
+	kv := NewDiskKV(tmp)
+	kvTest(t, kv)
+}

op-program/host/kvstore/kv.go

+package kvstore
+
+import (
+	"errors"
+
+	"github.com/ethereum/go-ethereum/common"
+)
+
+// NotFoundErr is returned when a pre-image cannot be found in the KV store.
+var NotFoundErr = errors.New("not found")
+
+// KV is a Key-Value store interface for pre-image data.
+type KV interface {
+	Put(k common.Hash, v []byte) error
+	Get(k common.Hash) ([]byte, error)
+}

op-program/host/kvstore/kv_test.go

+package kvstore
+
+import (
+	"testing"
+
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/stretchr/testify/require"
+)
+
+func kvTest(t *testing.T, kv KV) {
+	t.Run("roundtrip", func(t *testing.T) {
+		t.Parallel()
+		_, err := kv.Get(common.Hash{0xaa})
+		require.Equal(t, err, NotFoundErr, "file (in new tmp dir) does not exist yet")
+
+		require.NoError(t, kv.Put(common.Hash{0xaa}, []byte("hello world")))
+		dat, err := kv.Get(common.Hash{0xaa})
+		require.NoError(t, err, "pre-image must exist now")
+		require.Equal(t, "hello world", string(dat), "pre-image must match")
+	})
+
+	t.Run("empty pre-image", func(t *testing.T) {
+		t.Parallel()
+		require.NoError(t, kv.Put(common.Hash{0xbb}, []byte{}))
+		dat, err := kv.Get(common.Hash{0xbb})
+		require.NoError(t, err, "pre-image must exist now")
+		require.Zero(t, len(dat), "pre-image must be empty")
+	})
+
+	t.Run("zero pre-image key", func(t *testing.T) {
+		t.Parallel()
+		// in case we give a pre-image a special empty key. If it was a hash then we wouldn't know the pre-image.
+		require.NoError(t, kv.Put(common.Hash{}, []byte("hello")))
+		dat, err := kv.Get(common.Hash{})
+		require.NoError(t, err, "pre-image must exist now")
+		require.Equal(t, "hello", string(dat), "pre-image must match")
+	})
+
+	t.Run("non-string value", func(t *testing.T) {
+		t.Parallel()
+		// in case we give a pre-image a special empty key. If it was a hash then we wouldn't know the pre-image.
+		require.NoError(t, kv.Put(common.Hash{0xcc}, []byte{4, 2}))
+		dat, err := kv.Get(common.Hash{0xcc})
+		require.NoError(t, err, "pre-image must exist now")
+		require.Equal(t, []byte{4, 2}, dat, "pre-image must match")
+	})
+}

op-program/host/kvstore/mem.go

+package kvstore
+
+import (
+	"sync"
+
+	"github.com/ethereum/go-ethereum/common"
+)
+
+// MemKV implements the KV store interface in memory, backed by a regular Go map.
+// This should only be used in testing, as large programs may require more pre-image data than available memory.
+// MemKV is safe for concurrent use.
+type MemKV struct {
+	sync.RWMutex
+	m map[common.Hash][]byte
+}
+
+var _ KV = (*MemKV)(nil)
+
+func NewMemKV() *MemKV {
+	return &MemKV{m: make(map[common.Hash][]byte)}
+}
+
+func (m *MemKV) Put(k common.Hash, v []byte) error {
+	m.Lock()
+	defer m.Unlock()
+	m.m[k] = v
+	return nil
+}
+
+func (m *MemKV) Get(k common.Hash) ([]byte, error) {
+	m.RLock()
+	defer m.RUnlock()
+	v, ok := m.m[k]
+	if !ok {
+		return nil, NotFoundErr
+	}
+	return v, nil
+}

op-program/host/kvstore/mem_test.go

+package kvstore
+
+import "testing"
+
+func TestMemKV(t *testing.T) {
+	kv := NewMemKV()
+	kvTest(t, kv)
+}