op-plasma: basic da server with S3 and leveldb storage (#9813)

* feat: basic plasma da server

* catch all not found errors

* feat: switch to minio and file server

* add file

* fix: handle feedback

* fix: tidy

* fix: tidy mods again

* chore: add S3 config info

go.mod

@@ -4,6 +4,10 @@ go 1.21
 
 require (
 	github.com/BurntSushi/toml v1.3.2
+	github.com/aws/aws-sdk-go-v2 v1.26.1
+	github.com/aws/aws-sdk-go-v2/config v1.18.45
+	github.com/aws/aws-sdk-go-v2/service/s3 v1.53.1
+	github.com/aws/smithy-go v1.20.2
 	github.com/btcsuite/btcd v0.24.0
 	github.com/btcsuite/btcd/chaincfg/chainhash v1.1.0
 	github.com/cockroachdb/pebble v0.0.0-20231018212520-f6cde3fc2fa4
@@ -65,6 +69,20 @@ require (
 	github.com/allegro/bigcache v1.2.1 // indirect
 	github.com/andybalholm/brotli v1.0.5 // indirect
 	github.com/armon/go-metrics v0.4.1 // indirect
+	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.2 // indirect
+	github.com/aws/aws-sdk-go-v2/credentials v1.13.43 // indirect
+	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.13 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.5 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.5 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/ini v1.3.45 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.5 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.2 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.7 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.7 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.5 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sso v1.15.2 // indirect
+	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.17.3 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sts v1.23.2 // indirect
 	github.com/aymerick/douceur v0.2.0 // indirect
 	github.com/benbjohnson/clock v1.3.5 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
@@ -151,7 +169,7 @@ require (
 	github.com/kataras/pio v0.0.12 // indirect
 	github.com/kataras/sitemap v0.0.6 // indirect
 	github.com/kataras/tunnel v0.0.4 // indirect
-	github.com/klauspost/compress v1.17.2 // indirect
+	github.com/klauspost/compress v1.17.6 // indirect
 	github.com/klauspost/cpuid/v2 v2.2.6 // indirect
 	github.com/koron/go-ssdp v0.0.4 // indirect
 	github.com/kr/pretty v0.3.1 // indirect

go.sum

@@ -48,6 +48,47 @@ github.com/armon/go-metrics v0.0.0-20190430140413-ec5e00d3c878/go.mod h1:3AMJUQh
 github.com/armon/go-metrics v0.3.8/go.mod h1:4O98XIr/9W0sxpJ8UaYkvjk10Iff7SnFrb4QAOwNTFc=
 github.com/armon/go-metrics v0.4.1 h1:hR91U9KYmb6bLBYLQjyM+3j+rcd/UhE+G78SFnF8gJA=
 github.com/armon/go-metrics v0.4.1/go.mod h1:E6amYzXo6aW1tqzoZGT755KkbgrJsSdpwZ+3JqfkOG4=
+github.com/aws/aws-sdk-go-v2 v1.21.2/go.mod h1:ErQhvNuEMhJjweavOYhxVkn2RUx7kQXVATHrjKtxIpM=
+github.com/aws/aws-sdk-go-v2 v1.26.1 h1:5554eUqIYVWpU0YmeeYZ0wU64H2VLBs8TlhRB2L+EkA=
+github.com/aws/aws-sdk-go-v2 v1.26.1/go.mod h1:ffIFB97e2yNsv4aTSGkqtHnppsIJzw7G7BReUZ3jCXM=
+github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.2 h1:x6xsQXGSmW6frevwDA+vi/wqhp1ct18mVXYN08/93to=
+github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.2/go.mod h1:lPprDr1e6cJdyYeGXnRaJoP4Md+cDBvi2eOj00BlGmg=
+github.com/aws/aws-sdk-go-v2/config v1.18.45 h1:Aka9bI7n8ysuwPeFdm77nfbyHCAKQ3z9ghB3S/38zes=
+github.com/aws/aws-sdk-go-v2/config v1.18.45/go.mod h1:ZwDUgFnQgsazQTnWfeLWk5GjeqTQTL8lMkoE1UXzxdE=
+github.com/aws/aws-sdk-go-v2/credentials v1.13.43 h1:LU8vo40zBlo3R7bAvBVy/ku4nxGEyZe9N8MqAeFTzF8=
+github.com/aws/aws-sdk-go-v2/credentials v1.13.43/go.mod h1:zWJBz1Yf1ZtX5NGax9ZdNjhhI4rgjfgsyk6vTY1yfVg=
+github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.13 h1:PIktER+hwIG286DqXyvVENjgLTAwGgoeriLDD5C+YlQ=
+github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.13/go.mod h1:f/Ib/qYjhV2/qdsf79H3QP/eRE4AkVyEf6sk7XfZ1tg=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.43/go.mod h1:auo+PiyLl0n1l8A0e8RIeR8tOzYPfZZH/JNlrJ8igTQ=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.5 h1:aw39xVGeRWlWx9EzGVnhOR4yOjQDHPQ6o6NmBlscyQg=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.5/go.mod h1:FSaRudD0dXiMPK2UjknVwwTYyZMRsHv3TtkabsZih5I=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.37/go.mod h1:Qe+2KtKml+FEsQF/DHmDV+xjtche/hwoF75EG4UlHW8=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.5 h1:PG1F3OD1szkuQPzDw3CIQsRIrtTlUC3lP84taWzHlq0=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.5/go.mod h1:jU1li6RFryMz+so64PpKtudI+QzbKoIEivqdf6LNpOc=
+github.com/aws/aws-sdk-go-v2/internal/ini v1.3.45 h1:hze8YsjSh8Wl1rYa1CJpRmXP21BvOBuc76YhW0HsuQ4=
+github.com/aws/aws-sdk-go-v2/internal/ini v1.3.45/go.mod h1:lD5M20o09/LCuQ2mE62Mb/iSdSlCNuj6H5ci7tW7OsE=
+github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.5 h1:81KE7vaZzrl7yHBYHVEzYB8sypz11NMOZ40YlWvPxsU=
+github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.5/go.mod h1:LIt2rg7Mcgn09Ygbdh/RdIm0rQ+3BNkbP1gyVMFtRK0=
+github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.2 h1:Ji0DY1xUsUr3I8cHps0G+XM3WWU16lP6yG8qu1GAZAs=
+github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.2/go.mod h1:5CsjAbs3NlGQyZNFACh+zztPDI7fU6eW9QsxjfnuBKg=
+github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.7 h1:ZMeFZ5yk+Ek+jNr1+uwCd2tG89t6oTS5yVWpa6yy2es=
+github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.7/go.mod h1:mxV05U+4JiHqIpGqqYXOHLPKUC6bDXC44bsUhNjOEwY=
+github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.37/go.mod h1:vBmDnwWXWxNPFRMmG2m/3MKOe+xEcMDo1tanpaWCcck=
+github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.7 h1:ogRAwT1/gxJBcSWDMZlgyFUM962F51A5CRhDLbxLdmo=
+github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.7/go.mod h1:YCsIZhXfRPLFFCl5xxY+1T9RKzOKjCut+28JSX2DnAk=
+github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.5 h1:f9RyWNtS8oH7cZlbn+/JNPpjUk5+5fLd5lM9M0i49Ys=
+github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.5/go.mod h1:h5CoMZV2VF297/VLhRhO1WF+XYWOzXo+4HsObA4HjBQ=
+github.com/aws/aws-sdk-go-v2/service/s3 v1.53.1 h1:6cnno47Me9bRykw9AEv9zkXE+5or7jz8TsskTTccbgc=
+github.com/aws/aws-sdk-go-v2/service/s3 v1.53.1/go.mod h1:qmdkIIAC+GCLASF7R2whgNrJADz0QZPX+Seiw/i4S3o=
+github.com/aws/aws-sdk-go-v2/service/sso v1.15.2 h1:JuPGc7IkOP4AaqcZSIcyqLpFSqBWK32rM9+a1g6u73k=
+github.com/aws/aws-sdk-go-v2/service/sso v1.15.2/go.mod h1:gsL4keucRCgW+xA85ALBpRFfdSLH4kHOVSnLMSuBECo=
+github.com/aws/aws-sdk-go-v2/service/ssooidc v1.17.3 h1:HFiiRkf1SdaAmV3/BHOFZ9DjFynPHj8G/UIO1lQS+fk=
+github.com/aws/aws-sdk-go-v2/service/ssooidc v1.17.3/go.mod h1:a7bHA82fyUXOm+ZSWKU6PIoBxrjSprdLoM8xPYvzYVg=
+github.com/aws/aws-sdk-go-v2/service/sts v1.23.2 h1:0BkLfgeDjfZnZ+MhB3ONb01u9pwFYTCZVhlsSSBvlbU=
+github.com/aws/aws-sdk-go-v2/service/sts v1.23.2/go.mod h1:Eows6e1uQEsc4ZaHANmsPRzAKcVDrcmjjWiih2+HUUQ=
+github.com/aws/smithy-go v1.15.0/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA=
+github.com/aws/smithy-go v1.20.2 h1:tbp628ireGtzcHDDmLT/6ADHidqnwgF57XOXZe6tp4Q=
+github.com/aws/smithy-go v1.20.2/go.mod h1:krry+ya/rV9RDcV/Q16kpu6ypI4K2czasz0NC3qS14E=
 github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuPk=
 github.com/aymerick/douceur v0.2.0/go.mod h1:wlT5vV2O3h55X9m7iVYN0TBM0NH/MmbLnd30/FjWUq4=
 github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=
@@ -299,6 +340,7 @@ github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMyw
 github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
 github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
 github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
 github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ=
@@ -474,6 +516,8 @@ github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD
 github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc=
 github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ=
 github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8=
+github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
+github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
 github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY=
 github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
 github.com/jrick/logrotate v1.0.0/go.mod h1:LNinyqDIJnpAur+b8yyulnQw/wDuN1+BYKlTRt3OuAQ=
@@ -499,8 +543,8 @@ github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQL
 github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
 github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23/go.mod h1:J+Gs4SYgM6CZQHDETBtE9HaSEkGmuNXF86RwHhHUvq4=
-github.com/klauspost/compress v1.17.2 h1:RlWWUY/Dr4fL8qk9YG7DTZ7PDgME2V4csBXA8L/ixi4=
-github.com/klauspost/compress v1.17.2/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE=
+github.com/klauspost/compress v1.17.6 h1:60eq2E/jlfwQXtvZEeBUYADs+BwKBWURIY+Gj2eRGjI=
+github.com/klauspost/compress v1.17.6/go.mod h1:/dCuZOvVtNoHsyb+cuJD3itjs3NbnF6KH9zAO4BDxPM=
 github.com/klauspost/cpuid/v2 v2.2.6 h1:ndNyv040zDGIDh8thGkXYjnFtiN02M1PVVF+JE/48xc=
 github.com/klauspost/cpuid/v2 v2.2.6/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws=
 github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=

op-plasma/.gitignore

+bin

op-plasma/Dockerfile

+FROM --platform=$BUILDPLATFORM golang:1.21.3-alpine3.18 as builder
+
+RUN apk add --no-cache make gcc musl-dev linux-headers git jq bash
+
+# We copy the go.mod/sum first, so the `go mod download` does not have to re-run if dependencies do not change.
+COPY ./go.mod /app/go.mod
+COPY ./go.sum /app/go.sum
+
+WORKDIR /app
+
+RUN echo "go mod cache: $(go env GOMODCACHE)"
+RUN echo "go build cache: $(go env GOCACHE)"
+
+RUN --mount=type=cache,target=/go/pkg/mod --mount=type=cache,target=/root/.cache/go-build go mod download
+
+# NOTE: the Dockerfile.dockerignore file effectively describes all dependencies
+COPY . /app
+
+ARG GIT_COMMIT
+ARG GIT_DATE
+
+ARG DASERVER_VERSION=v0.0.0
+
+ARG TARGETOS TARGETARCH
+
+RUN --mount=type=cache,target=/root/.cache/go-build cd op-plasma && make da-server  \
+    GOOS=$TARGETOS GOARCH=$TARGETARCH GITCOMMIT=$GIT_COMMIT GITDATE=$GIT_DATE VERSION="$DASERVER_VERSION"
+
+FROM alpine:3.18
+
+COPY --from=builder /app/op-plasma/bin/da-server /usr/local/bin/

op-plasma/Dockerfile.dockerignore

+*
+
+!/op-bindings
+!/op-service
+!/op-plasma
+!/go.mod
+!/go.sum

op-plasma/Makefile

+GITCOMMIT ?= $(shell git rev-parse HEAD)
+GITDATE ?= $(shell git show -s --format='%ct')
+VERSION := v0.0.0
+
+LDFLAGSSTRING +=-X main.GitCommit=$(GITCOMMIT)
+LDFLAGSSTRING +=-X main.GitDate=$(GITDATE)
+LDFLAGSSTRING +=-X main.Version=$(VERSION)
+LDFLAGS := -ldflags "$(LDFLAGSSTRING)"
+
+da-server:
+	env GO111MODULE=on GOOS=$(TARGETOS) GOARCH=$(TARGETARCH) go build -v $(LDFLAGS) -o ./bin/da-server ./cmd/daserver
+
+clean:
+	rm bin/da-server
+
+test:
+	go test -v ./...
+
+.PHONY: \
+	op-batcher \
+	clean \
+	test

op-plasma/cmd/daserver/README.md

+# Plasma DA Server
+
+## Introduction
+
+This simple DA server implementation supports local storage via file based storage and remote via S3.
+LevelDB is only recommended for usage in local devnets where connecting to S3 is not convenient.
+See the [S3 doc](https://aws.github.io/aws-sdk-go-v2/docs/configuring-sdk/) for more information
+on how to configure the S3 client.
+
+## S3 Configuration
+
+Depending on your cloud provider a wide array of configurations are available. The S3 client will
+load configurations from the environment, shared credentials and shared config files.
+Sample environment variables are provided below:
+
+```bash
+export AWS_ACCESS_KEY_ID=YOUR_ACCESS_KEY_ID
+export AWS_SECRET_ACCESS_KEY=YOUR_SECRET_ACCESS_KEY
+export AWS_SESSION_TOKEN=YOUR_SESSION_TOKEN
+export AWS_REGION=YOUR_REGION
+```
+
+You can find out more about AWS authentication [here](https://docs.aws.amazon.com/sdkref/latest/guide/creds-config-files.html).
+
+Additionally, these variables can be used with a google cloud S3 endpoint as well, i.e:
+
+```bash
+export AWS_ENDPOINT_URL="https://storage.googleapis.com"
+export AWS_ACCESS_KEY_ID=YOUR_GOOGLE_ACCESS_KEY_ID
+export AWS_SECRET_ACCESS_KEY=YOUR_GOOGLE_ACCESS_KEY_SECRET
+```

op-plasma/cmd/daserver/entrypoint.go

+package main
+
+import (
+	"fmt"
+
+	"github.com/urfave/cli/v2"
+
+	plasma "github.com/ethereum-optimism/optimism/op-plasma"
+	oplog "github.com/ethereum-optimism/optimism/op-service/log"
+	"github.com/ethereum-optimism/optimism/op-service/opio"
+)
+
+func StartDAServer(cliCtx *cli.Context) error {
+	if err := CheckRequired(cliCtx); err != nil {
+		return err
+	}
+
+	cfg := ReadCLIConfig(cliCtx)
+	if err := cfg.Check(); err != nil {
+		return err
+	}
+
+	logCfg := oplog.ReadCLIConfig(cliCtx)
+
+	l := oplog.NewLogger(oplog.AppOut(cliCtx), logCfg)
+	oplog.SetGlobalLogHandler(l.Handler())
+
+	l.Info("Initializing Plasma DA server...")
+
+	var store plasma.KVStore
+
+	if cfg.FileStoreEnabled() {
+		l.Info("Using file storage", "path", cfg.FileStoreDirPath)
+		store = NewFileStore(cfg.FileStoreDirPath)
+	} else if cfg.S3Enabled() {
+		l.Info("Using S3 storage", "bucket", cfg.S3Bucket)
+		s3, err := NewS3Store(cliCtx.Context, cfg.S3Bucket)
+		if err != nil {
+			return fmt.Errorf("failed to create S3 store: %w", err)
+		}
+		store = s3
+	}
+
+	server := plasma.NewDAServer(cliCtx.String(ListenAddrFlagName), cliCtx.Int(PortFlagName), store, l)
+
+	if err := server.Start(); err != nil {
+		return fmt.Errorf("failed to start the DA server")
+	}
+
+	defer func() {
+		if err := server.Stop(); err != nil {
+			l.Error("failed to stop DA server", "err", err)
+		}
+	}()
+
+	opio.BlockOnInterrupts()
+
+	return nil
+}

op-plasma/cmd/daserver/file.go

+package main
+
+import (
+	"context"
+	"encoding/hex"
+	"os"
+	"path"
+
+	plasma "github.com/ethereum-optimism/optimism/op-plasma"
+)
+
+type FileStore struct {
+	directory string
+}
+
+func NewFileStore(directory string) *FileStore {
+	return &FileStore{
+		directory: directory,
+	}
+}
+
+func (s *FileStore) Get(ctx context.Context, key []byte) ([]byte, error) {
+	data, err := os.ReadFile(s.fileName(key))
+	if err != nil {
+		if os.IsNotExist(err) {
+			return nil, plasma.ErrNotFound
+		}
+		return nil, err
+	}
+	return data, nil
+}
+
+func (s *FileStore) Put(ctx context.Context, key []byte, value []byte) error {
+	return os.WriteFile(s.fileName(key), value, 0600)
+}
+
+func (s *FileStore) fileName(key []byte) string {
+	return path.Join(s.directory, hex.EncodeToString(key))
+}

op-plasma/cmd/daserver/flags.go

+package main
+
+import (
+	"fmt"
+
+	"github.com/urfave/cli/v2"
+
+	opservice "github.com/ethereum-optimism/optimism/op-service"
+	oplog "github.com/ethereum-optimism/optimism/op-service/log"
+)
+
+const (
+	ListenAddrFlagName    = "addr"
+	PortFlagName          = "port"
+	S3BucketFlagName      = "s3.bucket"
+	FileStorePathFlagName = "file.path"
+)
+
+const EnvVarPrefix = "OP_PLASMA_DA_SERVER"
+
+func prefixEnvVars(name string) []string {
+	return opservice.PrefixEnvVar(EnvVarPrefix, name)
+}
+
+var (
+	ListenAddrFlag = &cli.StringFlag{
+		Name:    ListenAddrFlagName,
+		Usage:   "server listening address",
+		Value:   "127.0.0.1",
+		EnvVars: prefixEnvVars("ADDR"),
+	}
+	PortFlag = &cli.IntFlag{
+		Name:    PortFlagName,
+		Usage:   "server listening port",
+		Value:   3100,
+		EnvVars: prefixEnvVars("PORT"),
+	}
+	FileStorePathFlag = &cli.StringFlag{
+		Name:    FileStorePathFlagName,
+		Usage:   "path to directory for file storage",
+		EnvVars: prefixEnvVars("FILESTORE_PATH"),
+	}
+	S3BucketFlag = &cli.StringFlag{
+		Name:    S3BucketFlagName,
+		Usage:   "bucket name for S3 storage",
+		EnvVars: prefixEnvVars("S3_BUCKET"),
+	}
+)
+
+var requiredFlags = []cli.Flag{
+	ListenAddrFlag,
+	PortFlag,
+}
+
+var optionalFlags = []cli.Flag{
+	FileStorePathFlag,
+	S3BucketFlag,
+}
+
+func init() {
+	optionalFlags = append(optionalFlags, oplog.CLIFlags(EnvVarPrefix)...)
+	Flags = append(requiredFlags, optionalFlags...)
+}
+
+// Flags contains the list of configuration options available to the binary.
+var Flags []cli.Flag
+
+type CLIConfig struct {
+	FileStoreDirPath string
+	S3Bucket         string
+}
+
+func ReadCLIConfig(ctx *cli.Context) CLIConfig {
+	return CLIConfig{
+		FileStoreDirPath: ctx.String(FileStorePathFlagName),
+		S3Bucket:         ctx.String(S3BucketFlagName),
+	}
+}
+
+func (c CLIConfig) Check() error {
+	if !c.S3Enabled() && !c.FileStoreEnabled() {
+		return fmt.Errorf("at least one storage backend must be enabled")
+	}
+	if c.S3Enabled() && c.FileStoreEnabled() {
+		return fmt.Errorf("only one storage backend can be enabled")
+	}
+	return nil
+}
+
+func (c CLIConfig) S3Enabled() bool {
+	return c.S3Bucket != ""
+}
+
+func (c CLIConfig) FileStoreEnabled() bool {
+	return c.FileStoreDirPath != ""
+}
+
+func CheckRequired(ctx *cli.Context) error {
+	for _, f := range requiredFlags {
+		if !ctx.IsSet(f.Names()[0]) {
+			return fmt.Errorf("flag %s is required", f.Names()[0])
+		}
+	}
+	return nil
+}

op-plasma/cmd/daserver/main.go

+package main
+
+import (
+	"context"
+	"os"
+
+	"github.com/ethereum/go-ethereum/log"
+	"github.com/urfave/cli/v2"
+
+	opservice "github.com/ethereum-optimism/optimism/op-service"
+	"github.com/ethereum-optimism/optimism/op-service/cliapp"
+	oplog "github.com/ethereum-optimism/optimism/op-service/log"
+	"github.com/ethereum-optimism/optimism/op-service/opio"
+)
+
+var Version = "v0.0.1"
+
+func main() {
+	oplog.SetupDefaults()
+
+	app := cli.NewApp()
+	app.Flags = cliapp.ProtectFlags(Flags)
+	app.Version = opservice.FormatVersion(Version, "", "", "")
+	app.Name = "da-server"
+	app.Usage = "Plasma DA Storage Service"
+	app.Description = "Service for storing plasma DA inputs"
+	app.Action = StartDAServer
+
+	ctx := opio.WithInterruptBlocker(context.Background())
+	err := app.RunContext(ctx, os.Args)
+	if err != nil {
+		log.Crit("Application failed", "message", err)
+	}
+
+}

op-plasma/cmd/daserver/s3.go

+package main
+
+import (
+	"bytes"
+	"context"
+	"encoding/hex"
+	"errors"
+	"io"
+
+	"github.com/aws/aws-sdk-go-v2/aws"
+	"github.com/aws/aws-sdk-go-v2/config"
+	"github.com/aws/aws-sdk-go-v2/service/s3"
+	"github.com/aws/aws-sdk-go-v2/service/s3/types"
+	"github.com/aws/smithy-go"
+
+	plasma "github.com/ethereum-optimism/optimism/op-plasma"
+)
+
+type S3Store struct {
+	bucket string
+	client *s3.Client
+}
+
+func NewS3Store(ctx context.Context, bucket string) (*S3Store, error) {
+	sdkConfig, err := config.LoadDefaultConfig(ctx)
+	if err != nil {
+		return nil, err
+	}
+	return &S3Store{
+		bucket: bucket,
+		client: s3.NewFromConfig(sdkConfig),
+	}, nil
+}
+
+func (s *S3Store) Get(ctx context.Context, key []byte) ([]byte, error) {
+	result, err := s.client.GetObject(ctx, &s3.GetObjectInput{
+		Bucket: aws.String(s.bucket),
+		Key:    aws.String(hex.EncodeToString(key)),
+	})
+	if err != nil {
+		var apiError smithy.APIError
+		if errors.As(err, &apiError) {
+			switch apiError.(type) {
+			case *types.NotFound:
+				return nil, plasma.ErrNotFound
+			}
+		}
+		return nil, err
+	}
+	defer result.Body.Close()
+
+	data, err := io.ReadAll(result.Body)
+	if err != nil {
+		return nil, err
+	}
+
+	return data, nil
+}
+
+func (s *S3Store) Put(ctx context.Context, key []byte, value []byte) error {
+	_, err := s.client.PutObject(ctx, &s3.PutObjectInput{
+		Bucket: aws.String(s.bucket),
+		Key:    aws.String(hex.EncodeToString(key)),
+		Body:   bytes.NewReader(value),
+	})
+	return err
+}

op-plasma/daclient_test.go

@@ -2,74 +2,61 @@ package plasma
 
 import (
 	"context"
-	"io"
+	"fmt"
 	"math/rand"
-	"net/http"
-	"net/http/httptest"
+	"sync"
 	"testing"
 
 	"github.com/ethereum-optimism/optimism/op-service/testlog"
-	"github.com/ethereum/go-ethereum/common/hexutil"
-	"github.com/ethereum/go-ethereum/ethdb/memorydb"
+	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/log"
 	"github.com/stretchr/testify/require"
 )
 
+type MemStore struct {
+	db   map[string][]byte
+	lock sync.RWMutex
+}
+
+func NewMemStore() *MemStore {
+	return &MemStore{
+		db: make(map[string][]byte),
+	}
+}
+
+// Get retrieves the given key if it's present in the key-value store.
+func (s *MemStore) Get(ctx context.Context, key []byte) ([]byte, error) {
+	s.lock.RLock()
+	defer s.lock.RUnlock()
+
+	if entry, ok := s.db[string(key)]; ok {
+		return common.CopyBytes(entry), nil
+	}
+	return nil, ErrNotFound
+}
+
+// Put inserts the given value into the key-value store.
+func (s *MemStore) Put(ctx context.Context, key []byte, value []byte) error {
+	s.lock.Lock()
+	defer s.lock.Unlock()
+
+	s.db[string(key)] = common.CopyBytes(value)
+	return nil
+}
+
 func TestDAClient(t *testing.T) {
-	store := memorydb.New()
+	store := NewMemStore()
 	logger := testlog.Logger(t, log.LevelDebug)
 
 	ctx := context.Background()
 
-	mux := http.NewServeMux()
-	mux.Handle("/get/", http.StripPrefix("/get/", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		logger.Debug("GET", "url", r.URL)
-
-		comm, err := hexutil.Decode(r.URL.String())
-		if err != nil {
-			w.WriteHeader(http.StatusBadRequest)
-			return
-		}
-
-		input, err := store.Get(comm)
-		if err != nil {
-			w.WriteHeader(http.StatusNotFound)
-			return
-		}
-		if _, err := w.Write(input); err != nil {
-			w.WriteHeader(http.StatusInternalServerError)
-			return
-		}
-	})))
-	mux.Handle("/put/", http.StripPrefix("/put/", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		logger.Debug("PUT", "url", r.URL)
-
-		input, err := io.ReadAll(r.Body)
-		if err != nil {
-			w.WriteHeader(http.StatusBadRequest)
-			return
-		}
-		comm, err := hexutil.Decode(r.URL.String())
-		if err != nil {
-			w.WriteHeader(http.StatusBadRequest)
-			return
-		}
-		if err := store.Put(comm, input); err != nil {
-			w.WriteHeader(http.StatusInternalServerError)
-			return
-		}
-
-		if _, err := w.Write(comm); err != nil {
-			w.WriteHeader(http.StatusInternalServerError)
-			return
-		}
-	})))
-
-	tsrv := httptest.NewServer(mux)
+	server := NewDAServer("127.0.0.1", 0, store, logger)
+
+	require.NoError(t, server.Start())
 
 	cfg := CLIConfig{
 		Enabled:      true,
-		DAServerURL:  tsrv.URL,
+		DAServerURL:  fmt.Sprintf("http://%s", server.Endpoint()),
 		VerifyOnRead: true,
 	}
 	require.NoError(t, cfg.Check())
@@ -91,7 +78,7 @@ func TestDAClient(t *testing.T) {
 	require.Equal(t, input, stored)
 
 	// set a bad commitment in the store
-	require.NoError(t, store.Put(comm.Encode(), []byte("bad data")))
+	require.NoError(t, store.Put(ctx, comm.Encode(), []byte("bad data")))
 
 	_, err = client.GetInput(ctx, comm)
 	require.ErrorIs(t, err, ErrCommitmentMismatch)
@@ -106,7 +93,7 @@ func TestDAClient(t *testing.T) {
 	require.ErrorIs(t, err, ErrInvalidInput)
 
 	// server not responsive
-	tsrv.Close()
+	require.NoError(t, server.Stop())
 	_, err = client.SetInput(ctx, input)
 	require.Error(t, err)
 

op-plasma/daserver.go

+package plasma
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"io"
+	"net"
+	"net/http"
+	"path"
+	"strconv"
+	"time"
+
+	"github.com/ethereum-optimism/optimism/op-service/rpc"
+	"github.com/ethereum/go-ethereum/common/hexutil"
+	"github.com/ethereum/go-ethereum/log"
+)
+
+type KVStore interface {
+	// Get retrieves the given key if it's present in the key-value data store.
+	Get(ctx context.Context, key []byte) ([]byte, error)
+	// Put inserts the given value into the key-value data store.
+	Put(ctx context.Context, key []byte, value []byte) error
+}
+
+type DAServer struct {
+	log        log.Logger
+	endpoint   string
+	store      KVStore
+	tls        *rpc.ServerTLSConfig
+	httpServer *http.Server
+	listener   net.Listener
+}
+
+func NewDAServer(host string, port int, store KVStore, log log.Logger) *DAServer {
+	endpoint := net.JoinHostPort(host, strconv.Itoa(port))
+	return &DAServer{
+		log:      log,
+		endpoint: endpoint,
+		store:    store,
+		httpServer: &http.Server{
+			Addr: endpoint,
+		},
+	}
+}
+
+func (d *DAServer) Start() error {
+	mux := http.NewServeMux()
+
+	mux.HandleFunc("/get/", d.HandleGet)
+	mux.HandleFunc("/put/", d.HandlePut)
+
+	d.httpServer.Handler = mux
+
+	listener, err := net.Listen("tcp", d.endpoint)
+	if err != nil {
+		return fmt.Errorf("failed to listen: %w", err)
+	}
+	d.listener = listener
+
+	d.endpoint = listener.Addr().String()
+	errCh := make(chan error, 1)
+	go func() {
+		if d.tls != nil {
+			if err := d.httpServer.ServeTLS(d.listener, "", ""); err != nil {
+				errCh <- err
+			}
+		} else {
+			if err := d.httpServer.Serve(d.listener); err != nil {
+				errCh <- err
+			}
+		}
+	}()
+
+	// verify that the server comes up
+	tick := time.NewTimer(10 * time.Millisecond)
+	defer tick.Stop()
+
+	select {
+	case err := <-errCh:
+		return fmt.Errorf("http server failed: %w", err)
+	case <-tick.C:
+		return nil
+	}
+}
+
+func (d *DAServer) HandleGet(w http.ResponseWriter, r *http.Request) {
+	d.log.Debug("GET", "url", r.URL)
+
+	route := path.Dir(r.URL.Path)
+	if route != "/get" {
+		w.WriteHeader(http.StatusBadRequest)
+		return
+	}
+
+	key := path.Base(r.URL.Path)
+	comm, err := hexutil.Decode(key)
+	if err != nil {
+		w.WriteHeader(http.StatusBadRequest)
+		return
+	}
+
+	input, err := d.store.Get(r.Context(), comm)
+	if err != nil && errors.Is(err, ErrNotFound) {
+		w.WriteHeader(http.StatusNotFound)
+		return
+	}
+	if err != nil {
+		w.WriteHeader(http.StatusInternalServerError)
+		return
+	}
+	if _, err := w.Write(input); err != nil {
+		w.WriteHeader(http.StatusInternalServerError)
+		return
+	}
+}
+
+func (d *DAServer) HandlePut(w http.ResponseWriter, r *http.Request) {
+	d.log.Debug("PUT", "url", r.URL)
+
+	route := path.Dir(r.URL.Path)
+	if route != "/put" {
+		w.WriteHeader(http.StatusBadRequest)
+		return
+	}
+
+	input, err := io.ReadAll(r.Body)
+	if err != nil {
+		w.WriteHeader(http.StatusBadRequest)
+		return
+	}
+
+	key := path.Base(r.URL.Path)
+	comm, err := hexutil.Decode(key)
+	if err != nil {
+		w.WriteHeader(http.StatusBadRequest)
+		return
+	}
+
+	if err := d.store.Put(r.Context(), comm, input); err != nil {
+		w.WriteHeader(http.StatusInternalServerError)
+		return
+	}
+	if _, err := w.Write(comm); err != nil {
+		w.WriteHeader(http.StatusInternalServerError)
+		return
+	}
+}
+
+func (b *DAServer) Endpoint() string {
+	return b.listener.Addr().String()
+}
+
+func (b *DAServer) Stop() error {
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+	defer cancel()
+	_ = b.httpServer.Shutdown(ctx)
+	return nil
+}

op-ufm/go.mod

@@ -76,7 +76,7 @@ require (
 	github.com/kataras/pio v0.0.13 // indirect
 	github.com/kataras/sitemap v0.0.6 // indirect
 	github.com/kataras/tunnel v0.0.4 // indirect
-	github.com/klauspost/compress v1.17.4 // indirect
+	github.com/klauspost/compress v1.17.6 // indirect
 	github.com/kr/pretty v0.3.1 // indirect
 	github.com/kr/text v0.2.0 // indirect
 	github.com/mailgun/raymond/v2 v2.0.48 // indirect

op-ufm/go.sum

@@ -231,8 +231,8 @@ github.com/kataras/tunnel v0.0.4 h1:sCAqWuJV7nPzGrlb0os3j49lk2JhILT0rID38NHNLpA=
 github.com/kataras/tunnel v0.0.4/go.mod h1:9FkU4LaeifdMWqZu7o20ojmW4B7hdhv2CMLwfnHGpYw=
 github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
-github.com/klauspost/compress v1.17.4 h1:Ej5ixsIri7BrIjBkRZLTo6ghwrEtHFk7ijlczPW4fZ4=
-github.com/klauspost/compress v1.17.4/go.mod h1:/dCuZOvVtNoHsyb+cuJD3itjs3NbnF6KH9zAO4BDxPM=
+github.com/klauspost/compress v1.17.6 h1:60eq2E/jlfwQXtvZEeBUYADs+BwKBWURIY+Gj2eRGjI=
+github.com/klauspost/compress v1.17.6/go.mod h1:/dCuZOvVtNoHsyb+cuJD3itjs3NbnF6KH9zAO4BDxPM=
 github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
 github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=