op-batcher op-proposer op-service: Introduce op-service (#3202)

* op-batcher op-proposer op-service: Introduce op-service

op-service is a utilities library that automates much of the boilerplate setup we've been duplicating across different daemons. This PR adds the following to op-batcher:

- Common setup utilities for logging, metrics, pprof, and RPC.
- For each of the above functions, a set of CLI flags and configs that can be used to parse + validate CLI configuration for that module.
- A base RPC server that implementers can extend.
- HTTP metrics utilities.

* update dockerfiles

* code review updates

* op-service: temporary replace, until op-service go module is canonical and can be resolved
Co-authored-by: protolambda <proto@protolambda.com>

.circleci/config.yml

@@ -244,6 +244,11 @@ jobs:
           command: |
             golangci-lint run -E goimports,sqlclosecheck,bodyclose,asciicheck,misspell ./...
           working_directory: op-e2e
+      - run:
+          name: lint op-service
+          command: |
+            golangci-lint run -E goimports,sqlclosecheck,bodyclose,asciicheck,misspell ./...
+          working_directory: op-service
       - run:
           name: prep results dir
           command: mkdir -p /test-results
@@ -267,6 +272,11 @@ jobs:
           command: |
             gotestsum --junitfile /test-results/op-e2e.xml -- -coverpkg=github.com/ethereum-optimism/optimism/... -coverprofile=coverage.out -covermode=atomic ./...
           working_directory: op-e2e
+      - run:
+          name: test op-service
+          command: |
+            gotestsum --junitfile /test-results/op-service.xml -- -coverpkg=github.com/ethereum-optimism/optimism/... -coverprofile=coverage.out -covermode=atomic ./...
+          working_directory: op-service
       - store_test_results:
           path: /test-results
       - run:

go.work

@@ -13,9 +13,10 @@ use (
 	./op-exporter
 	./op-node
 	./op-proposer
+	./op-service
 	./proxyd
-	./teleportr
 	./state-surgery
+	./teleportr
 )
 
 replace github.com/ethereum/go-ethereum v1.10.21 => github.com/ethereum-optimism/reference-optimistic-geth v0.0.0-20220803173305-1c9d4cc76a6e

op-batcher/Dockerfile

@@ -7,6 +7,7 @@ COPY ./op-batcher/docker.go.work /app/go.work
 COPY ./op-bindings /app/op-bindings
 COPY ./op-node /app/op-node
 COPY ./op-proposer /app/op-proposer
+COPY ./op-service /app/op-service
 COPY ./op-batcher /app/op-batcher
 
 WORKDIR /app/op-batcher

op-batcher/batch_submitter.go

@@ -8,8 +8,6 @@ import (
 	"fmt"
 	"io"
 	"math/big"
-	"net"
-	"net/http"
 	_ "net/http/pprof"
 	"os"
 	"os/signal"
@@ -18,6 +16,12 @@ import (
 	"syscall"
 	"time"
 
+	oplog "github.com/ethereum-optimism/optimism/op-service/log"
+	opmetrics "github.com/ethereum-optimism/optimism/op-service/metrics"
+	oppprof "github.com/ethereum-optimism/optimism/op-service/pprof"
+	oprpc "github.com/ethereum-optimism/optimism/op-service/rpc"
+	"github.com/ethereum/go-ethereum/rpc"
+
 	"github.com/ethereum-optimism/optimism/op-batcher/sequencer"
 	"github.com/ethereum-optimism/optimism/op-node/eth"
 	"github.com/ethereum-optimism/optimism/op-node/rollup/derive"
@@ -30,7 +34,6 @@ import (
 	"github.com/ethereum/go-ethereum/crypto"
 	"github.com/ethereum/go-ethereum/ethclient"
 	"github.com/ethereum/go-ethereum/log"
-	"github.com/ethereum/go-ethereum/rpc"
 	hdwallet "github.com/miguelmota/go-ethereum-hdwallet"
 	"github.com/urfave/cli"
 )
@@ -45,26 +48,14 @@ const (
 // closure that executes the service and blocks until the service exits. The use
 // of a closure allows the parameters bound to the top-level main package, e.g.
 // GitVersion, to be captured and used once the function is executed.
-func Main(version string) func(ctx *cli.Context) error {
-	return func(ctx *cli.Context) error {
-		cfg := NewConfig(ctx)
-
-		// Set up our logging to stdout.
-		var logHandler log.Handler
-		if cfg.LogTerminal {
-			logHandler = log.StreamHandler(os.Stdout, log.TerminalFormat(true))
-		} else {
-			logHandler = log.StreamHandler(os.Stdout, log.JSONFormat())
-		}
-
-		logLevel, err := log.LvlFromString(cfg.LogLevel)
-		if err != nil {
-			return err
+func Main(version string) func(cliCtx *cli.Context) error {
+	return func(cliCtx *cli.Context) error {
+		cfg := NewConfig(cliCtx)
+		if err := cfg.Check(); err != nil {
+			return fmt.Errorf("invalid CLI flags: %w", err)
 		}
 
-		l := log.New()
-		l.SetHandler(log.LvlFilterHandler(logLevel, logHandler))
-
+		l := oplog.NewLogger(cfg.LogConfig)
 		l.Info("Initializing Batch Submitter")
 
 		batchSubmitter, err := NewBatchSubmitter(cfg, l)
@@ -81,28 +72,41 @@ func Main(version string) func(ctx *cli.Context) error {
 		}
 		defer batchSubmitter.Stop()
 
+		ctx, cancel := context.WithCancel(context.Background())
+
 		l.Info("Batch Submitter started")
-		if cfg.PprofEnabled {
-			var srv http.Server
-			srv.Addr = net.JoinHostPort(cfg.PprofAddr, cfg.PprofPort)
-			// Start pprof server + register it's shutdown
+		pprofConfig := cfg.PprofConfig
+		if pprofConfig.Enabled {
+			l.Info("starting pprof", "addr", pprofConfig.ListenAddr, "port", pprofConfig.ListenPort)
 			go func() {
-				l.Info("pprof server started", "addr", srv.Addr)
-				if err := srv.ListenAndServe(); err != http.ErrServerClosed {
-					l.Error("error in pprof server", "err", err)
-				} else {
-					l.Info("pprof server shutting down")
+				if err := oppprof.ListenAndServe(ctx, pprofConfig.ListenAddr, pprofConfig.ListenPort); err != nil {
+					l.Error("error starting pprof", "err", err)
 				}
-
 			}()
-			defer func() {
-				shutCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
-				defer cancel()
-				err := srv.Shutdown(shutCtx)
-				l.Info("pprof server shut down", "err", err)
+		}
+
+		registry := opmetrics.NewRegistry()
+		metricsCfg := cfg.MetricsConfig
+		if metricsCfg.Enabled {
+			l.Info("starting metrics server", "addr", metricsCfg.ListenAddr, "port", metricsCfg.ListenPort)
+			go func() {
+				if err := opmetrics.ListenAndServe(ctx, registry, metricsCfg.ListenAddr, metricsCfg.ListenPort); err != nil {
+					l.Error("error starting metrics server", err)
+				}
 			}()
 		}
 
+		rpcCfg := cfg.RPCConfig
+		server := oprpc.NewServer(
+			rpcCfg.ListenAddr,
+			rpcCfg.ListenPort,
+			version,
+		)
+		if err := server.Start(); err != nil {
+			cancel()
+			return fmt.Errorf("error starting RPC server: %w", err)
+		}
+
 		interruptChannel := make(chan os.Signal, 1)
 		signal.Notify(interruptChannel, []os.Signal{
 			os.Interrupt,
@@ -111,7 +115,8 @@ func Main(version string) func(ctx *cli.Context) error {
 			syscall.SIGQUIT,
 		}...)
 		<-interruptChannel
-
+		cancel()
+		_ = server.Stop()
 		return nil
 	}
 }

op-batcher/cmd/main.go

@@ -4,6 +4,8 @@ import (
 	"fmt"
 	"os"
 
+	oplog "github.com/ethereum-optimism/optimism/op-service/log"
+
 	"github.com/ethereum/go-ethereum/log"
 	"github.com/urfave/cli"
 
@@ -18,14 +20,7 @@ var (
 )
 
 func main() {
-	// Set up logger with a default INFO level in case we fail to parse flags,
-	// otherwise the final critical log won't show what the parsing error was.
-	log.Root().SetHandler(
-		log.LvlFilterHandler(
-			log.LvlInfo,
-			log.StreamHandler(os.Stdout, log.TerminalFormat(true)),
-		),
-	)
+	oplog.SetupDefaults()
 
 	app := cli.NewApp()
 	app.Flags = flags.Flags

op-batcher/config.go

@@ -3,6 +3,13 @@ package op_batcher
 import (
 	"time"
 
+	oprpc "github.com/ethereum-optimism/optimism/op-service/rpc"
+
+	opmetrics "github.com/ethereum-optimism/optimism/op-service/metrics"
+
+	oplog "github.com/ethereum-optimism/optimism/op-service/log"
+	oppprof "github.com/ethereum-optimism/optimism/op-service/pprof"
+
 	"github.com/urfave/cli"
 
 	"github.com/ethereum-optimism/optimism/op-batcher/flags"
@@ -63,19 +70,31 @@ type Config struct {
 	// transactions.
 	SequencerBatchInboxAddress string
 
+	RPCConfig oprpc.CLIConfig
+
 	/* Optional Params */
 
-	// LogLevel is the lowest log level that will be output.
-	LogLevel string
+	LogConfig oplog.CLIConfig
+
+	MetricsConfig opmetrics.CLIConfig
 
-	// LogTerminal if true, will log to stdout in terminal format. Otherwise the
-	// output will be in JSON format.
-	LogTerminal bool
+	PprofConfig oppprof.CLIConfig
+}
 
-	// Flags for the pprof server
-	PprofEnabled bool
-	PprofAddr    string
-	PprofPort    string
+func (c Config) Check() error {
+	if err := c.RPCConfig.Check(); err != nil {
+		return err
+	}
+	if err := c.LogConfig.Check(); err != nil {
+		return err
+	}
+	if err := c.MetricsConfig.Check(); err != nil {
+		return err
+	}
+	if err := c.PprofConfig.Check(); err != nil {
+		return err
+	}
+	return nil
 }
 
 // NewConfig parses the Config from the provided flags or environment variables.
@@ -96,11 +115,9 @@ func NewConfig(ctx *cli.Context) Config {
 		SequencerHDPath:            ctx.GlobalString(flags.SequencerHDPathFlag.Name),
 		PrivateKey:                 ctx.GlobalString(flags.PrivateKeyFlag.Name),
 		SequencerBatchInboxAddress: ctx.GlobalString(flags.SequencerBatchInboxAddressFlag.Name),
-		/* Optional Flags */
-		LogLevel:     ctx.GlobalString(flags.LogLevelFlag.Name),
-		LogTerminal:  ctx.GlobalBool(flags.LogTerminalFlag.Name),
-		PprofEnabled: ctx.GlobalBool(flags.PprofEnabledFlag.Name),
-		PprofAddr:    ctx.GlobalString(flags.PprofAddrFlag.Name),
-		PprofPort:    ctx.GlobalString(flags.PprofPortFlag.Name),
+		RPCConfig:                  oprpc.ReadCLIConfig(ctx),
+		LogConfig:                  oplog.ReadCLIConfig(ctx),
+		MetricsConfig:              opmetrics.ReadCLIConfig(ctx),
+		PprofConfig:                oppprof.ReadCLIConfig(ctx),
 	}
 }

op-batcher/docker.go.work

@@ -5,4 +5,5 @@ use (
 	./op-bindings
 	./op-node
 	./op-proposer
+	./op-service
 )

op-batcher/flags/flags.go

 package flags
 
 import (
+	opservice "github.com/ethereum-optimism/optimism/op-service"
+	oplog "github.com/ethereum-optimism/optimism/op-service/log"
+	opmetrics "github.com/ethereum-optimism/optimism/op-service/metrics"
+	oppprof "github.com/ethereum-optimism/optimism/op-service/pprof"
+	oprpc "github.com/ethereum-optimism/optimism/op-service/rpc"
 	"github.com/urfave/cli"
 )
 
-const envVarPrefix = "OP_BATCHER_"
-
-func prefixEnvVar(name string) string {
-	return envVarPrefix + name
-}
+const envVarPrefix = "OP_BATCHER"
 
 var (
 	/* Required Flags */
@@ -17,51 +18,51 @@ var (
 		Name:     "l1-eth-rpc",
 		Usage:    "HTTP provider URL for L1",
 		Required: true,
-		EnvVar:   prefixEnvVar("L1_ETH_RPC"),
+		EnvVar:   opservice.PrefixEnvVar(envVarPrefix, "L1_ETH_RPC"),
 	}
 	L2EthRpcFlag = cli.StringFlag{
 		Name:     "l2-eth-rpc",
 		Usage:    "HTTP provider URL for L2 execution engine",
 		Required: true,
-		EnvVar:   prefixEnvVar("L2_ETH_RPC"),
+		EnvVar:   opservice.PrefixEnvVar(envVarPrefix, "L2_ETH_RPC"),
 	}
 	RollupRpcFlag = cli.StringFlag{
 		Name:     "rollup-rpc",
 		Usage:    "HTTP provider URL for Rollup node",
 		Required: true,
-		EnvVar:   prefixEnvVar("ROLLUP_RPC"),
+		EnvVar:   opservice.PrefixEnvVar(envVarPrefix, "ROLLUP_RPC"),
 	}
 	MinL1TxSizeBytesFlag = cli.Uint64Flag{
 		Name:     "min-l1-tx-size-bytes",
 		Usage:    "The minimum size of a batch tx submitted to L1.",
 		Required: true,
-		EnvVar:   prefixEnvVar("MIN_L1_TX_SIZE_BYTES"),
+		EnvVar:   opservice.PrefixEnvVar(envVarPrefix, "MIN_L1_TX_SIZE_BYTES"),
 	}
 	MaxL1TxSizeBytesFlag = cli.Uint64Flag{
 		Name:     "max-l1-tx-size-bytes",
 		Usage:    "The maximum size of a batch tx submitted to L1.",
 		Required: true,
-		EnvVar:   prefixEnvVar("MAX_L1_TX_SIZE_BYTES"),
+		EnvVar:   opservice.PrefixEnvVar(envVarPrefix, "MAX_L1_TX_SIZE_BYTES"),
 	}
 	ChannelTimeoutFlag = cli.Uint64Flag{
 		Name:     "channel-timeout",
 		Usage:    "The maximum amount of time to attempt completing an opened channel, as opposed to submitting L2 blocks into a new channel.",
 		Required: true,
-		EnvVar:   prefixEnvVar("CHANNEL_TIMEOUT"),
+		EnvVar:   opservice.PrefixEnvVar(envVarPrefix, "CHANNEL_TIMEOUT"),
 	}
 	PollIntervalFlag = cli.DurationFlag{
 		Name: "poll-interval",
 		Usage: "Delay between querying L2 for more transactions and " +
 			"creating a new batch",
 		Required: true,
-		EnvVar:   prefixEnvVar("POLL_INTERVAL"),
+		EnvVar:   opservice.PrefixEnvVar(envVarPrefix, "POLL_INTERVAL"),
 	}
 	NumConfirmationsFlag = cli.Uint64Flag{
 		Name: "num-confirmations",
 		Usage: "Number of confirmations which we will wait after " +
 			"appending a new batch",
 		Required: true,
-		EnvVar:   prefixEnvVar("NUM_CONFIRMATIONS"),
+		EnvVar:   opservice.PrefixEnvVar(envVarPrefix, "NUM_CONFIRMATIONS"),
 	}
 	SafeAbortNonceTooLowCountFlag = cli.Uint64Flag{
 		Name: "safe-abort-nonce-too-low-count",
@@ -69,69 +70,37 @@ var (
 			"give up on a tx at a particular nonce without receiving " +
 			"confirmation",
 		Required: true,
-		EnvVar:   prefixEnvVar("SAFE_ABORT_NONCE_TOO_LOW_COUNT"),
+		EnvVar:   opservice.PrefixEnvVar(envVarPrefix, "SAFE_ABORT_NONCE_TOO_LOW_COUNT"),
 	}
 	ResubmissionTimeoutFlag = cli.DurationFlag{
 		Name: "resubmission-timeout",
 		Usage: "Duration we will wait before resubmitting a " +
 			"transaction to L1",
 		Required: true,
-		EnvVar:   prefixEnvVar("RESUBMISSION_TIMEOUT"),
+		EnvVar:   opservice.PrefixEnvVar(envVarPrefix, "RESUBMISSION_TIMEOUT"),
 	}
 	MnemonicFlag = cli.StringFlag{
 		Name: "mnemonic",
 		Usage: "The mnemonic used to derive the wallets for either the " +
 			"sequencer or the l2output",
-		EnvVar: prefixEnvVar("MNEMONIC"),
+		EnvVar: opservice.PrefixEnvVar(envVarPrefix, "MNEMONIC"),
 	}
 	SequencerHDPathFlag = cli.StringFlag{
 		Name: "sequencer-hd-path",
 		Usage: "The HD path used to derive the sequencer wallet from the " +
 			"mnemonic. The mnemonic flag must also be set.",
-		EnvVar: prefixEnvVar("SEQUENCER_HD_PATH"),
+		EnvVar: opservice.PrefixEnvVar(envVarPrefix, "SEQUENCER_HD_PATH"),
 	}
 	PrivateKeyFlag = cli.StringFlag{
 		Name:   "private-key",
 		Usage:  "The private key to use with the l2output wallet. Must not be used with mnemonic.",
-		EnvVar: prefixEnvVar("PRIVATE_KEY"),
+		EnvVar: opservice.PrefixEnvVar(envVarPrefix, "PRIVATE_KEY"),
 	}
 	SequencerBatchInboxAddressFlag = cli.StringFlag{
 		Name:     "sequencer-batch-inbox-address",
 		Usage:    "L1 Address to receive batch transactions",
 		Required: true,
-		EnvVar:   prefixEnvVar("SEQUENCER_BATCH_INBOX_ADDRESS"),
-	}
-
-	/* Optional Flags */
-
-	LogLevelFlag = cli.StringFlag{
-		Name:   "log.level",
-		Usage:  "The lowest log level that will be output",
-		Value:  "info",
-		EnvVar: prefixEnvVar("LOG_LEVEL"),
-	}
-	LogTerminalFlag = cli.BoolFlag{
-		Name: "log.terminal",
-		Usage: "If true, outputs logs in terminal format, otherwise prints " +
-			"in JSON format.",
-		EnvVar: prefixEnvVar("LOG_TERMINAL"),
-	}
-	PprofEnabledFlag = cli.BoolFlag{
-		Name:   "pprof.enabled",
-		Usage:  "Enable the pprof server",
-		EnvVar: prefixEnvVar("PPROF_ENABLED"),
-	}
-	PprofAddrFlag = cli.StringFlag{
-		Name:   "pprof.addr",
-		Usage:  "pprof listening address",
-		Value:  "0.0.0.0",
-		EnvVar: prefixEnvVar("PPROF_ADDR"),
-	}
-	PprofPortFlag = cli.IntFlag{
-		Name:   "pprof.port",
-		Usage:  "pprof listening port",
-		Value:  6060,
-		EnvVar: prefixEnvVar("PPROF_PORT"),
+		EnvVar:   opservice.PrefixEnvVar(envVarPrefix, "SEQUENCER_BATCH_INBOX_ADDRESS"),
 	}
 )
 
@@ -153,12 +122,17 @@ var optionalFlags = []cli.Flag{
 	MnemonicFlag,
 	SequencerHDPathFlag,
 	PrivateKeyFlag,
-	LogLevelFlag,
-	LogTerminalFlag,
-	PprofEnabledFlag,
-	PprofAddrFlag,
-	PprofPortFlag,
+}
+
+func init() {
+	requiredFlags = append(requiredFlags, oprpc.CLIFlags(envVarPrefix)...)
+
+	optionalFlags = append(optionalFlags, oplog.CLIFlags(envVarPrefix)...)
+	optionalFlags = append(optionalFlags, opmetrics.CLIFlags(envVarPrefix)...)
+	optionalFlags = append(optionalFlags, oppprof.CLIFlags(envVarPrefix)...)
+
+	Flags = append(requiredFlags, optionalFlags...)
 }
 
 // Flags contains the list of configuration options available to the binary.
-var Flags = append(requiredFlags, optionalFlags...)
+var Flags []cli.Flag

op-batcher/go.mod

@@ -2,12 +2,15 @@ module github.com/ethereum-optimism/optimism/op-batcher
 
 go 1.18
 
+replace github.com/ethereum-optimism/optimism/op-service v0.0.0 => ../op-service
+
 require (
 	github.com/ethereum-optimism/optimism/op-node v0.3.0
 	github.com/ethereum-optimism/optimism/op-proposer v0.3.0
+	github.com/ethereum-optimism/optimism/op-service v0.0.0
 	github.com/ethereum/go-ethereum v1.10.21
 	github.com/miguelmota/go-ethereum-hdwallet v0.1.1
-	github.com/urfave/cli v1.22.5
+	github.com/urfave/cli v1.22.9
 )
 
 require (
@@ -21,36 +24,51 @@ require (
 	github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect
 	github.com/deckarep/golang-set v1.8.0 // indirect
 	github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.1 // indirect
+	github.com/edsrzf/mmap-go v1.1.0 // indirect
 	github.com/ethereum-optimism/optimism/op-bindings v0.3.0 // indirect
+	github.com/fjl/memsize v0.0.1 // indirect
 	github.com/go-ole/go-ole v1.2.6 // indirect
 	github.com/go-stack/stack v1.8.1 // indirect
 	github.com/golang-jwt/jwt/v4 v4.3.0 // indirect
 	github.com/golang/protobuf v1.5.2 // indirect
 	github.com/golang/snappy v0.0.4 // indirect
 	github.com/gorilla/websocket v1.5.0 // indirect
+	github.com/hashicorp/go-bexpr v0.1.11 // indirect
 	github.com/hashicorp/golang-lru v0.5.5-0.20210104140557-80c98217689d // indirect
 	github.com/holiman/bloomfilter/v2 v2.0.3 // indirect
 	github.com/holiman/uint256 v1.2.0 // indirect
+	github.com/huin/goupnp v1.0.3 // indirect
+	github.com/jackpal/go-nat-pmp v1.0.2 // indirect
+	github.com/mattn/go-colorable v0.1.12 // indirect
+	github.com/mattn/go-isatty v0.0.14 // indirect
 	github.com/mattn/go-runewidth v0.0.13 // indirect
 	github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect
+	github.com/mitchellh/mapstructure v1.5.0 // indirect
+	github.com/mitchellh/pointerstructure v1.2.1 // indirect
 	github.com/olekukonko/tablewriter v0.0.5 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
-	github.com/prometheus/client_golang v1.12.2 // indirect
+	github.com/prometheus/client_golang v1.13.0 // indirect
 	github.com/prometheus/client_model v0.2.0 // indirect
-	github.com/prometheus/common v0.35.0 // indirect
-	github.com/prometheus/procfs v0.7.3 // indirect
+	github.com/prometheus/common v0.37.0 // indirect
+	github.com/prometheus/procfs v0.8.0 // indirect
 	github.com/prometheus/tsdb v0.10.0 // indirect
 	github.com/rivo/uniseg v0.2.0 // indirect
+	github.com/rs/cors v1.8.2 // indirect
 	github.com/russross/blackfriday/v2 v2.1.0 // indirect
 	github.com/shirou/gopsutil v3.21.11+incompatible // indirect
 	github.com/syndtr/goleveldb v1.0.1-0.20220614013038-64ee5596c38a // indirect
 	github.com/tklauser/go-sysconf v0.3.10 // indirect
 	github.com/tklauser/numcpus v0.4.0 // indirect
 	github.com/tyler-smith/go-bip39 v1.1.0 // indirect
+	github.com/urfave/cli/v2 v2.10.2 // indirect
+	github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect
 	github.com/yusufpapurcu/wmi v1.2.2 // indirect
 	golang.org/x/crypto v0.0.0-20220525230936-793ad666bf5e // indirect
+	golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f // indirect
 	golang.org/x/sys v0.0.0-20220701225701-179beb0bd1a1 // indirect
-	google.golang.org/protobuf v1.28.0 // indirect
+	golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 // indirect
+	golang.org/x/time v0.0.0-20220722155302-e5dcc9cfc0b9 // indirect
+	google.golang.org/protobuf v1.28.1 // indirect
 	gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce // indirect
 )
 

op-e2e/go.mod

@@ -2,11 +2,14 @@ module github.com/ethereum-optimism/optimism/op-e2e
 
 go 1.18
 
+replace github.com/ethereum-optimism/optimism/op-service v0.0.0 => ../op-service
+
 require (
 	github.com/ethereum-optimism/optimism/op-batcher v0.3.0
 	github.com/ethereum-optimism/optimism/op-bindings v0.3.0
 	github.com/ethereum-optimism/optimism/op-node v0.3.0
 	github.com/ethereum-optimism/optimism/op-proposer v0.3.0
+	github.com/ethereum-optimism/optimism/op-service v0.0.0
 	github.com/ethereum/go-ethereum v1.10.21
 	github.com/libp2p/go-libp2p v0.18.1
 	github.com/libp2p/go-libp2p-core v0.15.0
@@ -121,10 +124,10 @@ require (
 	github.com/opentracing/opentracing-go v1.2.0 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
-	github.com/prometheus/client_golang v1.12.2 // indirect
+	github.com/prometheus/client_golang v1.13.0 // indirect
 	github.com/prometheus/client_model v0.2.0 // indirect
-	github.com/prometheus/common v0.35.0 // indirect
-	github.com/prometheus/procfs v0.7.3 // indirect
+	github.com/prometheus/common v0.37.0 // indirect
+	github.com/prometheus/procfs v0.8.0 // indirect
 	github.com/prometheus/tsdb v0.10.0 // indirect
 	github.com/raulk/clock v1.1.0 // indirect
 	github.com/raulk/go-watchdog v1.2.0 // indirect
@@ -139,7 +142,7 @@ require (
 	github.com/tklauser/go-sysconf v0.3.10 // indirect
 	github.com/tklauser/numcpus v0.4.0 // indirect
 	github.com/tyler-smith/go-bip39 v1.1.0 // indirect
-	github.com/urfave/cli v1.22.5 // indirect
+	github.com/urfave/cli v1.22.9 // indirect
 	github.com/urfave/cli/v2 v2.10.2 // indirect
 	github.com/whyrusleeping/multiaddr-filter v0.0.0-20160516205228-e903e4adabd7 // indirect
 	github.com/whyrusleeping/timecache v0.0.0-20160911033111-cfcb2f1abfee // indirect
@@ -156,7 +159,7 @@ require (
 	golang.org/x/text v0.3.7 // indirect
 	golang.org/x/time v0.0.0-20220224211638-0e9765cccd65 // indirect
 	google.golang.org/grpc v1.46.2 // indirect
-	google.golang.org/protobuf v1.28.0 // indirect
+	google.golang.org/protobuf v1.28.1 // indirect
 	gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 )

op-e2e/go.sum

@@ -904,8 +904,8 @@ github.com/prometheus/client_golang v1.9.0/go.mod h1:FqZLKOZnGdFAhOK4nqGHa7D66Id
 github.com/prometheus/client_golang v1.10.0/go.mod h1:WJM3cc3yu7XKBKa/I8WeZm+V3eltZnBwfENSU7mdogU=
 github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0=
 github.com/prometheus/client_golang v1.12.1/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY=
-github.com/prometheus/client_golang v1.12.2 h1:51L9cDoUHVrXx4zWYlcLQIZ+d+VXHgqnYKkIuq4g/34=
-github.com/prometheus/client_golang v1.12.2/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY=
+github.com/prometheus/client_golang v1.13.0 h1:b71QUfeo5M8gq2+evJdTPfZhYMAU0uKPkyPJ7TPsloU=
+github.com/prometheus/client_golang v1.13.0/go.mod h1:vTeo+zgvILHsnnj/39Ou/1fPN5nJFOEMgftOUOmlvYQ=
 github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
 github.com/prometheus/client_model v0.0.0-20190115171406-56726106282f/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
 github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
@@ -925,8 +925,8 @@ github.com/prometheus/common v0.18.0/go.mod h1:U+gB1OBLb1lF3O42bTCL+FK18tX9Oar16
 github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc=
 github.com/prometheus/common v0.30.0/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls=
 github.com/prometheus/common v0.32.1/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls=
-github.com/prometheus/common v0.35.0 h1:Eyr+Pw2VymWejHqCugNaQXkAi6KayVNxaHeu6khmFBE=
-github.com/prometheus/common v0.35.0/go.mod h1:phzohg0JFMnBEFGxTDbfu3QyL5GI8gTQJFhYO5B3mfA=
+github.com/prometheus/common v0.37.0 h1:ccBbHCgIiT9uSoFY0vX8H3zsNR5eLt17/RQLUvn8pXE=
+github.com/prometheus/common v0.37.0/go.mod h1:phzohg0JFMnBEFGxTDbfu3QyL5GI8gTQJFhYO5B3mfA=
 github.com/prometheus/procfs v0.0.0-20180725123919-05ee40e3a273/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
 github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
 github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
@@ -935,8 +935,9 @@ github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+Gx
 github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU=
 github.com/prometheus/procfs v0.2.0/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU=
 github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA=
-github.com/prometheus/procfs v0.7.3 h1:4jVXhlkAyzOScmCkXBTOLRLTz8EeU+eyjrwB/EPq0VU=
 github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA=
+github.com/prometheus/procfs v0.8.0 h1:ODq8ZFEaYeCaZOJlZZdJA2AbQR98dSHSM1KW/You5mo=
+github.com/prometheus/procfs v0.8.0/go.mod h1:z7EfXMXOkbkqb9IINtpCn86r/to3BnA0uaxHdg830/4=
 github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU=
 github.com/prometheus/tsdb v0.10.0 h1:If5rVCMTp6W2SiRAQFlbpJNgVlgMEd+U2GZckwK38ic=
 github.com/prometheus/tsdb v0.10.0/go.mod h1:oi49uRhEe9dPUTlS3JRZOwJuVi6tmh10QSgwXEyGCt4=
@@ -1059,8 +1060,8 @@ github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljT
 github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA=
 github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0=
 github.com/urfave/cli v1.22.2/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0=
-github.com/urfave/cli v1.22.5 h1:lNq9sAHXK2qfdI8W+GRItjCEkI+2oR4d+MEHy1CKXoU=
-github.com/urfave/cli v1.22.5/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0=
+github.com/urfave/cli v1.22.9 h1:cv3/KhXGBGjEXLC4bH0sLuJ9BewaAbpk5oyMOveu4pw=
+github.com/urfave/cli v1.22.9/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0=
 github.com/urfave/cli/v2 v2.3.0/go.mod h1:LJmUH05zAU44vOAcrfzZQKsZbVcdbOG8rtL3/XcUArI=
 github.com/urfave/cli/v2 v2.10.2 h1:x3p8awjp/2arX+Nl/G2040AZpOCHS/eMJJ1/a+mye4Y=
 github.com/urfave/cli/v2 v2.10.2/go.mod h1:f8iq5LtQ/bLxafbdBSLPPNsgaW0l/2fYYEHhAyPlwvo=
@@ -1557,8 +1558,8 @@ google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlba
 google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
 google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
 google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
-google.golang.org/protobuf v1.28.0 h1:w43yiav+6bVFTBQFZX0r7ipe9JQ1QsbMgHwbBziscLw=
-google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
+google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w=
+google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
 gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=

op-e2e/setup.go

@@ -15,6 +15,7 @@ import (
 	"github.com/ethereum-optimism/optimism/op-node/p2p"
 	"github.com/ethereum-optimism/optimism/op-node/rollup"
 	l2os "github.com/ethereum-optimism/optimism/op-proposer"
+	oplog "github.com/ethereum-optimism/optimism/op-service/log"
 	"github.com/ethereum/go-ethereum/accounts"
 	"github.com/ethereum/go-ethereum/accounts/abi/bind"
 	"github.com/ethereum/go-ethereum/common"
@@ -554,8 +555,10 @@ func (cfg SystemConfig) start() (*System, error) {
 		NumConfirmations:          1,
 		ResubmissionTimeout:       3 * time.Second,
 		SafeAbortNonceTooLowCount: 3,
-		LogLevel:                  "info",
-		LogTerminal:               true,
+		LogConfig: oplog.CLIConfig{
+			Level:  "info",
+			Format: "text",
+		},
 		Mnemonic:       sys.cfg.Mnemonic,
 		L2OutputHDPath: sys.cfg.L2OutputHDPath,
 	}, "", sys.cfg.Loggers["proposer"])
@@ -579,8 +582,10 @@ func (cfg SystemConfig) start() (*System, error) {
 		NumConfirmations:          1,
 		ResubmissionTimeout:       5 * time.Second,
 		SafeAbortNonceTooLowCount: 3,
-		LogLevel:                   "info", // ignored if started in-process this way
-		LogTerminal:                true,   // ignored
+		LogConfig: oplog.CLIConfig{
+			Level:  "info",
+			Format: "text",
+		},
 		Mnemonic:                   sys.cfg.Mnemonic,
 		SequencerHDPath:            sys.cfg.BatchSubmitterHDPath,
 		SequencerBatchInboxAddress: sys.cfg.RollupConfig.BatchInboxAddress.String(),

op-proposer/Dockerfile

@@ -7,6 +7,7 @@ COPY ./op-proposer/docker.go.work /app/go.work
 COPY ./op-bindings /app/op-bindings
 COPY ./op-node /app/op-node
 COPY ./op-proposer /app/op-proposer
+COPY ./op-service /app/op-service
 
 WORKDIR /app/op-proposer
 

op-proposer/cmd/main.go

@@ -4,6 +4,8 @@ import (
 	"fmt"
 	"os"
 
+	oplog "github.com/ethereum-optimism/optimism/op-service/log"
+
 	proposer "github.com/ethereum-optimism/optimism/op-proposer"
 
 	"github.com/ethereum/go-ethereum/log"
@@ -19,14 +21,7 @@ var (
 )
 
 func main() {
-	// Set up logger with a default INFO level in case we fail to parse flags,
-	// otherwise the final critical log won't show what the parsing error was.
-	log.Root().SetHandler(
-		log.LvlFilterHandler(
-			log.LvlInfo,
-			log.StreamHandler(os.Stdout, log.TerminalFormat(true)),
-		),
-	)
+	oplog.SetupDefaults()
 
 	app := cli.NewApp()
 	app.Flags = flags.Flags

op-proposer/config.go

@@ -3,6 +3,11 @@ package op_proposer
 import (
 	"time"
 
+	oplog "github.com/ethereum-optimism/optimism/op-service/log"
+	opmetrics "github.com/ethereum-optimism/optimism/op-service/metrics"
+	oppprof "github.com/ethereum-optimism/optimism/op-service/pprof"
+	oprpc "github.com/ethereum-optimism/optimism/op-service/rpc"
+
 	"github.com/urfave/cli"
 
 	"github.com/ethereum-optimism/optimism/op-proposer/flags"
@@ -52,19 +57,31 @@ type Config struct {
 	// PrivateKey is the private key used for l2output transactions.
 	PrivateKey string
 
+	RPCConfig oprpc.CLIConfig
+
 	/* Optional Params */
 
-	// LogLevel is the lowest log level that will be output.
-	LogLevel string
+	LogConfig oplog.CLIConfig
 
-	// LogTerminal if true, will log to stdout in terminal format. Otherwise the
-	// output will be in JSON format.
-	LogTerminal bool
+	MetricsConfig opmetrics.CLIConfig
+
+	PprofConfig oppprof.CLIConfig
+}
 
-	// Flags for the pprof server
-	PprofEnabled bool
-	PprofAddr    string
-	PprofPort    string
+func (c Config) Check() error {
+	if err := c.RPCConfig.Check(); err != nil {
+		return err
+	}
+	if err := c.LogConfig.Check(); err != nil {
+		return err
+	}
+	if err := c.MetricsConfig.Check(); err != nil {
+		return err
+	}
+	if err := c.PprofConfig.Check(); err != nil {
+		return err
+	}
+	return nil
 }
 
 // NewConfig parses the Config from the provided flags or environment variables.
@@ -82,11 +99,9 @@ func NewConfig(ctx *cli.Context) Config {
 		Mnemonic:                  ctx.GlobalString(flags.MnemonicFlag.Name),
 		L2OutputHDPath:            ctx.GlobalString(flags.L2OutputHDPathFlag.Name),
 		PrivateKey:                ctx.GlobalString(flags.PrivateKeyFlag.Name),
-		/* Optional Flags */
-		LogLevel:     ctx.GlobalString(flags.LogLevelFlag.Name),
-		LogTerminal:  ctx.GlobalBool(flags.LogTerminalFlag.Name),
-		PprofEnabled: ctx.GlobalBool(flags.PprofEnabledFlag.Name),
-		PprofAddr:    ctx.GlobalString(flags.PprofAddrFlag.Name),
-		PprofPort:    ctx.GlobalString(flags.PprofPortFlag.Name),
+		RPCConfig:                 oprpc.ReadCLIConfig(ctx),
+		LogConfig:                 oplog.ReadCLIConfig(ctx),
+		MetricsConfig:             opmetrics.ReadCLIConfig(ctx),
+		PprofConfig:               oppprof.ReadCLIConfig(ctx),
 	}
 }

op-proposer/docker.go.work

@@ -4,4 +4,5 @@ use (
 	./op-bindings
 	./op-node
 	./op-proposer
+	./op-service
 )

op-proposer/flags/flags.go

 package flags
 
 import (
+	opservice "github.com/ethereum-optimism/optimism/op-service"
+	oplog "github.com/ethereum-optimism/optimism/op-service/log"
+	opmetrics "github.com/ethereum-optimism/optimism/op-service/metrics"
+	oppprof "github.com/ethereum-optimism/optimism/op-service/pprof"
+	oprpc "github.com/ethereum-optimism/optimism/op-service/rpc"
 	"github.com/urfave/cli"
 )
 
-const envVarPrefix = "OP_PROPOSER_"
-
-func prefixEnvVar(name string) string {
-	return envVarPrefix + name
-}
+const envVarPrefix = "OP_PROPOSER"
 
 var (
 	/* Required Flags */
@@ -17,39 +18,39 @@ var (
 		Name:     "l1-eth-rpc",
 		Usage:    "HTTP provider URL for L1",
 		Required: true,
-		EnvVar:   prefixEnvVar("L1_ETH_RPC"),
+		EnvVar:   opservice.PrefixEnvVar(envVarPrefix, "L1_ETH_RPC"),
 	}
 	L2EthRpcFlag = cli.StringFlag{
 		Name:     "l2-eth-rpc",
 		Usage:    "HTTP provider URL for L2",
 		Required: true,
-		EnvVar:   prefixEnvVar("L2_ETH_RPC"),
+		EnvVar:   opservice.PrefixEnvVar(envVarPrefix, "L2_ETH_RPC"),
 	}
 	RollupRpcFlag = cli.StringFlag{
 		Name:     "rollup-rpc",
 		Usage:    "HTTP provider URL for the rollup node",
 		Required: true,
-		EnvVar:   prefixEnvVar("ROLLUP_RPC"),
+		EnvVar:   opservice.PrefixEnvVar(envVarPrefix, "ROLLUP_RPC"),
 	}
 	L2OOAddressFlag = cli.StringFlag{
 		Name:     "l2oo-address",
 		Usage:    "Address of the L2OutputOracle contract",
 		Required: true,
-		EnvVar:   prefixEnvVar("L2OO_ADDRESS"),
+		EnvVar:   opservice.PrefixEnvVar(envVarPrefix, "L2OO_ADDRESS"),
 	}
 	PollIntervalFlag = cli.DurationFlag{
 		Name: "poll-interval",
 		Usage: "Delay between querying L2 for more transactions and " +
 			"creating a new batch",
 		Required: true,
-		EnvVar:   prefixEnvVar("POLL_INTERVAL"),
+		EnvVar:   opservice.PrefixEnvVar(envVarPrefix, "POLL_INTERVAL"),
 	}
 	NumConfirmationsFlag = cli.Uint64Flag{
 		Name: "num-confirmations",
 		Usage: "Number of confirmations which we will wait after " +
 			"appending a new batch",
 		Required: true,
-		EnvVar:   prefixEnvVar("NUM_CONFIRMATIONS"),
+		EnvVar:   opservice.PrefixEnvVar(envVarPrefix, "NUM_CONFIRMATIONS"),
 	}
 	SafeAbortNonceTooLowCountFlag = cli.Uint64Flag{
 		Name: "safe-abort-nonce-too-low-count",
@@ -57,63 +58,31 @@ var (
 			"give up on a tx at a particular nonce without receiving " +
 			"confirmation",
 		Required: true,
-		EnvVar:   prefixEnvVar("SAFE_ABORT_NONCE_TOO_LOW_COUNT"),
+		EnvVar:   opservice.PrefixEnvVar(envVarPrefix, "SAFE_ABORT_NONCE_TOO_LOW_COUNT"),
 	}
 	ResubmissionTimeoutFlag = cli.DurationFlag{
 		Name: "resubmission-timeout",
 		Usage: "Duration we will wait before resubmitting a " +
 			"transaction to L1",
 		Required: true,
-		EnvVar:   prefixEnvVar("RESUBMISSION_TIMEOUT"),
+		EnvVar:   opservice.PrefixEnvVar(envVarPrefix, "RESUBMISSION_TIMEOUT"),
 	}
 	MnemonicFlag = cli.StringFlag{
 		Name: "mnemonic",
 		Usage: "The mnemonic used to derive the wallets for either the " +
 			"sequencer or the l2output",
-		EnvVar: prefixEnvVar("MNEMONIC"),
+		EnvVar: opservice.PrefixEnvVar(envVarPrefix, "MNEMONIC"),
 	}
 	L2OutputHDPathFlag = cli.StringFlag{
 		Name: "l2-output-hd-path",
 		Usage: "The HD path used to derive the l2output wallet from the " +
 			"mnemonic. The mnemonic flag must also be set.",
-		EnvVar: prefixEnvVar("L2_OUTPUT_HD_PATH"),
+		EnvVar: opservice.PrefixEnvVar(envVarPrefix, "L2_OUTPUT_HD_PATH"),
 	}
 	PrivateKeyFlag = cli.StringFlag{
 		Name:   "private-key",
 		Usage:  "The private key to use with the l2output wallet. Must not be used with mnemonic.",
-		EnvVar: prefixEnvVar("PRIVATE_KEY"),
-	}
-
-	/* Optional Flags */
-
-	LogLevelFlag = cli.StringFlag{
-		Name:   "log.level",
-		Usage:  "The lowest log level that will be output",
-		Value:  "info",
-		EnvVar: prefixEnvVar("LOG_LEVEL"),
-	}
-	LogTerminalFlag = cli.BoolFlag{
-		Name: "log.terminal",
-		Usage: "If true, outputs logs in terminal format, otherwise prints " +
-			"in JSON format.",
-		EnvVar: prefixEnvVar("LOG_TERMINAL"),
-	}
-	PprofEnabledFlag = cli.BoolFlag{
-		Name:   "pprof.enabled",
-		Usage:  "Enable the pprof server",
-		EnvVar: prefixEnvVar("PPROF_ENABLED"),
-	}
-	PprofAddrFlag = cli.StringFlag{
-		Name:   "pprof.addr",
-		Usage:  "pprof listening address",
-		Value:  "0.0.0.0",
-		EnvVar: prefixEnvVar("PPROF_ADDR"),
-	}
-	PprofPortFlag = cli.IntFlag{
-		Name:   "pprof.port",
-		Usage:  "pprof listening port",
-		Value:  6060,
-		EnvVar: prefixEnvVar("PPROF_PORT"),
+		EnvVar: opservice.PrefixEnvVar(envVarPrefix, "PRIVATE_KEY"),
 	}
 )
 
@@ -132,12 +101,17 @@ var optionalFlags = []cli.Flag{
 	MnemonicFlag,
 	L2OutputHDPathFlag,
 	PrivateKeyFlag,
-	LogLevelFlag,
-	LogTerminalFlag,
-	PprofEnabledFlag,
-	PprofAddrFlag,
-	PprofPortFlag,
+}
+
+func init() {
+	requiredFlags = append(requiredFlags, oprpc.CLIFlags(envVarPrefix)...)
+
+	optionalFlags = append(optionalFlags, oplog.CLIFlags(envVarPrefix)...)
+	optionalFlags = append(optionalFlags, opmetrics.CLIFlags(envVarPrefix)...)
+	optionalFlags = append(optionalFlags, oppprof.CLIFlags(envVarPrefix)...)
+
+	Flags = append(requiredFlags, optionalFlags...)
 }
 
 // Flags contains the list of configuration options available to the binary.
-var Flags = append(requiredFlags, optionalFlags...)
+var Flags []cli.Flag

op-proposer/flags/flags_test.go

-package flags
-
-import (
-	"testing"
-
-	"github.com/stretchr/testify/require"
-	"github.com/urfave/cli"
-)
-
-// TestRequiredFlagsSetRequired asserts that all flags deemed required properly
-// have the Required field set to true.
-func TestRequiredFlagsSetRequired(t *testing.T) {
-	for _, flag := range requiredFlags {
-		reqFlag, ok := flag.(cli.RequiredFlag)
-		require.True(t, ok)
-		require.True(t, reqFlag.IsRequired())
-	}
-}
-
-// TestOptionalFlagsDontSetRequired asserts that all flags deemed optional set
-// the Required field to false.
-func TestOptionalFlagsDontSetRequired(t *testing.T) {
-	for _, flag := range optionalFlags {
-		reqFlag, ok := flag.(cli.RequiredFlag)
-		require.True(t, ok)
-		require.False(t, reqFlag.IsRequired())
-	}
-}

op-proposer/go.mod

@@ -2,13 +2,16 @@ module github.com/ethereum-optimism/optimism/op-proposer
 
 go 1.18
 
+replace github.com/ethereum-optimism/optimism/op-service v0.0.0 => ../op-service
+
 require (
 	github.com/ethereum-optimism/optimism/op-bindings v0.3.0
 	github.com/ethereum-optimism/optimism/op-node v0.3.0
+	github.com/ethereum-optimism/optimism/op-service v0.0.0
 	github.com/ethereum/go-ethereum v1.10.21
 	github.com/miguelmota/go-ethereum-hdwallet v0.1.1
 	github.com/stretchr/testify v1.8.0
-	github.com/urfave/cli v1.22.5
+	github.com/urfave/cli v1.22.9
 )
 
 require (
@@ -23,7 +26,8 @@ require (
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/deckarep/golang-set v1.8.0 // indirect
 	github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.1 // indirect
-	github.com/go-kit/kit v0.10.0 // indirect
+	github.com/edsrzf/mmap-go v1.1.0 // indirect
+	github.com/fjl/memsize v0.0.1 // indirect
 	github.com/go-ole/go-ole v1.2.6 // indirect
 	github.com/go-stack/stack v1.8.1 // indirect
 	github.com/golang-jwt/jwt/v4 v4.3.0 // indirect
@@ -31,31 +35,45 @@ require (
 	github.com/golang/snappy v0.0.4 // indirect
 	github.com/google/uuid v1.3.0 // indirect
 	github.com/gorilla/websocket v1.5.0 // indirect
+	github.com/hashicorp/go-bexpr v0.1.11 // indirect
 	github.com/hashicorp/golang-lru v0.5.5-0.20210104140557-80c98217689d // indirect
 	github.com/holiman/bloomfilter/v2 v2.0.3 // indirect
 	github.com/holiman/uint256 v1.2.0 // indirect
+	github.com/huin/goupnp v1.0.3 // indirect
+	github.com/jackpal/go-nat-pmp v1.0.2 // indirect
+	github.com/kr/text v0.2.0 // indirect
+	github.com/mattn/go-colorable v0.1.12 // indirect
+	github.com/mattn/go-isatty v0.0.14 // indirect
 	github.com/mattn/go-runewidth v0.0.13 // indirect
 	github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect
+	github.com/mitchellh/mapstructure v1.5.0 // indirect
+	github.com/mitchellh/pointerstructure v1.2.1 // indirect
 	github.com/olekukonko/tablewriter v0.0.5 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
-	github.com/prometheus/client_golang v1.12.2 // indirect
+	github.com/prometheus/client_golang v1.13.0 // indirect
 	github.com/prometheus/client_model v0.2.0 // indirect
-	github.com/prometheus/common v0.35.0 // indirect
-	github.com/prometheus/procfs v0.7.3 // indirect
+	github.com/prometheus/common v0.37.0 // indirect
+	github.com/prometheus/procfs v0.8.0 // indirect
 	github.com/prometheus/tsdb v0.10.0 // indirect
 	github.com/rivo/uniseg v0.2.0 // indirect
 	github.com/rjeczalik/notify v0.9.2 // indirect
+	github.com/rs/cors v1.8.2 // indirect
 	github.com/russross/blackfriday/v2 v2.1.0 // indirect
 	github.com/shirou/gopsutil v3.21.11+incompatible // indirect
 	github.com/syndtr/goleveldb v1.0.1-0.20220614013038-64ee5596c38a // indirect
 	github.com/tklauser/go-sysconf v0.3.10 // indirect
 	github.com/tklauser/numcpus v0.4.0 // indirect
 	github.com/tyler-smith/go-bip39 v1.1.0 // indirect
+	github.com/urfave/cli/v2 v2.10.2 // indirect
+	github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect
 	github.com/yusufpapurcu/wmi v1.2.2 // indirect
 	golang.org/x/crypto v0.0.0-20220525230936-793ad666bf5e // indirect
+	golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f // indirect
 	golang.org/x/sys v0.0.0-20220701225701-179beb0bd1a1 // indirect
-	google.golang.org/protobuf v1.28.0 // indirect
+	golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 // indirect
+	golang.org/x/time v0.0.0-20220722155302-e5dcc9cfc0b9 // indirect
+	google.golang.org/protobuf v1.28.1 // indirect
 	gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 )

op-proposer/l2_output_submitter.go

@@ -5,8 +5,6 @@ import (
 	"crypto/ecdsa"
 	"errors"
 	"fmt"
-	"net"
-	"net/http"
 	_ "net/http/pprof"
 	"os"
 	"os/signal"
@@ -16,6 +14,11 @@ import (
 
 	"github.com/ethereum/go-ethereum/crypto"
 
+	oplog "github.com/ethereum-optimism/optimism/op-service/log"
+	opmetrics "github.com/ethereum-optimism/optimism/op-service/metrics"
+	oppprof "github.com/ethereum-optimism/optimism/op-service/pprof"
+	oprpc "github.com/ethereum-optimism/optimism/op-service/rpc"
+
 	"github.com/ethereum-optimism/optimism/op-proposer/drivers/l2output"
 	"github.com/ethereum-optimism/optimism/op-proposer/rollupclient"
 	"github.com/ethereum-optimism/optimism/op-proposer/txmgr"
@@ -39,25 +42,13 @@ const (
 // of a closure allows the parameters bound to the top-level main package, e.g.
 // GitVersion, to be captured and used once the function is executed.
 func Main(version string) func(ctx *cli.Context) error {
-	return func(ctx *cli.Context) error {
-		cfg := NewConfig(ctx)
-
-		// Set up our logging to stdout.
-		var logHandler log.Handler
-		if cfg.LogTerminal {
-			logHandler = log.StreamHandler(os.Stdout, log.TerminalFormat(true))
-		} else {
-			logHandler = log.StreamHandler(os.Stdout, log.JSONFormat())
-		}
-
-		logLevel, err := log.LvlFromString(cfg.LogLevel)
-		if err != nil {
-			return err
+	return func(cliCtx *cli.Context) error {
+		cfg := NewConfig(cliCtx)
+		if err := cfg.Check(); err != nil {
+			return fmt.Errorf("invalid CLI flags: %w", err)
 		}
 
-		l := log.New()
-		l.SetHandler(log.LvlFilterHandler(logLevel, logHandler))
-
+		l := oplog.NewLogger(cfg.LogConfig)
 		l.Info("Initializing L2 Output Submitter")
 
 		l2OutputSubmitter, err := NewL2OutputSubmitter(cfg, version, l)
@@ -74,29 +65,41 @@ func Main(version string) func(ctx *cli.Context) error {
 		}
 		defer l2OutputSubmitter.Stop()
 
-		l.Info("L2 Output Submitter started")
+		ctx, cancel := context.WithCancel(context.Background())
 
-		if cfg.PprofEnabled {
-			var srv http.Server
-			srv.Addr = net.JoinHostPort(cfg.PprofAddr, cfg.PprofPort)
-			// Start pprof server + register it's shutdown
+		l.Info("L2 Output Submitter started")
+		pprofConfig := cfg.PprofConfig
+		if pprofConfig.Enabled {
+			l.Info("starting pprof", "addr", pprofConfig.ListenAddr, "port", pprofConfig.ListenPort)
 			go func() {
-				l.Info("pprof server started", "addr", srv.Addr)
-				if err := srv.ListenAndServe(); err != http.ErrServerClosed {
-					l.Error("error in pprof server", "err", err)
-				} else {
-					l.Info("pprof server shutting down")
+				if err := oppprof.ListenAndServe(ctx, pprofConfig.ListenAddr, pprofConfig.ListenPort); err != nil {
+					l.Error("error starting pprof", "err", err)
 				}
-
 			}()
-			defer func() {
-				shutCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
-				defer cancel()
-				err := srv.Shutdown(shutCtx)
-				l.Info("pprof server shut down", "err", err)
+		}
+
+		registry := opmetrics.NewRegistry()
+		metricsCfg := cfg.MetricsConfig
+		if metricsCfg.Enabled {
+			l.Info("starting metrics server", "addr", metricsCfg.ListenAddr, "port", metricsCfg.ListenPort)
+			go func() {
+				if err := opmetrics.ListenAndServe(ctx, registry, metricsCfg.ListenAddr, metricsCfg.ListenPort); err != nil {
+					l.Error("error starting metrics server", err)
+				}
 			}()
 		}
 
+		rpcCfg := cfg.RPCConfig
+		server := oprpc.NewServer(
+			rpcCfg.ListenAddr,
+			rpcCfg.ListenPort,
+			version,
+		)
+		if err := server.Start(); err != nil {
+			cancel()
+			return fmt.Errorf("error starting RPC server: %w", err)
+		}
+
 		interruptChannel := make(chan os.Signal, 1)
 		signal.Notify(interruptChannel, []os.Signal{
 			os.Interrupt,
@@ -105,6 +108,7 @@ func Main(version string) func(ctx *cli.Context) error {
 			syscall.SIGQUIT,
 		}...)
 		<-interruptChannel
+		cancel()
 
 		return nil
 	}

op-service/Makefile

+test:
+	go test -v ./...
+
+lint:
+	golangci-lint run -E asciicheck,goimports,misspell ./...
+
+.PHONY: \
+	test \
+	lint

op-service/go.mod

+module github.com/ethereum-optimism/optimism/op-service
+
+go 1.18
+
+require (
+	github.com/ethereum/go-ethereum v1.10.21
+	github.com/prometheus/client_golang v1.13.0
+	github.com/stretchr/testify v1.8.0
+	github.com/urfave/cli v1.22.9
+	golang.org/x/term v0.0.0-20210927222741-03fcf44c2211
+)
+
+require (
+	github.com/VictoriaMetrics/fastcache v1.9.0 // indirect
+	github.com/beorn7/perks v1.0.1 // indirect
+	github.com/btcsuite/btcd/btcec/v2 v2.2.0 // indirect
+	github.com/cespare/xxhash/v2 v2.1.2 // indirect
+	github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect
+	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/deckarep/golang-set v1.8.0 // indirect
+	github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.1 // indirect
+	github.com/fjl/memsize v0.0.1 // indirect
+	github.com/go-kit/kit v0.10.0 // indirect
+	github.com/go-ole/go-ole v1.2.6 // indirect
+	github.com/go-stack/stack v1.8.1 // indirect
+	github.com/golang-jwt/jwt/v4 v4.3.0 // indirect
+	github.com/golang/protobuf v1.5.2 // indirect
+	github.com/golang/snappy v0.0.4 // indirect
+	github.com/gorilla/websocket v1.5.0 // indirect
+	github.com/hashicorp/go-bexpr v0.1.11 // indirect
+	github.com/hashicorp/golang-lru v0.5.5-0.20210104140557-80c98217689d // indirect
+	github.com/huin/goupnp v1.0.3 // indirect
+	github.com/jackpal/go-nat-pmp v1.0.2 // indirect
+	github.com/kr/pretty v0.3.0 // indirect
+	github.com/mattn/go-colorable v0.1.12 // indirect
+	github.com/mattn/go-isatty v0.0.14 // indirect
+	github.com/mattn/go-runewidth v0.0.13 // indirect
+	github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect
+	github.com/mitchellh/mapstructure v1.5.0 // indirect
+	github.com/mitchellh/pointerstructure v1.2.1 // indirect
+	github.com/olekukonko/tablewriter v0.0.5 // indirect
+	github.com/pkg/errors v0.9.1 // indirect
+	github.com/pmezard/go-difflib v1.0.0 // indirect
+	github.com/prometheus/client_model v0.2.0 // indirect
+	github.com/prometheus/common v0.37.0 // indirect
+	github.com/prometheus/procfs v0.8.0 // indirect
+	github.com/prometheus/tsdb v0.10.0 // indirect
+	github.com/rivo/uniseg v0.2.0 // indirect
+	github.com/rogpeppe/go-internal v1.8.1 // indirect
+	github.com/rs/cors v1.8.2 // indirect
+	github.com/russross/blackfriday/v2 v2.1.0 // indirect
+	github.com/shirou/gopsutil v3.21.11+incompatible // indirect
+	github.com/syndtr/goleveldb v1.0.1-0.20220614013038-64ee5596c38a // indirect
+	github.com/tklauser/go-sysconf v0.3.10 // indirect
+	github.com/tklauser/numcpus v0.4.0 // indirect
+	github.com/urfave/cli/v2 v2.10.2 // indirect
+	github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect
+	github.com/yusufpapurcu/wmi v1.2.2 // indirect
+	golang.org/x/crypto v0.0.0-20220525230936-793ad666bf5e // indirect
+	golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f // indirect
+	golang.org/x/sys v0.0.0-20220701225701-179beb0bd1a1 // indirect
+	google.golang.org/protobuf v1.28.1 // indirect
+	gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect
+	gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
+)
+
+replace github.com/ethereum/go-ethereum v1.10.21 => github.com/ethereum-optimism/reference-optimistic-geth v0.0.0-20220803173305-1c9d4cc76a6e

op-service/httputil/server.go

+package httputil
+
+import (
+	"context"
+	"fmt"
+	"net/http"
+	"time"
+)
+
+func ListenAndServeContext(ctx context.Context, server *http.Server) error {
+	errCh := make(chan error)
+	go func() {
+		errCh <- server.ListenAndServe()
+	}()
+
+	// verify that the server comes up
+	tick := time.NewTimer(10 * time.Millisecond)
+	select {
+	case err := <-errCh:
+		return fmt.Errorf("http server failed: %w", err)
+	case <-tick.C:
+		break
+	}
+
+	<-ctx.Done()
+	return ctx.Err()
+}

op-service/httputil/wrapped_response_writer.go

+package httputil
+
+import "net/http"
+
+type WrappedResponseWriter struct {
+	StatusCode  int
+	ResponseLen int
+
+	w           http.ResponseWriter
+	wroteHeader bool
+}
+
+func NewWrappedResponseWriter(w http.ResponseWriter) *WrappedResponseWriter {
+	return &WrappedResponseWriter{
+		StatusCode: 200,
+		w:          w,
+	}
+}
+
+func (w *WrappedResponseWriter) Header() http.Header {
+	return w.w.Header()
+}
+
+func (w *WrappedResponseWriter) Write(bytes []byte) (int, error) {
+	n, err := w.w.Write(bytes)
+	w.ResponseLen += n
+	return n, err
+}
+
+func (w *WrappedResponseWriter) WriteHeader(statusCode int) {
+	if w.wroteHeader {
+		return
+	}
+
+	w.wroteHeader = true
+	w.StatusCode = statusCode
+	w.w.WriteHeader(statusCode)
+}

op-service/log/cli.go

+package log
+
+import (
+	"fmt"
+	"os"
+	"strings"
+
+	opservice "github.com/ethereum-optimism/optimism/op-service"
+	"github.com/ethereum/go-ethereum/log"
+	"github.com/urfave/cli"
+	"golang.org/x/term"
+)
+
+const (
+	LevelFlagName  = "log.level"
+	FormatFlagName = "log.format"
+	ColorFlagName  = "log.color"
+)
+
+func CLIFlags(envPrefix string) []cli.Flag {
+	return []cli.Flag{
+		cli.StringFlag{
+			Name:   LevelFlagName,
+			Usage:  "The lowest log level that will be output",
+			Value:  "info",
+			EnvVar: opservice.PrefixEnvVar(envPrefix, "LOG_LEVEL"),
+		},
+		cli.StringFlag{
+			Name:   FormatFlagName,
+			Usage:  "Format the log output. Supported formats: 'text', 'json'",
+			Value:  "text",
+			EnvVar: opservice.PrefixEnvVar(envPrefix, "LOG_FORMAT"),
+		},
+		cli.BoolFlag{
+			Name:   ColorFlagName,
+			Usage:  "Color the log output",
+			EnvVar: opservice.PrefixEnvVar(envPrefix, "LOG_COLOR"),
+		},
+	}
+}
+
+type CLIConfig struct {
+	Level  string // Log level: trace, debug, info, warn, error, crit. Capitals are accepted too.
+	Color  bool   // Color the log output. Defaults to true if terminal is detected.
+	Format string // Format the log output. Supported formats: 'text', 'json'
+}
+
+func (cfg CLIConfig) Check() error {
+	switch cfg.Format {
+	case "json", "json-pretty", "terminal", "text":
+	default:
+		return fmt.Errorf("unrecognized log format: %s", cfg.Format)
+	}
+
+	level := strings.ToLower(cfg.Level)
+	_, err := log.LvlFromString(level)
+	if err != nil {
+		return fmt.Errorf("unrecognized log level: %w", err)
+	}
+	return nil
+}
+
+func NewLogger(cfg CLIConfig) log.Logger {
+	handler := log.StreamHandler(os.Stdout, Format(cfg.Format, cfg.Color))
+	handler = log.SyncHandler(handler)
+	log.LvlFilterHandler(Level(cfg.Level), handler)
+	logger := log.New()
+	logger.SetHandler(handler)
+	return logger
+}
+
+func DefaultCLIConfig() CLIConfig {
+	return CLIConfig{
+		Level:  "info",
+		Format: "text",
+		Color:  term.IsTerminal(int(os.Stdout.Fd())),
+	}
+}
+
+func ReadCLIConfig(ctx *cli.Context) CLIConfig {
+	cfg := DefaultCLIConfig()
+	cfg.Level = ctx.GlobalString(LevelFlagName)
+	cfg.Format = ctx.GlobalString(FormatFlagName)
+	cfg.Color = ctx.GlobalBool(ColorFlagName)
+	return cfg
+}
+
+// Format turns a string and color into a structured Format object
+func Format(lf string, color bool) log.Format {
+	switch lf {
+	case "json":
+		return log.JSONFormat()
+	case "json-pretty":
+		return log.JSONFormatEx(true, true)
+	case "text", "terminal":
+		return log.TerminalFormat(color)
+	default:
+		panic("Failed to create `log.Format` from options")
+	}
+}
+
+// Level parses the level string into an appropriate object
+func Level(s string) log.Lvl {
+	s = strings.ToLower(s) // ignore case
+	l, err := log.LvlFromString(s)
+	if err != nil {
+		panic(fmt.Sprintf("Could not parse log level: %v", err))
+	}
+	return l
+}

op-service/log/defaults.go

+package log
+
+import (
+	"os"
+
+	"github.com/ethereum/go-ethereum/log"
+)
+
+func SetupDefaults() {
+	log.Root().SetHandler(
+		log.LvlFilterHandler(
+			log.LvlInfo,
+			log.StreamHandler(os.Stdout, log.TerminalFormat(true)),
+		),
+	)
+}

op-service/log/http.go

+package log
+
+import (
+	"net/http"
+	"time"
+
+	"github.com/ethereum-optimism/optimism/op-service/httputil"
+	"github.com/ethereum/go-ethereum/log"
+)
+
+func NewLoggingMiddleware(lgr log.Logger, next http.Handler) http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		ww := httputil.NewWrappedResponseWriter(w)
+		start := time.Now()
+		next.ServeHTTP(ww, r)
+		lgr.Debug(
+			"served HTTP request",
+			"status", ww.StatusCode,
+			"response_len", ww.ResponseLen,
+			"path", r.URL.EscapedPath(),
+			"duration", time.Since(start),
+			"remote_addr", r.RemoteAddr,
+		)
+	})
+}

op-service/metrics/cli.go

+package metrics
+
+import (
+	"errors"
+	"math"
+
+	opservice "github.com/ethereum-optimism/optimism/op-service"
+
+	"github.com/urfave/cli"
+)
+
+const (
+	EnabledFlagName    = "metrics.enabled"
+	ListenAddrFlagName = "metrics.addr"
+	PortFlagName       = "metrics.port"
+)
+
+func CLIFlags(envPrefix string) []cli.Flag {
+	return []cli.Flag{
+		cli.BoolFlag{
+			Name:   EnabledFlagName,
+			Usage:  "Enable the metrics server",
+			EnvVar: opservice.PrefixEnvVar(envPrefix, "METRICS_ENABLED"),
+		},
+		cli.StringFlag{
+			Name:   ListenAddrFlagName,
+			Usage:  "Metrics listening address",
+			Value:  "0.0.0.0",
+			EnvVar: opservice.PrefixEnvVar(envPrefix, "METRICS_ADDR"),
+		},
+		cli.IntFlag{
+			Name:   PortFlagName,
+			Usage:  "Metrics listening port",
+			Value:  7300,
+			EnvVar: opservice.PrefixEnvVar(envPrefix, "METRICS_PORT"),
+		},
+	}
+}
+
+type CLIConfig struct {
+	Enabled    bool
+	ListenAddr string
+	ListenPort int
+}
+
+func (m CLIConfig) Check() error {
+	if !m.Enabled {
+		return nil
+	}
+
+	if m.ListenPort < 0 || m.ListenPort > math.MaxUint16 {
+		return errors.New("invalid metrics port")
+	}
+
+	return nil
+}
+
+func ReadCLIConfig(ctx *cli.Context) CLIConfig {
+	return CLIConfig{
+		Enabled:    ctx.GlobalBool(EnabledFlagName),
+		ListenAddr: ctx.GlobalString(ListenAddrFlagName),
+		ListenPort: ctx.GlobalInt(PortFlagName),
+	}
+}

op-service/metrics/http.go

+package metrics
+
+import (
+	"net/http"
+	"strconv"
+	"time"
+
+	"github.com/ethereum-optimism/optimism/op-service/httputil"
+
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promauto"
+)
+
+var httpLabels = []string{
+	"method",
+	"response_code",
+}
+
+type HTTPParams struct {
+	Method     string
+	StatusCode int
+}
+
+type HTTPRecorder interface {
+	RecordHTTPRequestDuration(params *HTTPParams, dur time.Duration)
+	RecordHTTPResponseSize(params *HTTPParams, size int)
+	RecordInflightRequest(params *HTTPParams, quantity int)
+	RecordHTTPRequest(params *HTTPParams)
+	RecordHTTPResponse(params *HTTPParams)
+}
+
+type noopHTTPRecorder struct{}
+
+var NoopHTTPRecorder = new(noopHTTPRecorder)
+
+func (n *noopHTTPRecorder) RecordHTTPRequestDuration(*HTTPParams, time.Duration) {}
+
+func (n *noopHTTPRecorder) RecordHTTPResponseSize(*HTTPParams, int) {}
+
+func (n *noopHTTPRecorder) RecordInflightRequest(*HTTPParams, int) {}
+
+func (n *noopHTTPRecorder) RecordHTTPRequest(*HTTPParams) {}
+
+func (n *noopHTTPRecorder) RecordHTTPResponse(*HTTPParams) {}
+
+type PromHTTPRecorder struct {
+	HTTPRequestDuration  *prometheus.HistogramVec
+	HTTPResponseSize     *prometheus.HistogramVec
+	HTTPInflightRequests *prometheus.GaugeVec
+	HTTPRequests         *prometheus.CounterVec
+	HTTPResponses        *prometheus.CounterVec
+}
+
+func NewPromHTTPRecorder(r *prometheus.Registry, ns string) HTTPRecorder {
+	return &PromHTTPRecorder{
+		HTTPRequestDuration: promauto.With(r).NewHistogramVec(prometheus.HistogramOpts{
+			Namespace: ns,
+			Name:      "http_request_duration_ms",
+			Help:      "Tracks HTTP request durations, in ms",
+			Buckets:   prometheus.DefBuckets,
+		}, httpLabels),
+		HTTPResponseSize: promauto.With(r).NewHistogramVec(prometheus.HistogramOpts{
+			Namespace: ns,
+			Name:      "http_response_size",
+			Help:      "Tracks HTTP response sizes",
+			Buckets:   prometheus.DefBuckets,
+		}, httpLabels),
+		HTTPInflightRequests: promauto.With(r).NewGaugeVec(prometheus.GaugeOpts{
+			Namespace: ns,
+			Name:      "http_inflight_requests_count",
+			Help:      "Tracks currently in-flight requests",
+		}, []string{"method"}),
+		HTTPRequests: promauto.With(r).NewCounterVec(prometheus.CounterOpts{
+			Namespace: ns,
+			Name:      "http_requests_count_total",
+			Help:      "Tracks total HTTP requests",
+		}, []string{"method"}),
+		HTTPResponses: promauto.With(r).NewCounterVec(prometheus.CounterOpts{
+			Namespace: ns,
+			Name:      "http_responses_count_total",
+			Help:      "Tracks total HTTP responses",
+		}, httpLabels),
+	}
+}
+
+func (p *PromHTTPRecorder) RecordHTTPRequestDuration(params *HTTPParams, dur time.Duration) {
+	p.HTTPRequestDuration.WithLabelValues(params.Method, strconv.Itoa(params.StatusCode)).
+		Observe(float64(dur.Milliseconds()))
+}
+
+func (p *PromHTTPRecorder) RecordHTTPResponseSize(params *HTTPParams, size int) {
+	p.HTTPResponseSize.WithLabelValues(params.Method, strconv.Itoa(params.StatusCode)).Observe(float64(size))
+}
+
+func (p *PromHTTPRecorder) RecordInflightRequest(params *HTTPParams, quantity int) {
+	p.HTTPInflightRequests.WithLabelValues(params.Method).Add(float64(quantity))
+}
+
+func (p *PromHTTPRecorder) RecordHTTPRequest(params *HTTPParams) {
+	p.HTTPRequests.WithLabelValues(params.Method).Inc()
+}
+
+func (p *PromHTTPRecorder) RecordHTTPResponse(params *HTTPParams) {
+	p.HTTPResponses.WithLabelValues(params.Method, strconv.Itoa(params.StatusCode)).Inc()
+}
+
+func NewHTTPRecordingMiddleware(rec HTTPRecorder, next http.Handler) http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		ww := httputil.NewWrappedResponseWriter(w)
+		params := &HTTPParams{
+			Method: r.Method,
+		}
+		rec.RecordInflightRequest(params, 1)
+		rec.RecordHTTPRequest(params)
+		start := time.Now()
+		next.ServeHTTP(ww, r)
+		params.StatusCode = ww.StatusCode
+		dur := time.Since(start)
+		rec.RecordHTTPResponse(params)
+		rec.RecordHTTPResponseSize(params, ww.ResponseLen)
+		rec.RecordHTTPRequestDuration(params, dur)
+		rec.RecordInflightRequest(params, -1)
+	})
+}

op-service/metrics/node_info.go

+package metrics
+
+import (
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promauto"
+)
+
+type NodeRecorder interface {
+	RecordUp()
+	RecordInfo(version string)
+}
+
+type noopNodeRecorder struct{}
+
+var NoopNodeRecorder = new(noopNodeRecorder)
+
+func (n *noopNodeRecorder) RecordUp() {}
+
+func (n *noopNodeRecorder) RecordInfo(string) {}
+
+type PromNodeRecorder struct {
+	Up   prometheus.Gauge
+	Info *prometheus.GaugeVec
+}
+
+func NewPromNodeRecorder(r *prometheus.Registry, ns string) NodeRecorder {
+	return &PromNodeRecorder{
+		Up: promauto.With(r).NewGauge(prometheus.GaugeOpts{
+			Namespace: ns,
+			Name:      "up",
+			Help:      "1 if the node has finished starting up",
+		}),
+		Info: promauto.With(r).NewGaugeVec(prometheus.GaugeOpts{
+			Namespace: ns,
+			Name:      "info",
+			Help:      "Pseudo-metric tracking version and config info",
+		}, []string{
+			"version",
+		}),
+	}
+}
+
+func (p *PromNodeRecorder) RecordUp() {
+	p.Up.Set(1)
+}
+
+func (p *PromNodeRecorder) RecordInfo(version string) {
+	p.Info.WithLabelValues(version).Set(1)
+}

op-service/metrics/registry.go

+package metrics
+
+import (
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/collectors"
+)
+
+func NewRegistry() *prometheus.Registry {
+	registry := prometheus.NewRegistry()
+	registry.MustRegister(collectors.NewProcessCollector(collectors.ProcessCollectorOpts{}))
+	registry.MustRegister(collectors.NewGoCollector())
+	return registry
+}

op-service/metrics/server.go

+package metrics
+
+import (
+	"context"
+	"net"
+	"net/http"
+	"strconv"
+
+	"github.com/ethereum-optimism/optimism/op-service/httputil"
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promhttp"
+)
+
+func ListenAndServe(ctx context.Context, r *prometheus.Registry, hostname string, port int) error {
+	addr := net.JoinHostPort(hostname, strconv.Itoa(port))
+	server := &http.Server{
+		Addr: addr,
+		Handler: promhttp.InstrumentMetricHandler(
+			r, promhttp.HandlerFor(r, promhttp.HandlerOpts{}),
+		),
+	}
+	return httputil.ListenAndServeContext(ctx, server)
+}

op-service/pprof/cli.go

+package pprof
+
+import (
+	"errors"
+	"math"
+
+	opservice "github.com/ethereum-optimism/optimism/op-service"
+	"github.com/urfave/cli"
+)
+
+const (
+	EnabledFlagName    = "pprof.enabled"
+	ListenAddrFlagName = "pprof.addr"
+	PortFlagName       = "pprof.port"
+)
+
+func CLIFlags(envPrefix string) []cli.Flag {
+	return []cli.Flag{
+		cli.BoolFlag{
+			Name:   EnabledFlagName,
+			Usage:  "Enable the pprof server",
+			EnvVar: opservice.PrefixEnvVar(envPrefix, "PPROF_ENABLED"),
+		},
+		cli.StringFlag{
+			Name:   ListenAddrFlagName,
+			Usage:  "pprof listening address",
+			Value:  "0.0.0.0",
+			EnvVar: opservice.PrefixEnvVar(envPrefix, "PPROF_ADDR"),
+		},
+		cli.IntFlag{
+			Name:   PortFlagName,
+			Usage:  "pprof listening port",
+			Value:  6060,
+			EnvVar: opservice.PrefixEnvVar(envPrefix, "PPROF_PORT"),
+		},
+	}
+}
+
+type CLIConfig struct {
+	Enabled    bool
+	ListenAddr string
+	ListenPort int
+}
+
+func (m CLIConfig) Check() error {
+	if !m.Enabled {
+		return nil
+	}
+
+	if m.ListenPort < 0 || m.ListenPort > math.MaxUint16 {
+		return errors.New("invalid pprof port")
+	}
+
+	return nil
+}
+
+func ReadCLIConfig(ctx *cli.Context) CLIConfig {
+	return CLIConfig{
+		Enabled:    ctx.GlobalBool(EnabledFlagName),
+		ListenAddr: ctx.GlobalString(ListenAddrFlagName),
+		ListenPort: ctx.GlobalInt(PortFlagName),
+	}
+}

op-service/pprof/server.go

+package pprof
+
+import (
+	"context"
+	"net"
+	"net/http"
+	"net/http/pprof"
+	"strconv"
+
+	"github.com/ethereum-optimism/optimism/op-service/httputil"
+)
+
+func ListenAndServe(ctx context.Context, hostname string, port int) error {
+	mux := http.NewServeMux()
+
+	// have to do below to support multiple servers, since the
+	// pprof import only uses DefaultServeMux
+	mux.Handle("/debug/pprof/", http.HandlerFunc(pprof.Index))
+	mux.Handle("/debug/pprof/cmdline", http.HandlerFunc(pprof.Cmdline))
+	mux.Handle("/debug/pprof/profile", http.HandlerFunc(pprof.Profile))
+	mux.Handle("/debug/pprof/symbol", http.HandlerFunc(pprof.Symbol))
+	mux.Handle("/debug/pprof/trace", http.HandlerFunc(pprof.Trace))
+
+	addr := net.JoinHostPort(hostname, strconv.Itoa(port))
+	server := &http.Server{
+		Addr:    addr,
+		Handler: mux,
+	}
+	return httputil.ListenAndServeContext(ctx, server)
+}

op-service/rpc/cli.go

+package rpc
+
+import (
+	"errors"
+	"math"
+
+	opservice "github.com/ethereum-optimism/optimism/op-service"
+	"github.com/urfave/cli"
+)
+
+const (
+	ListenAddrFlagName = "rpc.addr"
+	PortFlagName       = "rpc.port"
+)
+
+func CLIFlags(envPrefix string) []cli.Flag {
+	return []cli.Flag{
+		cli.StringFlag{
+			Name:   ListenAddrFlagName,
+			Usage:  "rpc listening address",
+			Value:  "0.0.0.0",
+			EnvVar: opservice.PrefixEnvVar(envPrefix, "RPC_ADDR"),
+		},
+		cli.IntFlag{
+			Name:   PortFlagName,
+			Usage:  "rpc listening port",
+			Value:  8545,
+			EnvVar: opservice.PrefixEnvVar(envPrefix, "RPC_PORT"),
+		},
+	}
+}
+
+type CLIConfig struct {
+	ListenAddr string
+	ListenPort int
+}
+
+func (c CLIConfig) Check() error {
+	if c.ListenPort < 0 || c.ListenPort > math.MaxUint16 {
+		return errors.New("invalid RPC port")
+	}
+
+	return nil
+}
+
+func ReadCLIConfig(ctx *cli.Context) CLIConfig {
+	return CLIConfig{
+		ListenAddr: ctx.GlobalString(ListenAddrFlagName),
+		ListenPort: ctx.GlobalInt(PortFlagName),
+	}
+}

op-service/rpc/server.go

+package rpc
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"net"
+	"net/http"
+	"strconv"
+	"time"
+
+	oplog "github.com/ethereum-optimism/optimism/op-service/log"
+	opmetrics "github.com/ethereum-optimism/optimism/op-service/metrics"
+
+	"github.com/ethereum/go-ethereum/log"
+	"github.com/ethereum/go-ethereum/node"
+	"github.com/ethereum/go-ethereum/rpc"
+)
+
+var wildcardHosts = []string{"*"}
+
+type Server struct {
+	endpoint       string
+	apis           []rpc.API
+	appVersion     string
+	healthzHandler http.Handler
+	corsHosts      []string
+	vHosts         []string
+	jwtSecret      []byte
+	rpcPath        string
+	healthzPath    string
+	httpRecorder   opmetrics.HTTPRecorder
+	httpServer     *http.Server
+	log            log.Logger
+}
+
+type ServerOption func(b *Server)
+
+func WithAPIs(apis []rpc.API) ServerOption {
+	return func(b *Server) {
+		b.apis = apis
+	}
+}
+
+func WithHealthzHandler(hdlr http.Handler) ServerOption {
+	return func(b *Server) {
+		b.healthzHandler = hdlr
+	}
+}
+
+func WithCORSHosts(hosts []string) ServerOption {
+	return func(b *Server) {
+		b.corsHosts = hosts
+	}
+}
+
+func WithVHosts(hosts []string) ServerOption {
+	return func(b *Server) {
+		b.vHosts = hosts
+	}
+}
+
+func WithJWTSecret(secret []byte) ServerOption {
+	return func(b *Server) {
+		b.jwtSecret = secret
+	}
+}
+
+func WithRPCPath(path string) ServerOption {
+	return func(b *Server) {
+		b.rpcPath = path
+	}
+}
+
+func WithHealthzPath(path string) ServerOption {
+	return func(b *Server) {
+		b.healthzPath = path
+	}
+}
+
+func WithHTTPRecorder(recorder opmetrics.HTTPRecorder) ServerOption {
+	return func(b *Server) {
+		b.httpRecorder = recorder
+	}
+}
+
+func WithLogger(lgr log.Logger) ServerOption {
+	return func(b *Server) {
+		b.log = lgr
+	}
+}
+
+func NewServer(host string, port int, appVersion string, opts ...ServerOption) *Server {
+	endpoint := net.JoinHostPort(host, strconv.Itoa(port))
+	bs := &Server{
+		endpoint:       endpoint,
+		appVersion:     appVersion,
+		healthzHandler: defaultHealthzHandler(appVersion),
+		corsHosts:      wildcardHosts,
+		vHosts:         wildcardHosts,
+		rpcPath:        "/",
+		healthzPath:    "/healthz",
+		httpRecorder:   opmetrics.NoopHTTPRecorder,
+		httpServer: &http.Server{
+			Addr: endpoint,
+		},
+		log: log.Root(),
+	}
+	for _, opt := range opts {
+		opt(bs)
+	}
+	bs.AddAPI(rpc.API{
+		Namespace: "health",
+		Service: &healthzAPI{
+			appVersion: appVersion,
+		},
+	})
+	return bs
+}
+
+func (b *Server) Endpoint() string {
+	return b.endpoint
+}
+
+func (b *Server) AddAPI(api rpc.API) {
+	b.apis = append(b.apis, api)
+}
+
+func (b *Server) Start() error {
+	srv := rpc.NewServer()
+	if err := node.RegisterApis(b.apis, nil, srv); err != nil {
+		return fmt.Errorf("error registering APIs: %w", err)
+	}
+
+	nodeHdlr := node.NewHTTPHandlerStack(srv, b.corsHosts, b.vHosts, b.jwtSecret)
+	mux := http.NewServeMux()
+	mux.Handle(b.rpcPath, nodeHdlr)
+	mux.Handle(b.healthzPath, b.healthzHandler)
+	metricsMW := oplog.NewLoggingMiddleware(b.log, opmetrics.NewHTTPRecordingMiddleware(b.httpRecorder, mux))
+	b.httpServer.Handler = metricsMW
+	errCh := make(chan error, 1)
+	go func() {
+		if err := b.httpServer.ListenAndServe(); err != nil {
+			errCh <- err
+		}
+	}()
+
+	// verify that the server comes up
+	tick := time.NewTimer(10 * time.Millisecond)
+	select {
+	case err := <-errCh:
+		return fmt.Errorf("http server failed: %w", err)
+	case <-tick.C:
+		return nil
+	}
+}
+
+func (b *Server) Stop() error {
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+	defer cancel()
+	_ = b.httpServer.Shutdown(ctx)
+	return nil
+}
+
+type HealthzResponse struct {
+	Version string `json:"version"`
+}
+
+func defaultHealthzHandler(appVersion string) http.HandlerFunc {
+	return func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Set("Content-Type", "application/json")
+		enc := json.NewEncoder(w)
+		_ = enc.Encode(&HealthzResponse{Version: appVersion})
+	}
+}
+
+type healthzAPI struct {
+	appVersion string
+}
+
+func (h *healthzAPI) Status() string {
+	return h.appVersion
+}

op-service/rpc/server_test.go

+package rpc
+
+import (
+	"fmt"
+	"io/ioutil"
+	"math/rand"
+	"net/http"
+	"testing"
+
+	"github.com/ethereum/go-ethereum/rpc"
+	"github.com/stretchr/testify/require"
+)
+
+type testAPI struct{}
+
+func (t *testAPI) Frobnicate(n int) int {
+	return n * 2
+}
+
+func TestBaseServer(t *testing.T) {
+	appVersion := "test"
+	server := NewServer(
+		"127.0.0.1",
+		10000+rand.Intn(22768),
+		appVersion,
+		WithAPIs([]rpc.API{
+			{
+				Namespace: "test",
+				Service:   new(testAPI),
+			},
+		}),
+	)
+	require.NoError(t, server.Start())
+	defer func() {
+		_ = server.Stop()
+	}()
+
+	rpcClient, err := rpc.Dial(fmt.Sprintf("http://%s", server.endpoint))
+	require.NoError(t, err)
+
+	t.Run("supports GET /healthz", func(t *testing.T) {
+		res, err := http.Get(fmt.Sprintf("http://%s/healthz", server.endpoint))
+		require.NoError(t, err)
+		body, err := ioutil.ReadAll(res.Body)
+		require.NoError(t, err)
+		require.EqualValues(t, fmt.Sprintf("{\"version\":\"%s\"}\n", appVersion), string(body))
+	})
+
+	t.Run("supports health_status", func(t *testing.T) {
+		var res string
+		require.NoError(t, rpcClient.Call(&res, "health_status"))
+		require.Equal(t, appVersion, res)
+	})
+
+	t.Run("supports additional RPC APIs", func(t *testing.T) {
+		var res int
+		require.NoError(t, rpcClient.Call(&res, "test_frobnicate", 2))
+		require.Equal(t, 4, res)
+	})
+}

op-service/util.go

+package op_service
+
+func PrefixEnvVar(prefix, suffix string) string {
+	return prefix + "_" + suffix
+}