go/proxyd: Add support for env vars in the config

Certain config parameters can now be read from the environment if environment variables as provided as config values. The following parameters can be read from the environment:

- `backends.*.rpc_url`
- `backends.*.ws_url`
- `backends.*.password`
- `authentication.*`

Example backend config:

```toml
[backends]
[backends.from_env]
rpc_url = "$SECRET_RPC_URL"
ws_url = "$SECRET_WS_URL"
```

Example authentication config. Note the additional double quotes around the secret key:

```toml
[authentication]
"$SECRET_AUTH_KEY" = "auth_key_alias"
```

Env vars can be escaped with two backslashes (`\\`). We have to use two backslashes because it's also the escape character in TOML.

Env vars defined in the config but not defined in the actual environment will result in an error that prevents `proxyd` from starting.

Metadata:

- Fixes: ENG-1728

.changeset/new-squids-hunt.md

+---
+'@eth-optimism/proxyd': minor
+---
+
+Adds ability to specify env vars in config

go/proxyd/config.go

 package proxyd
 
+import (
+	"fmt"
+	"os"
+	"strings"
+)
+
 type ServerConfig struct {
 	RPCHost          string `toml:"rpc_host"`
 	RPCPort          int    `toml:"rpc_port"`
@@ -26,11 +32,11 @@ type BackendOptions struct {
 }
 
 type BackendConfig struct {
-	Username   string `toml:"username"`
-	Password   string `toml:"password"`
-	RPCURL     string `toml:"rpc_url"`
-	WSURL      string `toml:"ws_url"`
-	MaxRPS     int    `toml:"max_rps"`
+	Username       string `toml:"username"`
+	Password       string `toml:"password"`
+	RPCURL         string `toml:"rpc_url"`
+	WSURL          string `toml:"ws_url"`
+	MaxRPS         int    `toml:"max_rps"`
 	MaxWSConns     int    `toml:"max_ws_conns"`
 	CAFile         string `toml:"ca_file"`
 	ClientCertFile string `toml:"client_cert_file"`
@@ -59,3 +65,19 @@ type Config struct {
 	RPCMethodMappings map[string]string   `toml:"rpc_method_mappings"`
 	WSMethodWhitelist []string            `toml:"ws_method_whitelist"`
 }
+
+func ReadFromEnvOrConfig(value string) (string, error) {
+	if strings.HasPrefix(value, "$") {
+		envValue := os.Getenv(strings.TrimPrefix(value, "$"))
+		if envValue == "" {
+			return "", fmt.Errorf("config env var %s not found", value)
+		}
+		return envValue, nil
+	}
+
+	if strings.HasPrefix(value, "\\") {
+		return strings.TrimPrefix(value, "\\"), nil
+	}
+
+	return value, nil
+}

go/proxyd/example.config.toml

@@ -44,11 +44,15 @@ out_of_service_seconds = 600
 [backends]
 # A map of backends by name.
 [backends.infura]
-# The URL to contact the backend at.
+# The URL to contact the backend at. Will be read from the environment
+# if an environment variable prefixed with $ is provided.
 rpc_url = ""
-# The WS URL to contact the backend at.
+# The WS URL to contact the backend at. Will be read from the environment
+# if an environment variable prefixed with $ is provided.
 ws_url = ""
 username = ""
+# An HTTP Basic password to authenticate with the backend. Will be read from
+# the environment if an environment variable prefixed with $ is provided.
 password = ""
 max_rps = 3
 max_ws_conns = 1
@@ -60,7 +64,6 @@ client_cert_file = ""
 client_key_file = ""
 
 [backends.alchemy]
-# The URL to contact the backend at.
 rpc_url = ""
 ws_url = ""
 username = ""
@@ -79,7 +82,10 @@ backends = ["alchemy"]
 # proxyd will only accept authenticated requests.
 [authentication]
 # Mapping of auth key to alias. The alias is used to provide a human-
-# readable name for the auth key in monitoring.
+# readable name for the auth key in monitoring. The auth key will be
+# read from the environment if an environment variable prefixed with $
+# is provided. Note that you will need to quote the environment variable
+# in order for it to be value TOML, e.g. "$FOO_AUTH_KEY" = "foo_alias".
 secret = "test"
 
 # Mapping of methods to backend groups.

go/proxyd/proxyd.go

@@ -47,10 +47,18 @@ func Start(config *Config) error {
 	for name, cfg := range config.Backends {
 		opts := make([]BackendOpt, 0)
 
-		if cfg.RPCURL == "" {
+		rpcURL, err := ReadFromEnvOrConfig(cfg.RPCURL)
+		if err != nil {
+			return err
+		}
+		wsURL, err := ReadFromEnvOrConfig(cfg.WSURL)
+		if err != nil {
+			return err
+		}
+		if rpcURL == "" {
 			return fmt.Errorf("must define an RPC URL for backend %s", name)
 		}
-		if cfg.WSURL == "" {
+		if wsURL == "" {
 			return fmt.Errorf("must define a WS URL for backend %s", name)
 		}
 
@@ -74,7 +82,11 @@ func Start(config *Config) error {
 			opts = append(opts, WithMaxWSConns(cfg.MaxWSConns))
 		}
 		if cfg.Password != "" {
-			opts = append(opts, WithBasicAuth(cfg.Username, cfg.Password))
+			passwordVal, err := ReadFromEnvOrConfig(cfg.Password)
+			if err != nil {
+				return err
+			}
+			opts = append(opts, WithBasicAuth(cfg.Username, passwordVal))
 		}
 		tlsConfig, err := configureBackendTLS(cfg)
 		if err != nil {
@@ -84,10 +96,10 @@ func Start(config *Config) error {
 			log.Info("using custom TLS config for backend", "name", name)
 			opts = append(opts, WithTLSConfig(tlsConfig))
 		}
-		back := NewBackend(name, cfg.RPCURL, cfg.WSURL, lim, opts...)
+		back := NewBackend(name, rpcURL, wsURL, lim, opts...)
 		backendNames = append(backendNames, name)
 		backendsByName[name] = back
-		log.Info("configured backend", "name", name, "rpc_url", cfg.RPCURL, "ws_url", cfg.WSURL)
+		log.Info("configured backend", "name", name, "rpc_url", rpcURL, "ws_url", wsURL)
 	}
 
 	backendGroups := make(map[string]*BackendGroup)
@@ -124,13 +136,26 @@ func Start(config *Config) error {
 		}
 	}
 
+	var resolvedAuth map[string]string
+
+	if config.Authentication != nil {
+		resolvedAuth = make(map[string]string)
+		for secret, alias := range config.Authentication {
+			resolvedSecret, err := ReadFromEnvOrConfig(secret)
+			if err != nil {
+				return err
+			}
+			resolvedAuth[resolvedSecret] = alias
+		}
+	}
+
 	srv := NewServer(
 		backendGroups,
 		wsBackendGroup,
 		NewStringSetFromStrings(config.WSMethodWhitelist),
 		config.RPCMethodMappings,
 		config.Server.MaxBodySizeBytes,
-		config.Authentication,
+		resolvedAuth,
 	)
 
 	if config.Metrics.Enabled {

go/proxyd/tls.go

@@ -20,7 +20,7 @@ func CreateTLSClient(ca string) (*tls.Config, error) {
 	}
 
 	return &tls.Config{
-		RootCAs:      roots,
+		RootCAs: roots,
 	}, nil
 }
 
@@ -30,4 +30,4 @@ func ParseKeyPair(crt, key string) (tls.Certificate, error) {
 		return tls.Certificate{}, wrapErr(err, "error loading x509 key pair")
 	}
 	return cert, nil
-}
+}
\ No newline at end of file