Commit 96a586e7 authored by Mark Tyneway's avatar Mark Tyneway Committed by GitHub

batch-submitter: updated config (#847)

* batch-submitter: backwards compatible configuration

* chore: add changeset

* deps: update

* js: move bcfg interface to core-utils

* batch-submitter: parse USE_SENTRY and add to env example

* chore: add changeset

* batch-submitter: parse as float instead of int

* batch-submitter: better error logging
parent 20242af4
---
'@eth-optimism/core-utils': patch
'@eth-optimism/data-transport-layer': patch
'@eth-optimism/message-relayer': patch
---
Migrate bcfg interface to core-utils
---
'@eth-optimism/batch-submitter': patch
---
Updates the configuration to use bcfg in a backwards compatible way
......@@ -7,6 +7,7 @@ DEBUG=info*,error*,warn*,debug*
# Leave the SENTRY_DSN variable unset during local development
SENTRY_DSN=
SENTRY_TRACE_RATE=
USE_SENTRY=
L1_NODE_WEB3_URL=http://localhost:9545
L2_NODE_WEB3_URL=http://localhost:8545
......
......@@ -37,6 +37,7 @@
"@eth-optimism/ynatm": "^0.2.2",
"@ethersproject/abstract-provider": "^5.0.5",
"@ethersproject/providers": "^5.0.14",
"bcfg": "^0.1.6",
"bluebird": "^3.7.2",
"dotenv": "^8.2.0",
"ethers": "5.0.0",
......
/* External Imports */
import { injectL2Context } from '@eth-optimism/core-utils'
import { injectL2Context, Bcfg } from '@eth-optimism/core-utils'
import { Logger, Metrics } from '@eth-optimism/common-ts'
import { exit } from 'process'
import { Signer, Wallet } from 'ethers'
import { JsonRpcProvider, TransactionReceipt } from '@ethersproject/providers'
import { config } from 'dotenv'
config()
import * as dotenv from 'dotenv'
import Config from 'bcfg'
/* Internal Imports */
import {
......@@ -16,89 +16,41 @@ import {
TX_BATCH_SUBMITTER_LOG_TAG,
} from '..'
const environment = process.env.NODE_ENV
const network = process.env.ETH_NETWORK_NAME
const release = `batch-submitter@${process.env.npm_package_version}`
/* Logger */
const name = 'oe:batch_submitter:init'
let logger
if (network) {
// Initialize Sentry for Batch Submitter deployed to a network
logger = new Logger({
name,
sentryOptions: {
release,
dsn: process.env.SENTRY_DSN,
tracesSampleRate: parseInt(process.env.SENTRY_TRACE_RATE, 10) || 0.05,
environment: network, // separate our Sentry errors by network instead of node environment
},
})
} else {
// Skip initializing Sentry
logger = new Logger({ name })
}
/* Metrics */
const metrics = new Metrics({
prefix: name,
labels: { environment, release, network },
})
interface RequiredEnvVars {
// The HTTP provider URL for L1.
L1_NODE_WEB3_URL: 'L1_NODE_WEB3_URL'
L1_NODE_WEB3_URL: string
// The HTTP provider URL for L2.
L2_NODE_WEB3_URL: 'L2_NODE_WEB3_URL'
L2_NODE_WEB3_URL: string
// The layer one address manager address
ADDRESS_MANAGER_ADDRESS: 'ADDRESS_MANAGER_ADDRESS'
ADDRESS_MANAGER_ADDRESS: string
// The minimum size in bytes of any L1 transactions generated by the batch submitter.
MIN_L1_TX_SIZE: 'MIN_L1_TX_SIZE'
MIN_L1_TX_SIZE: number
// The maximum size in bytes of any L1 transactions generated by the batch submitter.
MAX_L1_TX_SIZE: 'MAX_L1_TX_SIZE'
MAX_L1_TX_SIZE: number
// The maximum number of L2 transactions that can ever be in a batch.
MAX_TX_BATCH_COUNT: 'MAX_TX_BATCH_COUNT'
MAX_TX_BATCH_COUNT: number
// The maximum number of L2 state roots that can ever be in a batch.
MAX_STATE_BATCH_COUNT: 'MAX_STATE_BATCH_COUNT'
MAX_STATE_BATCH_COUNT: number
// The maximum amount of time (seconds) that we will wait before submitting an under-sized batch.
MAX_BATCH_SUBMISSION_TIME: 'MAX_BATCH_SUBMISSION_TIME'
MAX_BATCH_SUBMISSION_TIME: number
// The delay in milliseconds between querying L2 for more transactions / to create a new batch.
POLL_INTERVAL: 'POLL_INTERVAL'
POLL_INTERVAL: number
// The number of confirmations which we will wait after appending new batches.
NUM_CONFIRMATIONS: 'NUM_CONFIRMATIONS'
NUM_CONFIRMATIONS: number
// The number of seconds to wait before resubmitting a transaction.
RESUBMISSION_TIMEOUT: 'RESUBMISSION_TIMEOUT'
RESUBMISSION_TIMEOUT: number
// The number of confirmations that we should wait before submitting state roots for CTC elements.
FINALITY_CONFIRMATIONS: 'FINALITY_CONFIRMATIONS'
FINALITY_CONFIRMATIONS: number
// Whether or not to run the tx batch submitter.
RUN_TX_BATCH_SUBMITTER: 'true' | 'false' | 'RUN_TX_BATCH_SUBMITTER'
RUN_TX_BATCH_SUBMITTER: boolean
// Whether or not to run the state batch submitter.
RUN_STATE_BATCH_SUBMITTER: 'true' | 'false' | 'RUN_STATE_BATCH_SUBMITTER'
RUN_STATE_BATCH_SUBMITTER: boolean
// The safe minimum amount of ether the batch submitter key should
// hold before it starts to log errors.
SAFE_MINIMUM_ETHER_BALANCE: 'SAFE_MINIMUM_ETHER_BALANCE'
SAFE_MINIMUM_ETHER_BALANCE: number
// A boolean to clear the pending transactions in the mempool
// on start up.
CLEAR_PENDING_TXS: 'true' | 'false' | 'CLEAR_PENDING_TXS'
}
const requiredEnvVars: RequiredEnvVars = {
L1_NODE_WEB3_URL: 'L1_NODE_WEB3_URL',
L2_NODE_WEB3_URL: 'L2_NODE_WEB3_URL',
ADDRESS_MANAGER_ADDRESS: 'ADDRESS_MANAGER_ADDRESS',
MIN_L1_TX_SIZE: 'MIN_L1_TX_SIZE',
MAX_L1_TX_SIZE: 'MAX_L1_TX_SIZE',
MAX_TX_BATCH_COUNT: 'MAX_TX_BATCH_COUNT',
MAX_STATE_BATCH_COUNT: 'MAX_STATE_BATCH_COUNT',
MAX_BATCH_SUBMISSION_TIME: 'MAX_BATCH_SUBMISSION_TIME',
POLL_INTERVAL: 'POLL_INTERVAL',
NUM_CONFIRMATIONS: 'NUM_CONFIRMATIONS',
RESUBMISSION_TIMEOUT: 'RESUBMISSION_TIMEOUT',
FINALITY_CONFIRMATIONS: 'FINALITY_CONFIRMATIONS',
RUN_TX_BATCH_SUBMITTER: 'RUN_TX_BATCH_SUBMITTER',
RUN_STATE_BATCH_SUBMITTER: 'RUN_STATE_BATCH_SUBMITTER',
SAFE_MINIMUM_ETHER_BALANCE: 'SAFE_MINIMUM_ETHER_BALANCE',
CLEAR_PENDING_TXS: 'CLEAR_PENDING_TXS',
CLEAR_PENDING_TXS: boolean
}
/* Optional Env Vars
......@@ -112,50 +64,196 @@ const requiredEnvVars: RequiredEnvVars = {
* SEQUENCER_HD_PATH
* PROPOSER_HD_PATH
*/
const env = process.env
const FRAUD_SUBMISSION_ADDRESS = env.FRAUD_SUBMISSION_ADDRESS || 'no fraud'
const DISABLE_QUEUE_BATCH_APPEND = !!env.DISABLE_QUEUE_BATCH_APPEND
const MIN_GAS_PRICE_IN_GWEI = parseInt(env.MIN_GAS_PRICE_IN_GWEI, 10) || 0
const MAX_GAS_PRICE_IN_GWEI = parseInt(env.MAX_GAS_PRICE_IN_GWEI, 10) || 70
const GAS_RETRY_INCREMENT = parseInt(env.GAS_RETRY_INCREMENT, 10) || 5
const GAS_THRESHOLD_IN_GWEI = parseInt(env.GAS_THRESHOLD_IN_GWEI, 10) || 100
// Private keys & mnemonics
const SEQUENCER_PRIVATE_KEY = env.SEQUENCER_PRIVATE_KEY
const PROPOSER_PRIVATE_KEY =
env.PROPOSER_PRIVATE_KEY || env.SEQUENCER_PRIVATE_KEY // Kept for backwards compatibility
const SEQUENCER_MNEMONIC = env.SEQUENCER_MNEMONIC || env.MNEMONIC
const PROPOSER_MNEMONIC = env.PROPOSER_MNEMONIC || env.MNEMONIC
const SEQUENCER_HD_PATH = env.SEQUENCER_HD_PATH || env.HD_PATH
const PROPOSER_HD_PATH = env.PROPOSER_HD_PATH || env.HD_PATH
// Auto fix batch options -- TODO: Remove this very hacky config
const AUTO_FIX_BATCH_OPTIONS_CONF = env.AUTO_FIX_BATCH_OPTIONS_CONF
const autoFixBatchOptions: AutoFixBatchOptions = {
fixDoublePlayedDeposits: AUTO_FIX_BATCH_OPTIONS_CONF
? AUTO_FIX_BATCH_OPTIONS_CONF.includes('fixDoublePlayedDeposits')
: false,
fixMonotonicity: AUTO_FIX_BATCH_OPTIONS_CONF
? AUTO_FIX_BATCH_OPTIONS_CONF.includes('fixMonotonicity')
: false,
fixSkippedDeposits: AUTO_FIX_BATCH_OPTIONS_CONF
? AUTO_FIX_BATCH_OPTIONS_CONF.includes('fixSkippedDeposits')
: false,
}
export const run = async () => {
dotenv.config()
const config: Bcfg = new Config('batch-submitter')
config.load({
env: true,
argv: true,
})
// Parse config
const env = process.env
const environment = config.str('node-env', env.NODE_ENV)
const network = config.str('eth-network-name', env.ETH_NETWORK_NAME)
const release = `batch-submitter@${env.npm_package_version}`
const sentryDsn = config.str('sentry-dsn', env.SENTRY_DSN)
const sentryTraceRate = config.ufloat(
'sentry-trace-rate',
parseFloat(env.SENTRY_TRACE_RATE) || 0.05
)
/* Logger */
const name = 'oe:batch_submitter:init'
let logger
if (config.bool('use-sentry', env.USE_SENTRY === 'true')) {
// Initialize Sentry for Batch Submitter deployed to a network
logger = new Logger({
name,
sentryOptions: {
release,
dsn: sentryDsn,
tracesSampleRate: sentryTraceRate,
environment: network,
},
})
} else {
// Skip initializing Sentry
logger = new Logger({ name })
}
/* Metrics */
const metrics = new Metrics({
prefix: name,
labels: { environment, release, network },
})
const FRAUD_SUBMISSION_ADDRESS = config.str(
'fraud-submisison-address',
env.FRAUD_SUBMISSION_ADDRESS || 'no fraud'
)
const DISABLE_QUEUE_BATCH_APPEND = config.bool(
'disable-queue-batch-append',
!!env.DISABLE_QUEUE_BATCH_APPEND
)
const MIN_GAS_PRICE_IN_GWEI = config.uint(
'min-gas-price-in-gwei',
parseInt(env.MIN_GAS_PRICE_IN_GWEI, 10) || 0
)
const MAX_GAS_PRICE_IN_GWEI = config.uint(
'max-gas-price-in-gwei',
parseInt(env.MAX_GAS_PRICE_IN_GWEI, 10) || 70
)
const GAS_RETRY_INCREMENT = config.uint(
'gas-retry-increment',
parseInt(env.GAS_RETRY_INCREMENT, 10) || 5
)
const GAS_THRESHOLD_IN_GWEI = config.uint(
'gas-threshold-in-gwei',
parseInt(env.GAS_THRESHOLD_IN_GWEI, 10) || 100
)
// Private keys & mnemonics
const SEQUENCER_PRIVATE_KEY = config.str(
'sequencer-private-key',
env.SEQUENCER_PRIVATE_KEY
)
// Kept for backwards compatibility
const PROPOSER_PRIVATE_KEY = config.str(
'proposer-private-key',
env.PROPOSER_PRIVATE_KEY || env.SEQUENCER_PRIVATE_KEY
)
const SEQUENCER_MNEMONIC = config.str(
'sequencer-mnemonic',
env.SEQUENCER_MNEMONIC || env.MNEMONIC
)
const PROPOSER_MNEMONIC = config.str(
'proposer-mnemonic',
env.PROPOSER_MNEMONIC || env.MNEMONIC
)
const SEQUENCER_HD_PATH = config.str(
'sequencer-hd-path',
env.SEQUENCER_HD_PATH || env.HD_PATH
)
const PROPOSER_HD_PATH = config.str(
'proposer-hd-path',
env.PROPOSER_HD_PATH || env.HD_PATH
)
// Auto fix batch options -- TODO: Remove this very hacky config
const AUTO_FIX_BATCH_OPTIONS_CONF = config.str(
'auto-fix-batch-conf',
env.AUTO_FIX_BATCH_OPTIONS_CONF || ''
)
const autoFixBatchOptions: AutoFixBatchOptions = {
fixDoublePlayedDeposits: AUTO_FIX_BATCH_OPTIONS_CONF
? AUTO_FIX_BATCH_OPTIONS_CONF.includes('fixDoublePlayedDeposits')
: false,
fixMonotonicity: AUTO_FIX_BATCH_OPTIONS_CONF
? AUTO_FIX_BATCH_OPTIONS_CONF.includes('fixMonotonicity')
: false,
fixSkippedDeposits: AUTO_FIX_BATCH_OPTIONS_CONF
? AUTO_FIX_BATCH_OPTIONS_CONF.includes('fixSkippedDeposits')
: false,
}
logger.info('Starting batch submitter...')
for (const [i, val] of Object.entries(requiredEnvVars)) {
if (!process.env[val]) {
const requiredEnvVars: RequiredEnvVars = {
L1_NODE_WEB3_URL: config.str('l1-node-web3-url', env.L1_NODE_WEB3_URL),
L2_NODE_WEB3_URL: config.str('l2-node-web3-url', env.L2_NODE_WEB3_URL),
ADDRESS_MANAGER_ADDRESS: config.str(
'address-manager-address',
env.ADDRESS_MANAGER_ADDRESS
),
MIN_L1_TX_SIZE: config.uint(
'min-l1-tx-size',
parseInt(env.MIN_L1_TX_SIZE, 10)
),
MAX_L1_TX_SIZE: config.uint(
'max-l1-tx-size',
parseInt(env.MAX_L1_TX_SIZE, 10)
),
MAX_TX_BATCH_COUNT: config.uint(
'max-tx-batch-count',
parseInt(env.MAX_TX_BATCH_COUNT, 10)
),
MAX_STATE_BATCH_COUNT: config.uint(
'max-state-batch-count',
parseInt(env.MAX_STATE_BATCH_COUNT, 10)
),
MAX_BATCH_SUBMISSION_TIME: config.uint(
'max-batch-submisison-time',
parseInt(env.MAX_BATCH_SUBMISSION_TIME, 10)
),
POLL_INTERVAL: config.uint(
'poll-interval',
parseInt(env.POLL_INTERVAL, 10)
),
NUM_CONFIRMATIONS: config.uint(
'num-confirmations',
parseInt(env.NUM_CONFIRMATIONS, 10)
),
RESUBMISSION_TIMEOUT: config.uint(
'resubmission-timeout',
parseInt(env.RESUBMISSION_TIMEOUT, 10)
),
FINALITY_CONFIRMATIONS: config.uint(
'finality-confirmations',
parseInt(env.FINALITY_CONFIRMATIONS, 10)
),
RUN_TX_BATCH_SUBMITTER: config.bool(
'run-tx-batch-submitter',
env.RUN_TX_BATCH_SUBMITTER === 'true'
),
RUN_STATE_BATCH_SUBMITTER: config.bool(
'run-state-batch-submitter',
env.RUN_STATE_BATCH_SUBMITTER === 'true'
),
SAFE_MINIMUM_ETHER_BALANCE: config.ufloat(
'safe-minimum-ether-balance',
parseFloat(env.SAFE_MINIMUM_ETHER_BALANCE)
),
CLEAR_PENDING_TXS: config.bool(
'clear-pending-txs',
env.CLEAR_PENDING_TXS === 'true'
),
}
for (const [key, val] of Object.entries(requiredEnvVars)) {
if (val === null || val === undefined) {
logger.warn('Missing environment variable', {
varName: val,
key,
value: val,
})
exit(1)
}
requiredEnvVars[val] = process.env[val]
}
const clearPendingTxs = requiredEnvVars.CLEAR_PENDING_TXS === 'true'
const clearPendingTxs = requiredEnvVars.CLEAR_PENDING_TXS
const l1Provider = new JsonRpcProvider(requiredEnvVars.L1_NODE_WEB3_URL)
const l2Provider = injectL2Context(
......@@ -206,14 +304,14 @@ export const run = async () => {
const txBatchSubmitter = new TransactionBatchSubmitter(
sequencerSigner,
l2Provider,
parseInt(requiredEnvVars.MIN_L1_TX_SIZE, 10),
parseInt(requiredEnvVars.MAX_L1_TX_SIZE, 10),
parseInt(requiredEnvVars.MAX_TX_BATCH_COUNT, 10),
parseInt(requiredEnvVars.MAX_BATCH_SUBMISSION_TIME, 10) * 1_000,
parseInt(requiredEnvVars.NUM_CONFIRMATIONS, 10),
parseInt(requiredEnvVars.RESUBMISSION_TIMEOUT, 10) * 1_000,
requiredEnvVars.MIN_L1_TX_SIZE,
requiredEnvVars.MAX_L1_TX_SIZE,
requiredEnvVars.MAX_TX_BATCH_COUNT,
requiredEnvVars.MAX_BATCH_SUBMISSION_TIME * 1_000,
requiredEnvVars.NUM_CONFIRMATIONS,
requiredEnvVars.RESUBMISSION_TIMEOUT * 1_000,
requiredEnvVars.ADDRESS_MANAGER_ADDRESS,
parseFloat(requiredEnvVars.SAFE_MINIMUM_ETHER_BALANCE),
requiredEnvVars.SAFE_MINIMUM_ETHER_BALANCE,
MIN_GAS_PRICE_IN_GWEI,
MAX_GAS_PRICE_IN_GWEI,
GAS_RETRY_INCREMENT,
......@@ -230,15 +328,15 @@ export const run = async () => {
const stateBatchSubmitter = new StateBatchSubmitter(
proposerSigner,
l2Provider,
parseInt(requiredEnvVars.MIN_L1_TX_SIZE, 10),
parseInt(requiredEnvVars.MAX_L1_TX_SIZE, 10),
parseInt(requiredEnvVars.MAX_STATE_BATCH_COUNT, 10),
parseInt(requiredEnvVars.MAX_BATCH_SUBMISSION_TIME, 10) * 1_000,
parseInt(requiredEnvVars.NUM_CONFIRMATIONS, 10),
parseInt(requiredEnvVars.RESUBMISSION_TIMEOUT, 10) * 1_000,
parseInt(requiredEnvVars.FINALITY_CONFIRMATIONS, 10),
requiredEnvVars.MIN_L1_TX_SIZE,
requiredEnvVars.MAX_L1_TX_SIZE,
requiredEnvVars.MAX_STATE_BATCH_COUNT,
requiredEnvVars.MAX_BATCH_SUBMISSION_TIME * 1_000,
requiredEnvVars.NUM_CONFIRMATIONS,
requiredEnvVars.RESUBMISSION_TIMEOUT * 1_000,
requiredEnvVars.FINALITY_CONFIRMATIONS,
requiredEnvVars.ADDRESS_MANAGER_ADDRESS,
parseFloat(requiredEnvVars.SAFE_MINIMUM_ETHER_BALANCE),
requiredEnvVars.SAFE_MINIMUM_ETHER_BALANCE,
MIN_GAS_PRICE_IN_GWEI,
MAX_GAS_PRICE_IN_GWEI,
GAS_RETRY_INCREMENT,
......@@ -281,7 +379,7 @@ export const run = async () => {
})
await sequencerSigner.provider.waitForTransaction(
response.hash,
parseInt(requiredEnvVars.NUM_CONFIRMATIONS, 10)
requiredEnvVars.NUM_CONFIRMATIONS
)
}
}
......@@ -307,17 +405,15 @@ export const run = async () => {
logger.info('Retrying...')
}
// Sleep
await new Promise((r) =>
setTimeout(r, parseInt(requiredEnvVars.POLL_INTERVAL, 10))
)
await new Promise((r) => setTimeout(r, requiredEnvVars.POLL_INTERVAL))
}
}
// Run batch submitters in two seperate infinite loops!
if (requiredEnvVars.RUN_TX_BATCH_SUBMITTER === 'true') {
if (requiredEnvVars.RUN_TX_BATCH_SUBMITTER) {
loop(() => txBatchSubmitter.submitNextBatch())
}
if (requiredEnvVars.RUN_STATE_BATCH_SUBMITTER === 'true') {
if (requiredEnvVars.RUN_STATE_BATCH_SUBMITTER) {
loop(() => stateBatchSubmitter.submitNextBatch())
}
}
export interface Bcfg {
load: (options: { env?: boolean; argv?: boolean }) => void
str: (name: string, defaultValue?: string) => string
uint: (name: string, defaultValue?: number) => number
bool: (name: string, defaultValue?: boolean) => boolean
ufloat: (name: string, defaultValue?: number) => number
has: (name: string) => boolean
}
......@@ -4,3 +4,4 @@ export * from './watcher'
export * from './l2context'
export * from './events'
export * from './batches'
export * from './bcfg'
/* Imports: External */
import * as dotenv from 'dotenv'
import Config from 'bcfg' // TODO: Add some types for bcfg if we get the chance.
import { Bcfg } from '@eth-optimism/core-utils'
import Config from 'bcfg'
/* Imports: Internal */
import { L1DataTransportService } from './main/service'
interface Bcfg {
load: (options: { env?: boolean; argv?: boolean }) => void
str: (name: string, defaultValue?: string) => string
uint: (name: string, defaultValue?: number) => number
bool: (name: string, defaultValue?: boolean) => boolean
ufloat: (name: string, defaultValue?: number) => number
}
type ethNetwork = 'mainnet' | 'kovan' | 'goerli'
;(async () => {
try {
......
import { Wallet, providers } from 'ethers'
import { MessageRelayerService } from '../service'
import { Bcfg } from '@eth-optimism/core-utils'
import SpreadSheet from '../spreadsheet'
import * as dotenv from 'dotenv'
import Config from 'bcfg'
interface Bcfg {
load: (options: { env?: boolean; argv?: boolean }) => void
str: (name: string, defaultValue?: string) => string
uint: (name: string, defaultValue?: number) => number
bool: (name: string, defaultValue?: boolean) => boolean
ufloat: (name: string, defaultValue?: number) => number
}
dotenv.config()
const main = async () => {
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment