op-node: separate sequencing and publishing errors. Old payloads fail to...

op-node: separate sequencing and publishing errors. Old payloads fail to publish, but should not hold back catch-up work of sequencer (#3186)

op-node/metrics/metrics.go

@@ -42,6 +42,8 @@ type Metrics struct {
 	LastPipelineResetUnix prometheus.Gauge
 	UnsafePayloadsTotal   prometheus.Counter
 	DerivationErrorsTotal prometheus.Counter
+	SequencingErrorsTotal prometheus.Counter
+	PublishingErrorsTotal prometheus.Counter
 	Heads                 *prometheus.GaugeVec
 
 	TransactionsSequencedTotal prometheus.Counter
@@ -141,6 +143,16 @@ func NewMetrics(procName string) *Metrics {
 			Name:      "derivation_errors_total",
 			Help:      "Count of total derivation errors",
 		}),
+		SequencingErrorsTotal: promauto.With(registry).NewCounter(prometheus.CounterOpts{
+			Namespace: ns,
+			Name:      "sequencing_errors_total",
+			Help:      "Count of total sequencing errors",
+		}),
+		PublishingErrorsTotal: promauto.With(registry).NewCounter(prometheus.CounterOpts{
+			Namespace: ns,
+			Name:      "publishing_errors_total",
+			Help:      "Count of total p2p publishing errors",
+		}),
 		Heads: promauto.With(registry).NewGaugeVec(prometheus.GaugeOpts{
 			Namespace: ns,
 			Name:      "heads",
@@ -232,7 +244,6 @@ func (m *Metrics) SetHead(kind string, num uint64) {
 
 func (m *Metrics) RecordPipelineReset() {
 	m.PipelineResetsTotal.Inc()
-	m.DerivationErrorsTotal.Inc()
 	m.LastPipelineResetUnix.Set(float64(time.Now().Unix()))
 }
 

op-node/rollup/driver/state.go

@@ -252,7 +252,8 @@ func (s *state) createNewL2Block(ctx context.Context) error {
 	if s.network != nil {
 		if err := s.network.PublishL2Payload(ctx, payload); err != nil {
 			s.log.Warn("failed to publish newly created block", "id", payload.ID(), "err", err)
-			return err
+			s.metrics.PublishingErrorsTotal.Inc()
+			// publishing of unsafe data via p2p is optional. Errors are not severe enough to change/halt sequencing but should be logged and metered.
 		}
 	}
 
@@ -348,7 +349,7 @@ func (s *state) eventLoop() {
 			cancel()
 			if err != nil {
 				s.log.Error("Error creating new L2 block", "err", err)
-				s.metrics.DerivationErrorsTotal.Inc()
+				s.metrics.SequencingErrorsTotal.Inc()
 				break // if we fail, we wait for the next block creation trigger.
 			}