Merge pull request #7408 from ethereum-optimism/wyatt/ufm/bug-fixes-failure-tracking

UFM bug fixes, Metamask dashboard refactor, improved failures

ufm-test-services/grafana/dashboards/metamask.json

@@ -18,6 +18,7 @@
   "editable": true,
   "fiscalYearStartMonth": 0,
   "graphTooltip": 0,
+  "id": 1,
   "links": [],
   "liveNow": false,
   "panels": [
@@ -31,88 +32,57 @@
           "color": {
             "mode": "thresholds"
           },
+          "custom": {
+            "axisCenteredZero": true,
+            "axisColorMode": "series",
+            "axisGridShow": false,
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 0,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "insertNulls": false,
+            "lineInterpolation": "stepAfter",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "never",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "decimals": 0,
+          "displayName": "Number of Transactions (positive number = success, negative = failures)",
           "mappings": [],
           "thresholds": {
             "mode": "absolute",
             "steps": [
               {
-                "color": "red",
+                "color": "text",
                 "value": null
               },
               {
-                "color": "yellow",
-                "value": 1
+                "color": "red",
+                "value": -1
               },
-              {
-                "color": "green",
-                "value": 4
-              }
-            ]
-          }
-        },
-        "overrides": []
-      },
-      "gridPos": {
-        "h": 5,
-        "w": 6,
-        "x": 0,
-        "y": 0
-      },
-      "id": 1,
-      "options": {
-        "orientation": "auto",
-        "reduceOptions": {
-          "calcs": [
-            "lastNotNull"
-          ],
-          "fields": "",
-          "values": false
-        },
-        "showThresholdLabels": false,
-        "showThresholdMarkers": true
-      },
-      "pluginVersion": "10.1.2",
-      "targets": [
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "PBFA97CFB590B2093"
-          },
-          "disableTextWrap": false,
-          "editorMode": "builder",
-          "expr": "metamask_tx_success",
-          "fullMetaSearch": false,
-          "includeNullMetadata": true,
-          "instant": false,
-          "legendFormat": "__auto",
-          "range": true,
-          "refId": "A",
-          "useBackend": false
-        }
-      ],
-      "title": "Successful Transaction Since Last Failure",
-      "type": "gauge"
-    },
-    {
-      "datasource": {
-        "type": "prometheus",
-        "uid": "PBFA97CFB590B2093"
-      },
-      "fieldConfig": {
-        "defaults": {
-          "color": {
-            "mode": "thresholds"
-          },
-          "mappings": [],
-          "thresholds": {
-            "mode": "absolute",
-            "steps": [
               {
                 "color": "text",
-                "value": null
+                "value": 0
               },
               {
-                "color": "red",
+                "color": "green",
                 "value": 1
               }
             ]
@@ -121,25 +91,29 @@
         "overrides": []
       },
       "gridPos": {
-        "h": 5,
-        "w": 6,
-        "x": 6,
+        "h": 8,
+        "w": 24,
+        "x": 0,
         "y": 0
       },
-      "id": 2,
+      "id": 1,
       "options": {
-        "orientation": "auto",
-        "reduceOptions": {
+        "legend": {
           "calcs": [
-            "lastNotNull"
+            "last"
           ],
-          "fields": "",
-          "values": false
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": true
         },
-        "showThresholdLabels": false,
-        "showThresholdMarkers": true
+        "timezone": [
+          "browser"
+        ],
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
       },
-      "pluginVersion": "10.1.2",
       "targets": [
         {
           "datasource": {
@@ -148,7 +122,7 @@
           },
           "disableTextWrap": false,
           "editorMode": "builder",
-          "expr": "metamask_tx_failure",
+          "expr": "metamask_self_send",
           "fullMetaSearch": false,
           "includeNullMetadata": true,
           "instant": false,
@@ -158,8 +132,8 @@
           "useBackend": false
         }
       ],
-      "title": "Failed Transactions Since Last Success",
-      "type": "gauge"
+      "title": "Self Transferring on OP Goerli (positive number = success, negative = failures)",
+      "type": "timeseries"
     }
   ],
   "refresh": "5s",
@@ -170,7 +144,7 @@
     "list": []
   },
   "time": {
-    "from": "now-6h",
+    "from": "now-30m",
     "to": "now"
   },
   "timepicker": {},
@@ -179,4 +153,4 @@
   "uid": "f66f7076-c724-4f81-8ff9-58d6d99f2716",
   "version": 1,
   "weekStart": ""
-}
+}
\ No newline at end of file

ufm-test-services/metamask/playwright.config.ts

@@ -15,8 +15,6 @@ export default defineConfig({
   fullyParallel: true,
   /* Fail the build on CI if you accidentally left test.only in the source code. */
   forbidOnly: !!process.env.CI,
-  /* Retry on CI only */
-  retries: process.env.CI ? 2 : 0,
   /* Opt out of parallel tests on CI. */
   workers: process.env.CI ? 1 : undefined,
   /* Reporter to use. See https://playwright.dev/docs/test-reporters */

ufm-test-services/metamask/tests/metamask.spec.ts

@@ -6,8 +6,7 @@ import { mnemonicToAccount, privateKeyToAccount } from 'viem/accounts'
 
 import { testWithSynpress } from './testWithSynpressUtil'
 import {
-  incrementMetamaskTxCounter,
-  setMetamaskTxCounter,
+  incrementSelfSendTxGauge,
 } from './prometheusUtils'
 
 const env = z.object({
@@ -24,18 +23,31 @@ const expectedSender =
     : mnemonicToAccount(
         env.METAMASK_SECRET_WORDS_OR_PRIVATEKEY as string
       ).address.toLowerCase()
-const expectedRecipient = '0x8fcfbe8953433fd1f2e8375ee99057833e4e1e9e'
+const expectedRecipient = expectedSender
 
 let sharedPage: Page
+let wasSuccessful: boolean
+let handledFailure: boolean
 
 test.describe.configure({ mode: 'serial' })
 
+test.beforeAll(() => {
+  wasSuccessful = false
+  handledFailure = false
+})
+
 test.afterAll(async () => {
+  // This is handling failure scenarios such as Playwright timeouts
+  // where are not able to catch and respond to an error.
+  if (!wasSuccessful && !handledFailure) {
+    await incrementSelfSendTxGauge(false)
+  }
+
   await sharedPage.close()
 })
 
 testWithSynpress('Setup wallet and dApp', async ({ page }) => {
-  console.log('Seting up wallet and dApp...')
+  console.log('Setting up wallet and dApp...')
   sharedPage = page
   await sharedPage.goto('http://localhost:9011')
   console.log('Setup wallet and dApp')
@@ -66,8 +78,8 @@ testWithSynpress('Add OP Goerli network', async () => {
   try {
     await expect(sharedPage.locator('#chainId')).toHaveText(expectedChainId)
   } catch (error) {
-    await setMetamaskTxCounter(true, 0)
-    await incrementMetamaskTxCounter(false)
+    await incrementSelfSendTxGauge(false)
+    handledFailure = true
     throw error
   }
   console.log('Added OP Goerli network')
@@ -81,15 +93,15 @@ test(`Connect wallet with ${expectedSender}`, async () => {
   try {
     await expect(sharedPage.locator('#accounts')).toHaveText(expectedSender)
   } catch (error) {
-    await setMetamaskTxCounter(true, 0)
-    await incrementMetamaskTxCounter(false)
+    await incrementSelfSendTxGauge(false)
+    handledFailure = true
     throw error
   }
   console.log(`Connected wallet with ${expectedSender}`)
 })
 
-test('Send an EIP-1559 transaciton and verfiy success', async () => {
-  console.log('Sending an EIP-1559 transaciton and verfiy success...')
+test('Send an EIP-1559 transaction and verify success', async () => {
+  console.log('Sending an EIP-1559 transaction and verify success...')
   const expectedTransferAmount = '0x1'
   const expectedTxType = '0x2'
 
@@ -120,7 +132,7 @@ test('Send an EIP-1559 transaciton and verfiy success', async () => {
   // Waiting for RPC response to be populated on the page
   await sharedPage.waitForTimeout(2_000)
 
-  const transaction = JSON.parse(
+  const transactionReceipt = JSON.parse(
     (await sharedPage.locator('body > main').innerText()).replace(
       'Response: ',
       ''
@@ -128,13 +140,13 @@ test('Send an EIP-1559 transaciton and verfiy success', async () => {
   )
 
   try {
-    expect(transaction.status).toBe('0x1')
-    await setMetamaskTxCounter(false, 0)
-    await incrementMetamaskTxCounter(true)
+    expect(transactionReceipt.status).toBe('0x1')
+    wasSuccessful = true
+    await incrementSelfSendTxGauge(true)
   } catch (error) {
-    await setMetamaskTxCounter(true, 0)
-    await incrementMetamaskTxCounter(false)
+    await incrementSelfSendTxGauge(false)
+    handledFailure = true
     throw error
   }
-  console.log('Sent an EIP-1559 transaciton and verfied success')
+  console.log('Sent an EIP-1559 transaction and verified success')
 })

ufm-test-services/metamask/tests/prometheusUtils.ts

 import 'dotenv/config'
 import { z } from 'zod'
-import { Counter, Pushgateway } from 'prom-client'
+import { Gauge, Pushgateway } from 'prom-client'
 
 const env = z
   .object({
@@ -9,21 +9,15 @@ const env = z
   })
   .parse(process.env)
 
-const txSuccessMetricName = 'metamask_tx_success'
-const txFailureMetricName = 'metamask_tx_failure'
+const selfSendTransactionMetricName = 'metamask_self_send'
 
-const txSuccessCounter = new Counter({
-  name: txSuccessMetricName,
-  help: 'A counter signifying the number of successful transactions sent with Metamask since last failure',
-})
-const txFailureCounter = new Counter({
-  name: txFailureMetricName,
-  help: 'A counter signifying the number of failed transactions sent with Metamask since last successful transaction',
+const selfSendGauge = new Gauge({
+  name: selfSendTransactionMetricName,
+  help: 'A gauge signifying the number of transactions sent with Metamask',
 })
 
-export const getMetamaskTxCounterValue = async (isSuccess: boolean) => {
-  const metricName = isSuccess ? txSuccessMetricName : txFailureMetricName
-  const prometheusMetricQuery = `${env.PROMETHEUS_SERVER_URL}/api/v1/query?query=${metricName}`
+export const getSelfSendGaugeValue = async () => {
+  const prometheusMetricQuery = `${env.PROMETHEUS_SERVER_URL}/api/v1/query?query=${selfSendTransactionMetricName}`
 
   const response = await fetch(prometheusMetricQuery)
   if (!response.ok) {
@@ -37,12 +31,12 @@ export const getMetamaskTxCounterValue = async (isSuccess: boolean) => {
   // [
   //   {
   //     metric: {
-  //       __name__: 'metamask_tx_success',
-  //       exported_job: 'metamask_tx_count',
+  //       __name__: 'metamask_self_send',
+  //       exported_job: 'metamask_self_send_tx_count',
   //       instance: 'pushgateway:9091',
   //       job: 'pushgateway'
   //     },
-  //     value: [ 1695250414.474, '0' ]
+  //     value: [ 1695847795.646, '-1' ]
   //   }
   // ]
   try {
@@ -66,7 +60,9 @@ export const getMetamaskTxCounterValue = async (isSuccess: boolean) => {
     if (
       error.message === "Cannot read properties of undefined (reading 'value')"
     ) {
-      console.warn(`No data found for metric ${metricName} in Prometheus`)
+      console.warn(
+        `No data found for metric ${selfSendTransactionMetricName} in Prometheus`
+      )
       return undefined
     }
 
@@ -74,28 +70,26 @@ export const getMetamaskTxCounterValue = async (isSuccess: boolean) => {
   }
 }
 
-export const setMetamaskTxCounter = async (
-  isSuccess: boolean,
-  valueToSetTo: number
-) => {
-  const metricName = isSuccess ? txSuccessMetricName : txFailureMetricName
-  const txCounter = isSuccess ? txSuccessCounter : txFailureCounter
-
-  txCounter.reset()
-  console.log(`Setting ${metricName} to ${valueToSetTo}`)
-  txCounter.inc(valueToSetTo)
+export const setSelfSendTxGauge = async (valueToSetTo: number) => {
+  console.log(`Setting ${selfSendTransactionMetricName} to ${valueToSetTo}`)
+  selfSendGauge.set(valueToSetTo)
 
   const pushGateway = new Pushgateway(env.PROMETHEUS_PUSHGATEWAY_URL)
-  await pushGateway.pushAdd({ jobName: 'metamask_tx_count' })
+  await pushGateway.pushAdd({ jobName: 'metamask_self_send_tx_count' })
 }
 
-export const incrementMetamaskTxCounter = async (isSuccess: boolean) => {
-  const metricName = isSuccess ? txSuccessMetricName : txFailureMetricName
-  const currentMetricValue = (await getMetamaskTxCounterValue(true)) ?? 0
+export const incrementSelfSendTxGauge = async (isSuccess: boolean) => {
+  const currentMetricValue = (await getSelfSendGaugeValue()) ?? 0
+
+  let newMetricValue: number
+  if (isSuccess) {
+    newMetricValue = currentMetricValue >= 0 ? currentMetricValue + 1 : 1
+  } else {
+    newMetricValue = currentMetricValue < 0 ? currentMetricValue - 1 : -1
+  }
+
   console.log(
-    `Current value of ${metricName} is ${currentMetricValue}, incrementing to ${
-      currentMetricValue + 1
-    }`
+    `Current value of ${selfSendTransactionMetricName} is ${currentMetricValue}, incrementing to ${newMetricValue}`
   )
-  await setMetamaskTxCounter(isSuccess, currentMetricValue + 1)
+  await setSelfSendTxGauge(newMetricValue)
 }