Merge pull request #7320 from ethereum-optimism/wyatt/ufm/init-mm-metrics

Init Metamask Pass/Fail metrics

ufm-test-services/docker-compose.yml

@@ -35,7 +35,8 @@ services:
     environment:
       - GF_SECURITY_ADMIN_PASSWORD=${GRAFANA_ADMIN_PWD}
     volumes:
-      - ./datasources.yml:/etc/grafana/provisioning/datasources/datasources.yaml
+      - ./grafana/provisioning:/etc/grafana/provisioning
+      - ./grafana/dashboards:/var/lib/grafana/dashboards
     security_opt:
       - "no-new-privileges:true"
 

ufm-test-services/grafana/dashboards/metamask.json

+{
+  "annotations": {
+    "list": [
+      {
+        "builtIn": 1,
+        "datasource": {
+          "type": "grafana",
+          "uid": "-- Grafana --"
+        },
+        "enable": true,
+        "hide": true,
+        "iconColor": "rgba(0, 211, 255, 1)",
+        "name": "Annotations & Alerts",
+        "type": "dashboard"
+      }
+    ]
+  },
+  "editable": true,
+  "fiscalYearStartMonth": 0,
+  "graphTooltip": 0,
+  "links": [],
+  "liveNow": false,
+  "panels": [
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "PBFA97CFB590B2093"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "thresholds"
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "red",
+                "value": null
+              },
+              {
+                "color": "yellow",
+                "value": 1
+              },
+              {
+                "color": "green",
+                "value": 4
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 5,
+        "w": 6,
+        "x": 0,
+        "y": 0
+      },
+      "id": 1,
+      "options": {
+        "orientation": "auto",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "showThresholdLabels": false,
+        "showThresholdMarkers": true
+      },
+      "pluginVersion": "10.1.2",
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "PBFA97CFB590B2093"
+          },
+          "disableTextWrap": false,
+          "editorMode": "builder",
+          "expr": "metamask_tx_success",
+          "fullMetaSearch": false,
+          "includeNullMetadata": true,
+          "instant": false,
+          "legendFormat": "__auto",
+          "range": true,
+          "refId": "A",
+          "useBackend": false
+        }
+      ],
+      "title": "Number of Successful Transaction Since Last Failure",
+      "type": "gauge"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "PBFA97CFB590B2093"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "thresholds"
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "text",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 1
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 5,
+        "w": 6,
+        "x": 6,
+        "y": 0
+      },
+      "id": 2,
+      "options": {
+        "orientation": "auto",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "showThresholdLabels": false,
+        "showThresholdMarkers": true
+      },
+      "pluginVersion": "10.1.2",
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "PBFA97CFB590B2093"
+          },
+          "disableTextWrap": false,
+          "editorMode": "builder",
+          "expr": "metamask_tx_failure",
+          "fullMetaSearch": false,
+          "includeNullMetadata": true,
+          "instant": false,
+          "legendFormat": "__auto",
+          "range": true,
+          "refId": "A",
+          "useBackend": false
+        }
+      ],
+      "title": "Number of Failed Transactions Since Last Success",
+      "type": "gauge"
+    }
+  ],
+  "refresh": "5s",
+  "schemaVersion": 38,
+  "style": "dark",
+  "tags": [],
+  "templating": {
+    "list": []
+  },
+  "time": {
+    "from": "now-6h",
+    "to": "now"
+  },
+  "timepicker": {},
+  "timezone": "",
+  "title": "UFM: Metamask",
+  "uid": "f66f7076-c724-4f81-8ff9-58d6d99f2716",
+  "version": 1,
+  "weekStart": ""
+}

ufm-test-services/grafana/provisioning/dashboards/all.yml

+apiVersion: 1
+providers:
+- name: 'default'
+  orgId: 1
+  folder: ''
+  type: file
+  disableDeletion: false
+  options:
+    path: /var/lib/grafana/dashboards

ufm-test-services/metamask/README.md

+# User Facing Monitoring - Metamask Tests
+
+## Running Locally
+
+### Building Docker Image
+
+```bash
+docker build -t ufm-test-service-metamask .
+```
+
+### Running the Docker Container on MacOS
+
+The following steps were taken from [here](https://www.oddbird.net/2022/11/30/headed-playwright-in-docker/#macos)
+
+Apple’s operating system doesn’t include a built-in XServer, but we can use [XQuartz](https://www.xquartz.org/) to provide one:
+
+1. Install XQuartz: `brew install --cask xquartz``
+2. Open XQuartz, go to `Preferences -> Security`, and check `Allow connections from network clients`
+3. Restart your computer (restarting XQuartz might not be enough)
+4. Start XQuartz by executing `xhost +localhost` in your terminal
+5. Open Docker Desktop and edit settings to give access to `/tmp/.X11-unix` in `Preferences -> Resources -> File sharing`
+
+Once XQuartz is running with the right permissions, you can populate the environment variable and socket Docker args:
+
+```bash
+docker run --rm -it \
+-e DISPLAY=host.docker.internal:0 \
+-v /tmp/.X11-unix:/tmp/.X11-unix \
+ufm-test-service-metamask
+```

ufm-test-services/metamask/package.json

@@ -20,10 +20,14 @@
   "devDependencies": {
     "@metamask/test-dapp": "^7.1.0",
     "@playwright/test": "1.37.1",
-    "@synthetixio/synpress": "3.7.2-beta.5",
+    "@synthetixio/synpress": "3.7.2-beta.7",
     "dotenv": "^16.3.1",
     "static-server": "^2.2.1",
     "typescript": "^5.1.6",
     "viem": "^1.10.8"
+  },
+  "dependencies": {
+    "prom-client": "^14.2.0",
+    "zod": "^3.22.2"
   }
 }

ufm-test-services/metamask/tests/metamask.spec.ts

 import 'dotenv/config'
+import { z } from 'zod'
 import metamask from '@synthetixio/synpress/commands/metamask.js'
 import { expect, test, type Page } from '@playwright/test'
 import { mnemonicToAccount, privateKeyToAccount } from 'viem/accounts'
 
 import { testWithSynpress } from './testWithSynpressUtil'
+import {
+  getMetamaskTxCounterValue,
+  incrementMetamaskTxCounter,
+  setMetamaskTxCounter,
+} from './prometheusUtils'
+
+const env = z.object({
+  METAMASK_SECRET_WORDS_OR_PRIVATEKEY: z.string(),
+  OP_GOERLI_RPC_URL: z.string().url(),
+  METAMASK_DAPP_URL: z.string().url()
+}).parse(process.env)
 
 const expectedSender =
-  process.env.METAMASK_SECRET_WORDS_OR_PRIVATEKEY?.startsWith('0x')
+  env.METAMASK_SECRET_WORDS_OR_PRIVATEKEY?.startsWith('0x')
     ? privateKeyToAccount(
-        process.env.METAMASK_SECRET_WORDS_OR_PRIVATEKEY as `0x${string}`
+        env.METAMASK_SECRET_WORDS_OR_PRIVATEKEY as `0x${string}`
       ).address.toLowerCase()
     : mnemonicToAccount(
-        process.env.METAMASK_SECRET_WORDS_OR_PRIVATEKEY as string
+        env.METAMASK_SECRET_WORDS_OR_PRIVATEKEY as string
       ).address.toLowerCase()
 const expectedRecipient = '0x8fcfbe8953433fd1f2e8375ee99057833e4e1e9e'
 
@@ -35,7 +47,7 @@ testWithSynpress('Add OP Goerli network', async () => {
     name: 'op-goerli',
     rpcUrls: {
       default: {
-        http: [process.env.OP_GOERLI_RPC_URL],
+        http: [env.OP_GOERLI_RPC_URL],
       },
     },
     id: '420',
@@ -49,13 +61,26 @@ testWithSynpress('Add OP Goerli network', async () => {
     },
   })
 
+  try {
     await expect(sharedPage.locator('#chainId')).toHaveText(expectedChainId)
+  } catch (error) {
+    await setMetamaskTxCounter(true, 0)
+    await incrementMetamaskTxCounter(false)
+    throw error
+  }
 })
 
 test(`Connect wallet with ${expectedSender}`, async () => {
   await sharedPage.click('#connectButton')
   await metamask.acceptAccess()
+
+  try {
     await expect(sharedPage.locator('#accounts')).toHaveText(expectedSender)
+  } catch (error) {
+    await setMetamaskTxCounter(true, 0)
+    await incrementMetamaskTxCounter(false)
+    throw error
+  }
 })
 
 test('Send an EIP-1559 transaciton and verfiy success', async () => {
@@ -76,17 +101,14 @@ test('Send an EIP-1559 transaciton and verfiy success', async () => {
     })
   })
 
-  await metamask.confirmTransaction()
+  await metamask.confirmTransactionAndWaitForMining()
   const txHash = await txHashPromise
 
-    // Waiting for Infura (Metamask given provider) to index our transaction
-    await sharedPage.waitForTimeout(10_000)
-
   // Metamask test dApp allows us access to the Metamask RPC provider via loading this URL.
   // The RPC reponse will be populated onto the page that's loaded.
   // More info here: https://github.com/MetaMask/test-dapp/tree/main#usage
   await sharedPage.goto(
-    `${process.env.METAMASK_DAPP_URL}/request.html?method=eth_getTransactionReceipt&params=["${txHash}"]`
+    `${env.METAMASK_DAPP_URL}/request.html?method=eth_getTransactionReceipt&params=["${txHash}"]`
   )
 
   // Waiting for RPC response to be populated on the page
@@ -98,5 +120,14 @@ test('Send an EIP-1559 transaciton and verfiy success', async () => {
       ''
     )
   )
+
+  try {
     expect(transaction.status).toBe('0x1')
+    await setMetamaskTxCounter(false, 0)
+    await incrementMetamaskTxCounter(true)
+  } catch (error) {
+    await setMetamaskTxCounter(true, 0)
+    await incrementMetamaskTxCounter(false)
+    throw error
+  }
 })

ufm-test-services/metamask/tests/prometheusUtils.ts

+import 'dotenv/config'
+import { z } from 'zod'
+import { Counter, Pushgateway } from 'prom-client'
+
+const env = z
+  .object({
+    PROMETHEUS_SERVER_URL: z.string().url(),
+    PROMETHEUS_PUSHGATEWAY_URL: z.string().url(),
+  })
+  .parse(process.env)
+
+const txSuccessMetricName = 'metamask_tx_success'
+const txFailureMetricName = 'metamask_tx_failuree'
+
+const txSuccessCounter = new Counter({
+  name: txSuccessMetricName,
+  help: 'A counter signifying the number of successful transactions sent with Metamask since last failure',
+})
+const txFailureCounter = new Counter({
+  name: txFailureMetricName,
+  help: 'A counter signifying the number of failed transactions sent with Metamask since last successful transaction',
+})
+
+export const getMetamaskTxCounterValue = async (isSuccess: boolean) => {
+  const metricName = isSuccess ? txSuccessMetricName : txFailureMetricName
+  const prometheusMetricQuery = `${env.PROMETHEUS_SERVER_URL}/api/v1/query?query=${metricName}`
+
+  const response = await fetch(prometheusMetricQuery)
+  if (!response.ok) {
+    console.error(response.status)
+    console.error(response.statusText)
+    throw new Error(`Failed to fetch metric from: ${prometheusMetricQuery}`)
+  }
+
+  // The following is an example of the expect response from prometheusMetricQuery
+  // for response.json().data.result[0]:
+  // [
+  //   {
+  //     metric: {
+  //       __name__: 'metamask_tx_success',
+  //       exported_job: 'metamask_tx_count',
+  //       instance: 'pushgateway:9091',
+  //       job: 'pushgateway'
+  //     },
+  //     value: [ 1695250414.474, '0' ]
+  //   }
+  // ]
+  try {
+    const responseJson = z
+      .object({
+        data: z.object({
+          result: z.array(
+            z.object({
+              value: z.tuple([
+                z.number(),
+                z.number().or(z.string().transform((value) => parseInt(value))),
+              ]),
+            })
+          ),
+        }),
+      })
+      .parse(await response.json())
+
+    return responseJson.data.result[0].value[1]
+  } catch (error) {
+    if (
+      error.message === "Cannot read properties of undefined (reading 'value')"
+    ) {
+      console.warn(`No data found for metric ${metricName} in Prometheus`)
+      return undefined
+    }
+
+    throw error
+  }
+}
+
+export const setMetamaskTxCounter = async (
+  isSuccess: boolean,
+  valueToSetTo: number
+) => {
+  const metricName = isSuccess ? txSuccessMetricName : txFailureMetricName
+  const txCounter = isSuccess ? txSuccessCounter : txFailureCounter
+
+  txCounter.reset()
+  console.log(`Setting ${metricName} to ${valueToSetTo}`)
+  txCounter.inc(valueToSetTo)
+
+  const pushGateway = new Pushgateway(env.PROMETHEUS_PUSHGATEWAY_URL)
+  await pushGateway.pushAdd({ jobName: 'metamask_tx_count' })
+}
+
+export const incrementMetamaskTxCounter = async (isSuccess: boolean) => {
+  const metricName = isSuccess ? txSuccessMetricName : txFailureMetricName
+  const currentMetricValue = (await getMetamaskTxCounterValue(true)) ?? 0
+  console.log(
+    `Current value of ${metricName} is ${currentMetricValue}, incrementing to ${
+      currentMetricValue + 1
+    }`
+  )
+  await setMetamaskTxCounter(isSuccess, currentMetricValue + 1)
+}

ufm-test-services/prometheus.yml

 global:
-  scrape_interval: 5s
+  scrape_interval: 2s
 
 scrape_configs:
   - job_name: 'pushgateway'