Merge pull request #7357 from ethereum-optimism/wyatt/ufm/gh-workflow-and-refactors

Init Github workflow and some refactors

.github/workflows/ufm-test-service-1-hour.yml

+name: 'UFM Test Service: 1 hour'
+
+on:
+  schedule:
+    # Run every hour
+    - cron: '0 * * * *'
+
+jobs:
+  ufm_test_service_metamask:
+    name: 'UFM Test Service: Metamask'
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Run Docker Compose
+        run: docker-compose run metamask
+        env:
+          CI: ${{ secrets.CI }}
+          PROMETHEUS_SERVER_URL: ${{ secrets.PROMETHEUS_SERVER_URL }}
+          PROMETHEUS_PUSHGATEWAY_URL: ${{ secrets.PROMETHEUS_PUSHGATEWAY_URL }}
+          METAMASK_SECRET_WORDS_OR_PRIVATEKEY: ${{ secrets.METAMASK_SECRET_WORDS_OR_PRIVATEKEY }}
+          METAMASK_NETWORK: ${{ secrets.METAMASK_NETWORK || 'goerli' }}
+          METAMASK_PASSWORD: ${{ secrets.METAMASK_PASSWORD || 'T3st_P@ssw0rd!' }}
+          METAMASK_DAPP_URL: ${{ secrets.METAMASK_DAPP_URL || 'http://localhost:9011' }}
+          METAMASK_OP_GOERLI_RPC_URL: ${{ secrets.METAMASK_OP_GOERLI_RPC_URL }}

.gitignore

@@ -26,6 +26,7 @@ packages/contracts-bedrock/deployments/anvil
 # jetbrains
 .idea/
 
+.secrets
 .env
 .env*
 !.env.example

ufm-test-services/.env.example

+# Used by Test Services to perform certain actions if in CI environment
 CI=false
 
-GRAFANA_ADMIN_PWD=
+# This is the password used to login into Grafana dashboard as the admin user
+GRAFANA_ADMIN_PWD=op
 
-# Used to set the display to be used by playwright when running Metamask test service.
-# The following are to be used if running on MacOS
-# MM_DISPLAY=host.docker.internal:0
-# MM_DISPLAY_VOLUME=/tmp/.X11-unix:/tmp/.X11-unix
-MM_DISPLAY=
-MM_DISPLAY_VOLUME=
+# Used by Test Services to query metrics. http://prometheus will use Docker's built-in DNS
+PROMETHEUS_SERVER_URL="http://prometheus:9090"
+
+# Used by Test Services to push metrics. http://pushgateway will use Docker's built-in DNS
+PROMETHEUS_PUSHGATEWAY_URL="http://pushgateway:9091"
+
+# If true, Metamask Test Service will install Xvfb inside it's container and used that for Playwright tests.
+# If false, Metamask you will need to specify METAMASK_DISPLAY and METAMASK_DISPLAY_VOLUME so Playwright can connect to a display
+METAMASK_RUN_HEADLESS=true
+
+# The display used for running Playwright tests
+METAMASK_DISPLAY=host.docker.internal:0
+
+# The storage for Playwright to store test result, screenshots, videos, etc.
+METAMASK_DISPLAY_VOLUME=/tmp/.X11-unix:/tmp/.X11-unix
+
+# Mnemonic used to initialize Metamask, make sure there's enough ETH to run tests
+METAMASK_SECRET_WORDS_OR_PRIVATEKEY="test test test test test test test test test test test junk"
+
+# The initial network Metamask will be initialized with, Test Service will override with OP Goerli
+METAMASK_NETWORK="goerli"
+
+# The password to unlock Metamask
+METAMASK_PASSWORD="T3st_P@ssw0rd!"
+
+# The URL of the Metamask test dApp that will be spun up automatically for testing against
+METAMASK_DAPP_URL="http://localhost:9011"
+
+# The OP Goerli RPC provider to be used to read/write data
+METAMASK_OP_GOERLI_RPC_URL=""

ufm-test-services/.secrets.example

+# Used by Test Services to perform certain actions if in CI environment
+CI=true
+
+# This is the password used to login into Grafana dashboard as the admin user
+GRAFANA_ADMIN_PWD=op
+
+# Used by Test Services to query metrics. 172.17.0.1 is the default Docker gateway address
+PROMETHEUS_SERVER_URL="http://172.17.0.1:9090"
+
+# Used by Test Services to push metrics. 172.17.0.1 is the default Docker gateway address
+PROMETHEUS_PUSHGATEWAY_URL="http://172.17.0.1:9091"
+
+# If true, Metamask Test Service will install Xvfb inside it's container and used that for Playwright tests.
+# If false, Metamask you will need to specify METAMASK_DISPLAY and METAMASK_DISPLAY_VOLUME so Playwright can connect to a display
+METAMASK_RUN_HEADLESS=true
+
+# The display used for running Playwright tests
+METAMASK_DISPLAY=host.docker.internal:0
+
+# The storage for Playwright to store test result, screenshots, videos, etc.
+METAMASK_DISPLAY_VOLUME=/tmp/.X11-unix:/tmp/.X11-unix
+
+# Mnemonic used to initialize Metamask, make sure there's enough ETH to run tests
+METAMASK_SECRET_WORDS_OR_PRIVATEKEY="test test test test test test test test test test test junk"
+
+# The initial network Metamask will be initialized with, Test Service will override with OP Goerli.
+# Will be defaulted to goerli in Github workflow file
+METAMASK_NETWORK="goerli"
+
+# The password to unlock Metamask
+# Will be defaulted to T3st_P@ssw0rd! in Github workflow file
+METAMASK_PASSWORD="T3st_P@ssw0rd!"
+
+# The URL of the Metamask test dApp that will be spun up automatically for testing against
+# Will be defaulted to http://localhost:9011 in Github workflow file
+METAMASK_DAPP_URL="http://localhost:9011"
+
+# The OP Goerli RPC provider to be used to read/write data
+METAMASK_OP_GOERLI_RPC_URL=""

ufm-test-services/README.md

+# User Facing Monitoring
+
+This project allows you to create _Test Services_ which are Docker containers configured to run a set of tasks, generate metrics, and push those metrics to a Prometheus Pushgateway that can later be scraped and queried by a Grafana dashboard
+
+This project has two modes of execution: CI and local
+
+## CI Execution
+
+![Diagram of UFM execution flow in CI](./assets//ufm-ci-execution.svg)
+
+Starting from left to right in the above diagram:
+
+1. Github Workflow files are created for each time interval Test Services should be ran
+    - All Test Services that should be ran for a specific time interval (e.g. 1 hour) should be defined in the same Github Workflow file
+2. Github will run a workflow at it's specified time interval, triggering all of it's defined Test Services to run
+3. `docker-compose.yml` builds and runs each Test Service, setting any environment variables that can be sourced from Github secrets
+4. Each Test Service will run it's defined tasks, generate it's metrics, and push them to an already deployed instance of Prometheus Pushgateway
+5. An already deployed instance of Prometheus will scrape the Pushgateway for metrics
+6. An already deployed Grafana dashboard will query Prometheus for metric data to display
+
+### Mocking CI Execution Locally
+
+Thanks to [Act](https://github.com/nektos/act), Github actions can be "ran" locally for testing. Here's how:
+
+1. [Install Act](https://github.com/nektos/act#installation-through-package-managers)
+    - For MacOS: `brew install act`
+2. [Generate](https://docs.github.com/en/enterprise-server@3.6/authentication/keeping-your-account-and-data-secure/managing-your-personal-access-tokens#creating-a-personal-access-token) a Github access token
+    - I believe it only needs permission to read from repositories
+3. Copy secrets file: `cp .secrets.example .secrets` and fill it out
+4. Create `~/.actrc` and copy the following into it:
+```bash
+-P ubuntu-latest=catthehacker/ubuntu:full-18.04
+-P ubuntu-latest=catthehacker/ubuntu:full-18.04
+-P ubuntu-18.04=catthehacker/ubuntu:full-18.04
+```
+5. Spin up the Pushgateway, Prometheus, and Grafana containers: `docker-compose up pushgateway prometheus grafana`
+    - Optionally, you could specify a remote Pushgateway and Prometheus instance to push metrics to in `.secrets`
+        - `PROMETHEUS_SERVER_URL` and `PROMETHEUS_PUSHGATEWAY_URL`
+6. Run `act -W .github/workflows/YOUR_WORKFLOW_FILE.yml -s GITHUB_TOKEN=YOUR_TOKEN --secret-file ./ufm-test-services/.secrets --container-architecture linux/amd64`
+    - `--container-architecture linux/amd64` is necessary if running on MacOS, but may be different for you
+    - Downloading the Github Actions Docker image takes a while and is pretty big, so you might need to allocate more resources to Docker, or `docker prune`/remove no longer needed images/containers/volumes
+
+Following these steps will use `act` to mock the Github Actions environment using a Docker container. The Github Actions container will then spin up a nested container to run each Test Service. Each Test Service _should_ be configured to generate and push metrics to the given Pushgateway, so after `act` finishes execution, you should be able to login into Grafana and view the dashboards
+
+## Local Execution
+
+### Running With Scheduler
+
+![Diagram of UFM execution flow locally](./assets//ufm-local-execution.svg)
+
+Starting from left to right in the above diagram:
+
+1. Copy env file: `cp .env.example .env` and fill it out
+    - If you want to run local instances of the Pushgateway, Prometheus, and Grafana, you can run:
+        ```bash
+        docker-compose up pushgateway prometheus grafana
+        ```
+        to spin those up. Otherwise, you should override the defaults URLs in the `.env` for:
+        - `PROMETHEUS_SERVER_URL` and `PROMETHEUS_PUSHGATEWAY_URL`
+3. You'll need to setup some sort of scheduler to run your Test Services at specific time intervals
+    - For Linux/MacOS this can be accomplished using `cron`
+        - Edit your `cron` job file using `crontrab -e`
+        - Here is some example code to get you started, also found in `crontab.example` file:
+            ```bash
+            # Needs to point to docker, otherwise you'll get the error: exec: "docker": executable file not found in $PATH
+            PATH=/
+
+            # Runs every 1 hour
+            0 * * * * /usr/local/bin/docker-compose -f /path/to/docker-compose.yml --profile 1hour up -d
+
+            # Runs every 1 day
+            0 12 * * * /usr/local/bin/docker-compose -f /path/to/docker-compose.yml --profile 1day up -d
+
+            # Runs every 7 day
+            0 12 */7 * * /usr/local/bin/docker-compose -f /path/to/docker-compose.yml --profile 7day up -d
+            ```
+
+### Running Manually
+
+1. Copy env file: `cp .env.example .env` and fill it out
+2. Run `docker-compose` for which ever Test Service you'd like to run e.g.:
+    - `docker-compose run testService1`
+    - `docker-compose --profile 1hour up`
+
+## Test Services
+
+If you're trying to run a specific Test Service, make sure to check our their `README.md`s, as they may have some required prerequisites to setup before they'll run as expected

ufm-test-services/crontab-example

-# Needs to point to docker, otherwise you'll get the error: exec: "docker": executable file not found in $PATH
-PATH=/
-
-# Runs every minute
-* * * * * /usr/local/bin/docker-compose -f /path/to/docker-compose.yml --profile 1minute up -d
-
-# Runs every 5 minutes
-*/5 * * * * /usr/local/bin/docker-compose -f /path/to/docker-compose.yml --profile 5minutes up -d

ufm-test-services/crontab.example

 # Needs to point to docker, otherwise you'll get the error: exec: "docker": executable file not found in $PATH
 PATH=/
 
-# Runs every minute
-# * * * * * /usr/local/bin/docker-compose -f /path/to/docker-compose.yml --profile 1minute up -d
+# Runs every 1 hour
+0 * * * * /usr/local/bin/docker-compose -f /path/to/docker-compose.local.yml --profile 1hour up -d
 
-# Runs every 5 minutes
-*/5 * * * * /usr/local/bin/docker-compose -f /path/to/docker-compose.yml --profile 5minute up -d
+# Runs every 1 day
+0 12 * * * /usr/local/bin/docker-compose -f /path/to/docker-compose.local.yml --profile 1day up -d
+
+# Runs every 7 day
+0 12 */7 * * /usr/local/bin/docker-compose -f /path/to/docker-compose.local.yml --profile 7day up -d

ufm-test-services/docker-compose.yml

@@ -18,8 +18,6 @@ services:
       - "9090:9090"
     volumes:
       - ./prometheus.yml:/etc/prometheus/prometheus.yml
-    depends_on:
-      - pushgateway
     command:
       - '--config.file=/etc/prometheus/prometheus.yml'
     read_only: true
@@ -31,8 +29,6 @@ services:
     container_name: grafana
     ports:
       - "3000:3000"
-    depends_on:
-      - prometheus
     environment:
       - GF_SECURITY_ADMIN_PASSWORD=${GRAFANA_ADMIN_PWD}
     volumes:
@@ -42,23 +38,25 @@ services:
       - "no-new-privileges:true"
 
   metamask:
-    build: ./metamask
+    build:
+      context: ./metamask
+      dockerfile: Dockerfile
+      args:
+        - METAMASK_RUN_HEADLESS=${METAMASK_RUN_HEADLESS}
     container_name: test-service-metamask
-    profiles: ["30minute"]
-    depends_on:
-      - prometheus
+    profiles: ["5minute"]
     environment:
-      DISPLAY: ${MM_DISPLAY:-:0}
-      CI: ${CI:-true}
+      CI: ${CI}
+      DISPLAY: ${METAMASK_DISPLAY}
+      GRAFANA_ADMIN_PWD: ${GRAFANA_ADMIN_PWD}
+      PROMETHEUS_SERVER_URL: ${PROMETHEUS_SERVER_URL}
+      PROMETHEUS_PUSHGATEWAY_URL: ${PROMETHEUS_PUSHGATEWAY_URL}
+      METAMASK_RUN_HEADLESS: ${METAMASK_RUN_HEADLESS}
+      METAMASK_SECRET_WORDS_OR_PRIVATEKEY: ${METAMASK_SECRET_WORDS_OR_PRIVATEKEY}
+      METAMASK_NETWORK: ${METAMASK_NETWORK}
+      METAMASK_PASSWORD: ${METAMASK_PASSWORD}
+      METAMASK_DAPP_URL: ${METAMASK_DAPP_URL}
+      METAMASK_OP_GOERLI_RPC_URL: ${METAMASK_OP_GOERLI_RPC_URL}
     volumes:
-      - ${MM_DISPLAY_VOLUME}
-    restart: no
-
-  # Example Test Service Config
-  # my_test_service:
-  #   build: ./my_test_service_dir
-  #   container_name: my_test_service
-  #   profiles: ["1minute"]
-  #   depends_on:
-  #     - prometheus
-  #   restart: no
+      - ${METAMASK_DISPLAY_VOLUME:-/path/in/container/if/no/env/set}
+    restart: "no"

ufm-test-services/metamask/.env.example

-METAMASK_SECRET_WORDS_OR_PRIVATEKEY=""
-METAMASK_NETWORK="goerli"
-METAMASK_PASSWORD="Test@1234"
-METAMASK_DAPP_URL="http://localhost:9011"
-
-OP_GOERLI_RPC_URL=""

ufm-test-services/metamask/Dockerfile

+# Using the Playwright image
 FROM mcr.microsoft.com/playwright:v1.37.1-jammy
 
+# Setting the working directory
 WORKDIR /app
 
+# Update PATH
 ENV PATH /app/node_modules/.bin:$PATH
 
-COPY package.json /app/
+RUN if [ "$METAMASK_RUN_HEADLESS" = "true" ]; then \
+        apt-get update && \
+        apt-get install -y xvfb && \
+        rm -rf /var/lib/apt/lists/* ; \
+    fi
 
+# Copy necessary files and directories
+COPY package.json /app/
 RUN npm install
-
 COPY tests /app/tests/
-COPY .env /app/
 COPY playwright.config.ts /app/
 COPY start.sh /app/
 COPY tsconfig.json /app/
 
-CMD /bin/sh /app/start.sh
+# Start the script
+CMD /bin/bash /app/start.sh

ufm-test-services/metamask/README.md

@@ -20,7 +20,7 @@ Apple’s operating system doesn’t include a built-in XServer, but we can use
 4. Start XQuartz by executing `xhost +localhost` in your terminal
 5. Open Docker Desktop and edit settings to give access to `/tmp/.X11-unix` in `Preferences -> Resources -> File sharing`
 
-Once XQuartz is running with the right permissions, you can populate the environment variable and socket Docker args:
+Once XQuartz is running with the right permissions, you can populate the environment variable and socket Docker args (these envs are defaulted to the below values in `ufm-test-services/.env.example`):
 
 ```bash
 docker run --rm -it \

ufm-test-services/metamask/start.sh

 #!/bin/bash
 
-npm test
\ No newline at end of file
+if [ "$METAMASK_RUN_HEADLESS" == "true" ]; then
+    # Start Xvfb in the background on display :99
+    Xvfb :99 &
+
+    # Set the DISPLAY environment variable
+    export DISPLAY=:99
+fi
+
+npm test
+
+# If something goes wrong, Playwright generates this file, but only if there is an error.
+# npx playwright show-trace will log the Playwright error
+if [ -f "test-results/metamask-Setup-wallet-and-dApp-chromium-retry1/trace.zip" ]; then
+  npx playwright show-trace "test-results/metamask-Setup-wallet-and-dApp-chromium-retry1/trace.zip"
+fi
+

ufm-test-services/metamask/tests/metamask.spec.ts

@@ -6,14 +6,13 @@ import { mnemonicToAccount, privateKeyToAccount } from 'viem/accounts'
 
 import { testWithSynpress } from './testWithSynpressUtil'
 import {
-  getMetamaskTxCounterValue,
   incrementMetamaskTxCounter,
   setMetamaskTxCounter,
 } from './prometheusUtils'
 
 const env = z.object({
   METAMASK_SECRET_WORDS_OR_PRIVATEKEY: z.string(),
-  OP_GOERLI_RPC_URL: z.string().url(),
+  METAMASK_OP_GOERLI_RPC_URL: z.string().url(),
   METAMASK_DAPP_URL: z.string().url()
 }).parse(process.env)
 
@@ -36,18 +35,21 @@ test.afterAll(async () => {
 })
 
 testWithSynpress('Setup wallet and dApp', async ({ page }) => {
+  console.log('Seting up wallet and dApp...')
   sharedPage = page
   await sharedPage.goto('http://localhost:9011')
+  console.log('Setup wallet and dApp')
 })
 
 testWithSynpress('Add OP Goerli network', async () => {
+  console.log('Adding OP Goerli network...')
   const expectedChainId = '0x1a4'
 
   await metamask.addNetwork({
     name: 'op-goerli',
     rpcUrls: {
       default: {
-        http: [env.OP_GOERLI_RPC_URL],
+        http: [env.METAMASK_OP_GOERLI_RPC_URL],
       },
     },
     id: '420',
@@ -68,9 +70,11 @@ testWithSynpress('Add OP Goerli network', async () => {
     await incrementMetamaskTxCounter(false)
     throw error
   }
+  console.log('Added OP Goerli network')
 })
 
 test(`Connect wallet with ${expectedSender}`, async () => {
+  console.log(`Connecting wallet with ${expectedSender}...`)
   await sharedPage.click('#connectButton')
   await metamask.acceptAccess()
 
@@ -81,9 +85,11 @@ test(`Connect wallet with ${expectedSender}`, async () => {
     await incrementMetamaskTxCounter(false)
     throw error
   }
+  console.log(`Connected wallet with ${expectedSender}`)
 })
 
 test('Send an EIP-1559 transaciton and verfiy success', async () => {
+  console.log('Sending an EIP-1559 transaciton and verfiy success...')
   const expectedTransferAmount = '0x1'
   const expectedTxType = '0x2'
 
@@ -130,4 +136,5 @@ test('Send an EIP-1559 transaciton and verfiy success', async () => {
     await incrementMetamaskTxCounter(false)
     throw error
   }
+  console.log('Sent an EIP-1559 transaciton and verfied success')
 })

ufm-test-services/metamask/tests/prometheusUtils.ts

@@ -10,7 +10,7 @@ const env = z
   .parse(process.env)
 
 const txSuccessMetricName = 'metamask_tx_success'
-const txFailureMetricName = 'metamask_tx_failuree'
+const txFailureMetricName = 'metamask_tx_failure'
 
 const txSuccessCounter = new Counter({
   name: txSuccessMetricName,