ops: Improve check-changed (#4527)

7f8f0fe2 · Matthew Slipper · GitHub · 874457d0 · 7f8f0fe2 · 7f8f0fe2
Commit 7f8f0fe2 authored Dec 19, 2022 by Matthew Slipper Committed by GitHub Dec 19, 2022
Showing with 188 additions and 97 deletions

config.yml .circleci/config.yml +49 -97

.gitignore ops/check-changed/.gitignore +1 -0

main.py ops/check-changed/main.py +126 -0

requirements.txt ops/check-changed/requirements.txt +12 -0

No files found.
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -44,6 +44,19 @@ commands:
            gcloud auth login --brief --cred-file "<< parameters.gcp_cred_config_file_path >>"
            # Configure ADC
            echo "export GOOGLE_APPLICATION_CREDENTIALS='<< parameters.gcp_cred_config_file_path >>'" | tee -a "$BASH_ENV"
+  check-changed:
+    description: "Conditionally halts a step if certain modules change"
+    parameters:
+      patterns:
+        type: string
+        description: "Comma-separated list of dependencies"
+    steps:
+      - run:
+          name: "Check for changes"
+          command: |
+            cd ops/check-changed
+            pip3 install -r requirements.txt
+            python3 main.py "<<parameters.patterns>>"
 jobs:
  yarn-monorepo:
    docker:
@@ -51,14 +64,8 @@ jobs:
    resource_class: xlarge
    steps:
      - checkout
-      - run:
+      - check-changed:
-          name: Check if we should run
+          patterns: op-bindings,op-chain-ops,packages/
-          command: |
-            shopt -s inherit_errexit
-            CHANGED=$(check-changed "(op-bindings|op-chain-ops|packages/)" || echo "TRUE")
-            if [[ "$CHANGED" = "FALSE" ]]; then
-              circleci step halt
-            fi
      - run:
          name: Install dependencies
          command: yarn --frozen-lockfile
@@ -211,15 +218,9 @@ jobs:
    resource_class: large
    steps:
      - checkout
-      - attach_workspace: {at: "."}
+      - attach_workspace: { at: "." }
-      - run:
+      - check-changed:
-          name: Check if we should run
+          patterns: contracts-bedrock,hardhat-deploy-config
-          command: |
-            shopt -s inherit_errexit
-            CHANGED=$(check-changed "(contracts-bedrock|hardhat-deploy-config)" || echo "TRUE")
-            if [[ "$CHANGED" = "FALSE" ]]; then
-              circleci step halt
-            fi
      - run:
          name: lint
          command: yarn lint:check
@@ -268,15 +269,9 @@ jobs:
      - image: ethereumoptimism/ci-builder:latest
    steps:
      - checkout
-      - attach_workspace: {at: "."}
+      - attach_workspace: { at: "." }
-      - run:
+      - check-changed:
-          name: Check if we should run
+          patterns: contracts-bedrock,contracts
-          command: |
-            shopt -s inherit_errexit
-            CHANGED=$(check-changed "(contracts-bedrock/contracts)" || echo "TRUE")
-            if [[ "$CHANGED" = "FALSE" ]]; then
-              circleci step halt
-            fi
      - run:
          name: Compile with metadata hash
          command: yarn clean && yarn build:with-metadata
@@ -301,15 +296,9 @@ jobs:
    resource_class: <<parameters.size>>
    steps:
      - checkout
-      - attach_workspace: {at: "."}
+      - attach_workspace: { at: "." }
-      - run:
+      - check-changed:
-          name: Check if we should run
+          patterns: contracts-bedrock,contracts
-          command: |
-            shopt -s inherit_errexit
-            CHANGED=$(check-changed "(contracts-bedrock/contracts)" || echo "TRUE")
-            if [[ "$CHANGED" = "FALSE" ]]; then
-              circleci step halt
-            fi
      - run:
          name: Echidna Fuzz <<parameters.echidna_target>>
          command: yarn echidna:<<parameters.echidna_target>>
@@ -322,15 +311,9 @@ jobs:
    resource_class: medium
    steps:
      - checkout
-      - attach_workspace: {at: "."}
+      - attach_workspace: { at: "." }
-      - run:
+      - check-changed:
-          name: Check if we should run
+          patterns: contracts-bedrock,op-bindings
-          command: |
-            shopt -s inherit_errexit
-            CHANGED=$(check-changed "(contracts-bedrock|op-bindings)" || echo "TRUE")
-            if [[ "$CHANGED" = "FALSE" ]]; then
-              circleci step halt
-            fi
      - run:
          name: check go bindings
          command: make && git diff --exit-code
@@ -353,15 +336,9 @@ jobs:
    resource_class: large
    steps:
      - checkout
-      - attach_workspace: {at: "."}
+      - attach_workspace: { at: "." }
-      - run:
+      - check-changed:
-          name: Check if we should run
+          patterns: <<parameters.package_name>>,<<parameters.dependencies>>
-          command: |
-            shopt -s inherit_errexit
-            CHANGED=$(check-changed "(packages/<<parameters.package_name>>|packages/<<parameters.dependencies>>)" || echo "TRUE")
-            if [[ "$CHANGED" = "FALSE" ]]; then
-              circleci step halt
-            fi
      - run:
          name: Lint
          command: yarn lint && git diff --exit-code
@@ -379,6 +356,8 @@ jobs:
      image: ubuntu-2204:2022.07.1
    steps:
      - checkout
+      - check-changed:
+          patterns: specs/(.*)\.md$
      - run:
          name: yarn dev deps  # todo: what's the best way to pull in the dependencies for linting? yarn install above is using production env without dev dependencies
          command: yarn install --production=false
@@ -398,14 +377,8 @@ jobs:
      - image: ethereumoptimism/ci-builder:latest
    steps:
      - checkout
-      - run:
+      - check-changed:
-          name: Check if we should run
+          patterns: op-node
-          command: |
-            shopt -s inherit_errexit
-            CHANGED=$(check-changed "op-node" || echo "TRUE")
-            if [[ "$CHANGED" = "FALSE" ]]; then
-              circleci step halt
-            fi
      - run:
          name: Fuzz
          command: make fuzz
@@ -416,15 +389,9 @@ jobs:
      - image: ethereumoptimism/ci-builder:latest
    steps:
      - checkout
-      - attach_workspace: {at: "."}
+      - attach_workspace: { at: "." }
-      - run:
+      - check-changed:
-          name: Check if we should run
+          patterns: packages
-          command: |
-            shopt -s inherit_errexit
-            CHANGED=$(check-changed "packages/" || echo "TRUE")
-            if [[ "$CHANGED" = "FALSE" ]]; then
-              circleci step halt
-            fi
      # Note: The below needs to be manually configured whenever we
      # add a new package to CI.
      - run:
@@ -501,10 +468,12 @@ jobs:
        description: If the op-e2e package should use HTTP clients
        type: string
    docker:
-      - image: cimg/go:1.19
+      - image: ethereumoptimism/ci-builder:latest
    resource_class: xlarge
    steps:
      - checkout
+      - check-changed:
+          patterns: op-(.+),contracts-bedrock,contracts
      - run:
          name: prep results dir
          command: mkdir -p /tmp/test-results
@@ -541,15 +510,8 @@ jobs:
      - image: cimg/postgres:14.1
    steps:
      - checkout
-      - run:
+      - check-changed:
-          name: Check if we should run
+          patterns: <<parameters.working_directory>>,<<parameters.dependencies>>
-          command: |
-            shopt -s inherit_errexit
-            CHANGED=$(check-changed "(<<parameters.working_directory>>|<<parameters.dependencies>>)" || echo "TRUE")
-            echo $CHANGED
-            if [[ "$CHANGED" = "FALSE" ]]; then
-              circleci step halt
-            fi
      - run:
          name: Lint
          command: golangci-lint run -E goimports,sqlclosecheck,bodyclose,asciicheck,misspell,errorlint -e "errors.As" -e "errors.Is" ./...
@@ -576,14 +538,8 @@ jobs:
      - image: ethereumoptimism/ci-builder:latest
    steps:
      - checkout
-      - run:
+      - check-changed:
-          name: Check if we should run
+          patterns: l2geth
-          command: |
-            shopt -s inherit_errexit
-            CHANGED=$(check-changed "l2geth" || echo "TRUE")
-            if [[ "$CHANGED" = "FALSE" ]]; then
-              circleci step halt
-            fi
      - run:
          name: Test
          command: make test
@@ -602,6 +558,8 @@ jobs:
      DOCKER_BUILDKIT: 1
    steps:
      - checkout
+      - check-changed:
+          patterns: op-(.+),packages,integration-tests
      - run:
          name: Install latest golang
          command: |
@@ -681,14 +639,8 @@ jobs:
    parallelism: 3
    steps:
      - checkout
-      - run:
+      - check-changed:
-          name: Check if we should run
+          patterns: l2geth,common-ts,contracts,core-utils,message-relayer,data-transport-layer,replica-healthcheck,sdk,batch-submitter,gas-oracle,bss-core,integration-tests
-          command: |
-            shopt -s inherit_errexit
-            CHANGED=$(bash ./ops/docker/ci-builder/check-changed.sh "(l2geth|common-ts|contracts|core-utils|message-relayer|data-transport-layer|replica-healthcheck|sdk|batch-submitter|gas-oracle|bss-core|integration-tests)/" || echo "TRUE")
-            if [[ "$CHANGED" = "FALSE" ]]; then
-              circleci step halt
-            fi
      - run:
          name: Bring up the stack
          command: |
@@ -781,7 +733,7 @@ jobs:
      - go/load-cache
      - go/mod-download
      - go/save-cache
-      - run: {command: "go build ."}
+      - run: { command: "go build ." }
      - run:
          command: |
            ./hive \

--- a/ops/check-changed/.gitignore
+++ b/ops/check-changed/.gitignore
+venv
\ No newline at end of file
--- a/ops/check-changed/main.py
+++ b/ops/check-changed/main.py
+import logging.config
+import os
+import re
+import subprocess
+import sys
+from github import Github
+WHITELISTED_BRANCHES = {
+    'master',
+    'develop'
+}
+LOGGING_CONFIG = {
+    'version': 1,
+    'disable_existing_loggers': True,
+    'formatters': {
+        'standard': {
+            'format': '%(asctime)s [%(levelname)s]: %(message)s'
+        },
+    },
+    'handlers': {
+        'default': {
+            'level': 'INFO',
+            'formatter': 'standard',
+            'class': 'logging.StreamHandler',
+            'stream': 'ext://sys.stderr'
+        },
+    },
+    'loggers': {
+        '': {
+            'handlers': ['default'],
+            'level': 'INFO',
+            'propagate': False
+        },
+    }
+}
+logging.config.dictConfig(LOGGING_CONFIG)
+log = logging.getLogger(__name__)
+def main():
+    patterns = sys.argv[1].split(',')
+    patterns.append(r'^\.circleci/\.*')
+    fp = os.path.realpath(__file__)
+    monorepo_path = os.path.realpath(os.path.join(fp, '..', '..'))
+    log.info('Discovered monorepo path: %s', monorepo_path)
+    current_branch = git_cmd('rev-parse --abbrev-ref HEAD', monorepo_path)
+    log.info('Current branch: %s', current_branch)
+    if current_branch in WHITELISTED_BRANCHES:
+        log.info('Current branch %s is whitelisted, triggering build', current_branch)
+        exit_build()
+    pr_urls = os.getenv('CIRCLE_PULL_REQUESTS', None)
+    pr_urls = pr_urls.split(',') if pr_urls else []
+    if len(pr_urls) == 0:
+        log.info('Not a PR build, skipping build')
+        exit_nobuild()
+    if len(pr_urls) > 1:
+        log.warning('Multiple PR URLs found, choosing the first one. PRs found:')
+        for url in pr_urls:
+            log.warning(url)
+    gh_token = os.getenv('GITHUB_ACCESS_TOKEN')
+    if gh_token is None:
+        log.info('No GitHub access token found - likely a fork. Triggering build')
+        exit_build()
+    g = Github(gh_token)
+    try:
+        g.get_user()
+        repo = g.get_repo(os.getenv('CIRCLE_PROJECT_USERNAME') + '/' + os.getenv('CIRCLE_PROJECT_REPONAME'))
+    except Exception:
+        log.exception('Failed to get repo from GitHub')
+        exit_build()
+    pr = repo.get_pull(int(pr_urls[0].split('/')[-1]))
+    log.info('Found PR: %s', pr.url)
+    base_sha = pr.base.sha
+    head_sha = pr.head.sha
+    diffs = git_cmd('diff --name-only {}...{}'.format(base_sha, head_sha), monorepo_path).split('\n')
+    log.info('Found diff. Checking for matches...')
+    for diff in diffs:
+        if match_path(diff, patterns):
+            log.info('Match found, triggering build')
+            exit_build()
+        else:
+            log.info('❌ no match found on %s', diff)
+    log.info('No matches found, skipping build')
+    exit_nobuild()
+def git_cmd(cmd, cwd):
+    return subprocess.check_output(['git'] + cmd.split(' '), cwd=cwd).decode('utf-8').strip()
+def match_path(path, patterns):
+    for pattern in patterns:
+        if re.search(pattern, path):
+            log.info('✅ match found on %s: %s', path, pattern)
+            return True
+    return False
+def exit_build():
+    sys.exit(0)
+def exit_nobuild():
+    subprocess.check_call(['circleci', 'step', 'halt'])
+    sys.exit(0)
+if __name__ == '__main__':
+    try:
+        main()
+    except Exception:
+        log.exception('Unhandled exception, triggering build')
+        exit_build()
--- a/ops/check-changed/requirements.txt
+++ b/ops/check-changed/requirements.txt
+certifi==2022.12.7
+cffi==1.15.1
+charset-normalizer==2.1.1
+Deprecated==1.2.13
+idna==3.4
+pycparser==2.21
+PyGithub==1.57
+PyJWT==2.6.0
+PyNaCl==1.5.0
+requests==2.28.1
+urllib3==1.26.13
+wrapt==1.14.1