Run only affected test in PRs (#1565)

* test change

* script for finding affected test suites

* workflow integration

* workflow debug

* special directories cases

* test

* detect changes in npm modules

* add vscode task and fix ci

* fix lock file

* try to fix jobs deps

* add always() function to the job

* i am stupid

* clean up

* fix for case when file is empty

.github/workflows/checks.yml

@@ -115,11 +115,58 @@ jobs:
         run: yarn --frozen-lockfile --ignore-optional
 
       - name: Run Jest
-        run: yarn test:jest
+        run: yarn test:jest --onlyChanged=${{ github.event_name == 'pull_request' }} --passWithNoTests
+
+  pw_affected_tests:
+    name: Resolve affected Playwright tests
+    runs-on: ubuntu-latest
+    needs: [ code_quality, envs_validation ]
+    if: github.event_name == 'pull_request'
+    steps:
+      - name: Checkout repo
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Setup node
+        uses: actions/setup-node@v4
+        with:
+          node-version: 20.11.0
+          cache: 'yarn'
+
+      - name: Cache node_modules
+        uses: actions/cache@v4
+        id: cache-node-modules
+        with:
+          path: |
+            node_modules
+          key: node_modules-${{ runner.os }}-${{ hashFiles('yarn.lock') }}
+      
+      - name: Install dependencies
+        if: steps.cache-node-modules.outputs.cache-hit != 'true'
+        run: yarn --frozen-lockfile --ignore-optional
+
+      - name: Install script dependencies 
+        run: cd ./deploy/tools/affected-tests && yarn --frozen-lockfile
+      
+      - name: Run script
+        run: yarn test:pw:detect-affected
+
+      - name: Upload result file
+        uses: actions/upload-artifact@v4
+        with:
+          name: playwright-affected-tests
+          path: ./playwright/affected-tests.txt
+          retention-days: 3
 
   pw_tests:
     name: 'Playwright tests / Project: ${{ matrix.project }}'
-    needs: [ code_quality, envs_validation ]
+    needs: [ code_quality, envs_validation, pw_affected_tests ]
+    if: |
+      always() &&
+      needs.code_quality.result == 'success' && 
+      needs.envs_validation.result == 'success' && 
+      (needs.pw_affected_tests.result == 'success' || needs.pw_affected_tests.result == 'skipped') 
     runs-on: ubuntu-latest
     container:
       image: mcr.microsoft.com/playwright:v1.41.1-focal
@@ -156,8 +203,15 @@ jobs:
         if: steps.cache-node-modules.outputs.cache-hit != 'true'
         run: yarn --frozen-lockfile --ignore-optional
 
+      - name: Download affected tests list
+        if: ${{ needs.pw_affected_tests.result == 'success' }}
+        uses: actions/download-artifact@v4
+        with:
+          name: playwright-affected-tests
+          path: ./playwright
+
       - name: Run PlayWright
-        run: yarn test:pw:ci
+        run: yarn test:pw:ci --affected=${{ github.event_name == 'pull_request' }} --pass-with-no-tests
         env:
           HOME: /root
           PW_PROJECT: ${{ matrix.project }}

.gitignore

@@ -51,5 +51,6 @@ yarn-error.log*
 /playwright/.cache/
 /playwright/.browser/
 /playwright/envs.js
+/playwright/affected-tests.txt
 
 **.dec**
\ No newline at end of file

.vscode/tasks.json

@@ -155,6 +155,27 @@
                 "instanceLimit": 1
             }
         },
+        {
+            "type": "shell",
+            "command": "yarn test:pw:detect-affected",
+            "problemMatcher": [],
+            "label": "pw: detect affected",
+            "detail": "detect PW tests affected by changes in current branch",
+            "presentation": {
+                "reveal": "always",
+                "panel": "shared",
+                "focus": true,
+                "close": false,
+                "revealProblems": "onProblem",
+            },
+            "icon": {
+                "color": "terminal.ansiBlue",
+                "id": "diff"
+            },
+            "runOptions": {
+                "instanceLimit": 1
+            },
+        },
 
         // JEST TESTS
         {
@@ -305,6 +326,7 @@
             "options": [
               "",
               "--update-snapshots",
+              "--update-snapshots --affected",
               "--ui",
             ],
             "default": ""

deploy/tools/affected-tests/.gitignore

+/node_modules
\ No newline at end of file

deploy/tools/affected-tests/index.js

+/* eslint-disable no-console */
+const { execSync } = require('child_process');
+const dependencyTree = require('dependency-tree');
+const fs = require('fs');
+const path = require('path');
+
+const ROOT_DIR = path.resolve(__dirname, '../../../');
+
+const TARGET_FILE = path.resolve(ROOT_DIR, './playwright/affected-tests.txt');
+
+const NON_EXISTENT_DEPS = [];
+
+const DIRECTORIES_WITH_TESTS = [
+  path.resolve(ROOT_DIR, './ui'),
+];
+
+function getAllPwFilesInDirectory(directory) {
+  const files = fs.readdirSync(directory, { recursive: true });
+  return files
+    .filter((file) => file.endsWith('.pw.tsx'))
+    .map((file) => path.join(directory, file));
+}
+
+function getFileDeps(filename, changedNpmModules) {
+  return dependencyTree.toList({
+    filename,
+    directory: ROOT_DIR,
+    filter: (path) => {
+      if (path.indexOf('node_modules') === -1) {
+        return true;
+      }
+
+      if (changedNpmModules.some((module) => path.startsWith(module))) {
+        return true;
+      }
+
+      return false;
+    },
+    tsConfig: path.resolve(ROOT_DIR, './tsconfig.json'),
+    nonExistent: NON_EXISTENT_DEPS,
+  });
+}
+
+async function getChangedFiles() {
+  const command = process.env.CI ?
+    `git diff --name-only origin/${ process.env.GITHUB_BASE_REF } ${ process.env.GITHUB_SHA } -- ${ ROOT_DIR }` :
+    `git diff --name-only main $(git branch --show-current) -- ${ ROOT_DIR }`;
+
+  console.log('Executing command: ', command);
+  const files = execSync(command)
+    .toString()
+    .trim()
+    .split('\n')
+    .filter(Boolean);
+
+  return files.map((file) => path.join(ROOT_DIR, file));
+}
+
+function checkChangesInChakraTheme(changedFiles) {
+  const themeDir = path.resolve(ROOT_DIR, './theme');
+  return changedFiles.some((file) => file.startsWith(themeDir));
+}
+
+function checkChangesInSvgSprite(changedFiles) {
+  const iconDir = path.resolve(ROOT_DIR, './icons');
+  const areIconsChanged = changedFiles.some((file) => file.startsWith(iconDir));
+
+  if (!areIconsChanged) {
+    return false;
+  }
+
+  const svgNamesFile = path.resolve(ROOT_DIR, './public/icons/name.d.ts');
+  const areSvgNamesChanged = changedFiles.some((file) => file === svgNamesFile);
+
+  if (!areSvgNamesChanged) {
+    // If only the icons have changed and not the names in the SVG file, we will need to run all tests.
+    // This is because we cannot correctly identify the test files that depend on these changes.
+    return true;
+  }
+
+  // If the icon names have changed, then there should be changes in the components that use them.
+  // Otherwise, typescript would complain about that.
+  return false;
+}
+
+function createTargetFile(content) {
+  fs.writeFileSync(TARGET_FILE, content);
+}
+
+function getPackageJsonUpdatedProps(packageJsonFile) {
+  const command = process.env.CI ?
+    `git diff --unified=0 origin/${ process.env.GITHUB_BASE_REF } ${ process.env.GITHUB_SHA } -- ${ packageJsonFile }` :
+    `git diff --unified=0 main $(git branch --show-current) -- ${ packageJsonFile }`;
+
+  console.log('Executing command: ', command);
+  const changedLines = execSync(command)
+    .toString()
+    .trim()
+    .split('\n')
+    .filter(Boolean)
+    .filter((line) => line.startsWith('+ ') || line.startsWith('- '));
+
+  const changedProps = [ ...new Set(
+    changedLines
+      .map((line) => line.replaceAll(' ', '').replaceAll('+', '').replaceAll('-', ''))
+      .map((line) => line.split(':')[0].replaceAll('"', '')),
+  ) ];
+
+  return changedProps;
+}
+
+function getUpdatedNpmModules(changedFiles) {
+  const packageJsonFile = path.resolve(ROOT_DIR, './package.json');
+
+  if (!changedFiles.includes(packageJsonFile)) {
+    return [];
+  }
+
+  try {
+    const packageJsonContent = JSON.parse(fs.readFileSync(packageJsonFile, 'utf-8'));
+    const usedNpmModules = [
+      ...Object.keys(packageJsonContent.dependencies || {}),
+      ...Object.keys(packageJsonContent.devDependencies || {}),
+    ];
+    const updatedProps = getPackageJsonUpdatedProps(packageJsonFile);
+
+    return updatedProps.filter((prop) => usedNpmModules.includes(prop));
+  } catch (error) {}
+}
+
+async function run() {
+  // NOTES:
+  // - The absence of TARGET_FILE implies that all tests should be run.
+  // - The empty TARGET_FILE implies that no tests should be run.
+
+  const start = Date.now();
+
+  fs.unlink(TARGET_FILE, () => {});
+
+  const changedFiles = await getChangedFiles();
+
+  if (!changedFiles.length) {
+    createTargetFile('');
+    console.log('No changed files found. Exiting...');
+    return;
+  }
+
+  console.log('Changed files in the branch: ', changedFiles);
+
+  if (checkChangesInChakraTheme(changedFiles)) {
+    console.log('Changes in Chakra theme detected. It is advisable to run all test suites. Exiting...');
+    return;
+  }
+
+  if (checkChangesInSvgSprite(changedFiles)) {
+    console.log('There are some changes in the SVG sprite that cannot be linked to a specific component. It is advisable to run all test suites. Exiting...');
+    return;
+  }
+
+  let changedNpmModules = getUpdatedNpmModules(changedFiles);
+
+  if (!changedNpmModules) {
+    console.log('Some error occurred while detecting changed NPM modules. It is advisable to run all test suites. Exiting...');
+    return;
+  }
+
+  console.log('Changed NPM modules in the branch: ', changedNpmModules);
+
+  changedNpmModules = [
+    ...changedNpmModules,
+    ...changedNpmModules.map((module) => `@types/${ module }`), // there are some deps that are resolved to .d.ts files
+  ].map((module) => path.resolve(ROOT_DIR, `./node_modules/${ module }`));
+
+  const allTestFiles = DIRECTORIES_WITH_TESTS.reduce((acc, dir) => {
+    return acc.concat(getAllPwFilesInDirectory(dir));
+  }, []);
+
+  const isDepChanged = (dep) => changedFiles.includes(dep) || changedNpmModules.some((module) => dep.startsWith(module));
+
+  const testFilesToRun = allTestFiles
+    .map((file) => ({ file, deps: getFileDeps(file, changedNpmModules) }))
+    .filter(({ deps }) => deps.some(isDepChanged));
+  const testFileNamesToRun = testFilesToRun.map(({ file }) => path.relative(ROOT_DIR, file));
+
+  if (!testFileNamesToRun.length) {
+    createTargetFile('');
+    console.log('No tests to run. Exiting...');
+    return;
+  }
+
+  createTargetFile(testFileNamesToRun.join('\n'));
+
+  const end = Date.now();
+
+  const testFilesToRunWithFilteredDeps = testFilesToRun.map(({ file, deps }) => ({
+    file,
+    deps: deps.filter(isDepChanged),
+  }));
+
+  console.log('Total time: ', ((end - start) / 1_000).toLocaleString());
+  console.log('Total test to run: ', testFileNamesToRun.length);
+  console.log('Tests to run with changed deps: ', testFilesToRunWithFilteredDeps);
+  console.log('Non existent deps: ', NON_EXISTENT_DEPS);
+}
+
+run();

deploy/tools/affected-tests/package.json

+{
+  "name": "affected-tests",
+  "version": "1.0.0",
+  "main": "index.js",
+  "author": "Vasilii (tom) Goriunov <tom@ohhhh.me>",
+  "license": "MIT",
+  "dependencies": {
+    "dependency-tree": "10.0.9"
+  }
+}

package.json

@@ -26,6 +26,7 @@
     "test:pw:local": "export NODE_PATH=$(pwd)/node_modules && yarn test:pw",
     "test:pw:docker": "docker run --rm --network host -v $(pwd):/work/ -w /work/ -it mcr.microsoft.com/playwright:v1.41.1-focal ./tools/scripts/pw.docker.sh",
     "test:pw:ci": "yarn test:pw --project=$PW_PROJECT",
+    "test:pw:detect-affected": "node ./deploy/tools/affected-tests/index.js",
     "test:jest": "jest",
     "test:jest:watch": "jest --watch",
     "favicon:generate:dev": "./tools/scripts/favicon-generator.dev.sh"

tools/scripts/pw.sh

@@ -10,7 +10,73 @@ dotenv \
 
 yarn svg:build-sprite
 
+# Check if the "--affected" argument is present in the script args
+check_affected_flag() {
+    local affected_flag=false
+
+    for arg in "$@"; do
+        if [[ "$arg" = "--affected"* ]]; then
+            # Extract the value after the equals sign
+            is_affected_value=${is_affected_arg#*=}
+
+            if [ "$is_affected_value" != "false" ]; then
+                affected_flag=true
+            fi
+            
+            break
+        fi
+    done
+
+    echo "$affected_flag"
+}
+
+# Remove the "--affected" argument from the script args
+filter_arguments() {
+    local args=()
+
+    for arg in "$@"; do
+        if [[ "$arg" != "--affected"* ]]; then
+            args+=("$arg")
+        fi
+    done
+
+    echo "${args[@]}"
+}
+
+get_files_to_run() {
+  local is_affected=$1
+  local files_to_run=""
+
+  if [ "$is_affected" = true ]; then
+      affected_tests_file="./playwright/affected-tests.txt"
+
+      if [ -f "$affected_tests_file" ]; then
+            file_content=$(<"$affected_tests_file")
+            files_to_run="${file_content//$'\n'/$' '}"
+
+            if [ -z "$files_to_run" ]; then
+                exit 1
+            fi
+      fi
+  fi
+
+  echo "$files_to_run"
+}
+
+args=$(filter_arguments "$@")
+affected_flag=$(check_affected_flag "$@")
+files_to_run=$(get_files_to_run "$affected_flag")
+if [ $? -eq 1 ]; then
+    echo "No affected tests found in the file. Exiting..."
+    exit 0
+fi
+
+echo "Running Playwright tests with the following arguments: $args"
+echo "Affected flag: $affected_flag"
+echo "Files to run: $files_to_run"
+
 dotenv \
   -v NODE_OPTIONS=\"--max-old-space-size=4096\" \
   -e $config_file \
-  -- playwright test -c playwright-ct.config.ts "$@"
+  -- playwright test -c playwright-ct.config.ts $files_to_run $args
+