feat(ci): add tests for semgrep rules (#12563)

Adds rule tests for all semgrep rules and fixes a few bugs that
were found during testing. Moves semgrep rules into the semgrep
folder without the . prefix because the prefix caused semgrep to
be unable to run the tests.

.circleci/config.yml

@@ -676,6 +676,8 @@ jobs:
       - run:
           name: print forge version
           command: forge --version
+      - run-contracts-check:
+          command: semgrep-test-validity-check
       - run-contracts-check:
           command: semgrep
       - run-contracts-check:
@@ -1298,7 +1300,10 @@ workflows:
       - semgrep-scan
       - semgrep-scan:
           name: semgrep-scan-local
-          scan_command: semgrep scan --timeout=100 --config=./.semgrep --error .
+          scan_command: semgrep scan --timeout=100 --config=./semgrep --error .
+      - semgrep-scan:
+          name: semgrep-test
+          scan_command: semgrep scan --test semgrep/
       - go-lint
       - fuzz-golang:
           name: fuzz-golang-<<matrix.package_name>>

.semgrepignore

@@ -9,7 +9,7 @@ vendor/
 *.min.js
 
 # Semgrep rules folder
-.semgrep
+semgrep/
 
 # Semgrep-action log folder
 .semgrep_logs/

justfile

@@ -3,7 +3,11 @@ issues:
 
 # Runs semgrep on the entire monorepo.
 semgrep:
-  semgrep scan --config=.semgrep --error .
+  semgrep scan --config=semgrep --error .
+
+# Runs semgrep tests.
+semgrep-test:
+  semgrep scan --test semgrep/
 
 lint-shellcheck:
   find . -type f -name '*.sh' -not -path '*/node_modules/*' -not -path './packages/contracts-bedrock/lib/*' -not -path './packages/contracts-bedrock/kout*/*' -exec sh -c 'echo \"Checking $1\"; shellcheck \"$1\"' _ {} \\;

packages/contracts-bedrock/justfile

@@ -163,6 +163,10 @@ semver-natspec-check-no-build:
 # Checks that semver natspec is equal to the actual semver version.
 semver-natspec-check: build semver-natspec-check-no-build
 
+# Checks that the semgrep tests are valid.
+semgrep-test-validity-check:
+  forge fmt ../../semgrep/sol-rules.t.sol --check
+
 # Checks that forge test names are correctly formatted.
 lint-forge-tests-check:
   go run ./scripts/checks/names
@@ -191,12 +195,13 @@ validate-spacers: build validate-spacers-no-build
 
 # Runs semgrep on the contracts.
 semgrep:
-  cd ../../ && semgrep scan --config=.semgrep ./packages/contracts-bedrock
+  cd ../../ && semgrep scan --config=semgrep ./packages/contracts-bedrock
 
 # TODO: Also run lint-forge-tests-check but we need to fix the test names first.
 # Runs all checks.
 check:
   @just gas-snapshot-check-no-build \
+  semgrep-test-validity-check \
   unused-imports-check-no-build \
   snapshots-check-no-build \
   lint-check \

.semgrep/sol-rules.yaml → semgrep/sol-rules.yaml

@@ -10,16 +10,34 @@ rules:
     severity: ERROR
     message: vm.expectRevert is followed by a low-level call but not followed by assertion expecting revert
     patterns:
-      - pattern: |
-          vm.expectRevert(...);
-          $CALL;
-          $CHECK;
+      - pattern-either:
+          - pattern: |
+              vm.expectRevert(...);
+              $CALL
+              $CHECK
+          - pattern: |
+              vm.expectRevert(...);
+              $CALL
       - metavariable-pattern:
           metavariable: $CALL
           patterns:
             - pattern-regex: \.call\(.*\)|\.delegatecall\(.*\)
-      - focus-metavariable: $CHECK
-      - pattern-not-regex: assertTrue\(revertsAsExpected\)
+      - pattern-not-inside:
+          patterns:
+            - pattern: |
+                vm.expectRevert(...);
+                $CALL;
+                assertTrue(revertsAsExpected);
+
+  - id: sol-safety-expectrevert-no-args
+    languages: [solidity]
+    severity: ERROR
+    message: vm.expectRevert() must specify the revert reason
+    patterns:
+      - pattern: vm.expectRevert()
+    paths:
+      exclude:
+        - packages/contracts-bedrock/test
 
   - id: sol-style-input-arg-fmt
     languages: [solidity]
@@ -62,15 +80,6 @@ rules:
       exclude:
         - packages/contracts-bedrock/test/safe-tools/CompatibilityFallbackHandler_1_3_0.sol
 
-  - id: sol-expectrevert-no-args
-    languages: [solidity]
-    severity: ERROR
-    message: vm.expectRevert() must specify the revert reason
-    patterns:
-      - pattern: vm.expectRevert()
-    paths:
-      exclude:
-        - packages/contracts-bedrock/test
 
   - id: sol-style-malformed-require
     languages: [solidity]
@@ -103,6 +112,9 @@ rules:
       - pattern-not: revert $ERR(...);
       - focus-metavariable: $MSG
       - pattern-not-regex: \"(\w+:\s[^"]+)\"
+      - pattern-not-regex: string\.concat\(\"(\w+:\s[^"]+)\"\,[^"]+\)
+      - pattern-not-regex: \"([a-zA-Z0-9\s]+-[a-zA-Z0-9\s]+)\"
+      - pattern-not-regex: \"([a-zA-Z0-9\s]+-[a-zA-Z0-9\s]+-[a-zA-Z0-9\s]+)\"
     paths:
       exclude:
         - packages/contracts-bedrock/test