ci: added k8s test (#294)

.circleci/config.yml

@@ -6,7 +6,7 @@ orbs:
 executors:
   ubuntu_vm:
     machine:
-      image: ubuntu-2004:202201-02
+      image: ubuntu-2204:2023.07.2
 
 parameters:
   should-enable-check-latest-version-workflow:
@@ -16,6 +16,10 @@ parameters:
   should-enable-build-workflow:
     type: boolean
     default: true
+  kurtosis-cluster-setting-abs-filepath:
+    type: string
+    default: "/home/circleci/.local/share/kurtosis/cluster-setting"
+
 
 # Install go version 1.19 to run our integration tests on the minimal version we support
 setup_kurtosis: &setup_kurtosis
@@ -26,6 +30,57 @@ setup_kurtosis: &setup_kurtosis
       kurtosis analytics disable
       kurtosis engine restart
 
+# Steps to prepare a job for Kubernetes testing (with K3S K8S distribution)
+setup_kurtosis_k3s: &setup_kurtosis_k3s
+  pre-steps:
+    - run:
+        name: Install Kurtosis
+        command: |
+          echo "deb [trusted=yes] https://apt.fury.io/kurtosis-tech/ /" | sudo tee /etc/apt/sources.list.d/kurtosis.list
+          sudo apt update
+          sudo apt install kurtosis-cli
+          kurtosis analytics disable
+    - run:
+        name: Install K3D and create the K3D/K3S cluster on Docker
+        command: |
+          curl -s https://raw.githubusercontent.com/k3d-io/k3d/main/install.sh | bash
+          k3d cluster create --servers 1 --no-lb --wait --verbose
+    # Edit the Kurtosis config file in order to have K3S cluster configuration
+    - run:
+        name: Add K3S cluster config in Kurtosis config file
+        command: |
+          KURTOSIS_CONFIG_FILEPATH=$(kurtosis config path)
+          cat \<< EOF > "$KURTOSIS_CONFIG_FILEPATH"
+          config-version: 2
+          should-send-metrics: true
+          kurtosis-clusters:
+            docker:
+              type: docker
+            k3d-k3s-default:
+              type: kubernetes
+              config:
+                kubernetes-cluster-name: k3d-k3s-default
+                storage-class: standard
+                enclave-size-in-megabytes: 2048
+          EOF
+    # Set the K3S cluster with some previous steps to force this cluster type in the cluster-setting file. This save us to start the engine with the cluster set command
+    # because we are going to start it on the next step
+    - run:
+        name: Set K3S Kurtosis cluster
+        command: |
+          KURTOSIS_CLUSTER_SETTING_FILEPATH="<< pipeline.parameters.kurtosis-cluster-setting-abs-filepath >>"
+          echo -n "k3d-k3s-default" > "$KURTOSIS_CLUSTER_SETTING_FILEPATH"
+          echo "Kurtosis cluster-setting file content: $(cat $KURTOSIS_CLUSTER_SETTING_FILEPATH)"
+          kurtosis cluster set k3d-k3s-default --cli-log-level trace
+    - run:
+        name: Start Kurtosis engine in K8s backend
+        command: |
+          kurtosis engine start --enclave-pool-size 2
+    - run:
+        name: Run Kurtosis gateway
+        command: "kurtosis gateway"
+        background: true
+
 
 # NOTE: Because CircleCI jobs run on separate machines from each other, we duplicate steps (like checkout) between jobs. This is because doing the "correct" DRY
 #  refactoring of, "one job for checkout, one job for build Docker image, etc." would require a) persisting files between jobs and b) persisting Docker images between
@@ -170,6 +225,12 @@ jobs:
       - checkout
       - run: kurtosis run ${PWD} "$(cat ./.circleci/tests/mev.json)"
 
+  mev_k8s:
+    executor: ubuntu_vm
+    steps:
+      - checkout
+      - run: kurtosis run ${PWD} "$(cat ./.circleci/tests/mev.json)"
+
   nimbus_mev:
     executor: ubuntu_vm
     steps:
@@ -177,6 +238,19 @@ jobs:
       - checkout
       - run: kurtosis run ${PWD} "$(cat ./.circleci/tests/nimbus-mev.json)"
 
+  mix_with_tools_k8s:
+    executor: ubuntu_vm
+    steps:
+      - checkout
+      - run: kurtosis run ${PWD} "$(cat ./.circleci/tests/mix-with-tools.json)"
+
+  mix_with_tools_docker:
+    executor: ubuntu_vm
+    steps:
+      - <<: *setup_kurtosis
+      - checkout
+      - run: kurtosis run ${PWD} "$(cat ./.circleci/tests/mix-with-tools.json)"
+
 workflows:
   check_latest_version:
     when: << pipeline.parameters.should-enable-check-latest-version-workflow >>
@@ -195,6 +269,11 @@ workflows:
       - reth-all
       - teku-all
       - nimbus_mev
+      - mev_k8s:
+          <<: *setup_kurtosis_k3s
+      - mix_with_tools_k8s:
+          <<: *setup_kurtosis_k3s
+      - mix_with_tools_docker
 
   build:
     when: << pipeline.parameters.should-enable-build-workflow >>
@@ -234,3 +313,10 @@ workflows:
             branches:
               ignore:
                 - main
+
+      - mev_k8s:
+          <<: *setup_kurtosis_k3s
+          filters:
+            branches:
+              ignore:
+                - main

.circleci/tests/mix-with-tools.json

+{
+    "participants": [
+      {
+        "el_client_type": "geth",
+        "cl_client_type": "teku"
+      },
+      {
+        "el_client_type": "nethermind",
+        "cl_client_type": "prysm"
+      },
+      {
+        "el_client_type": "erigon",
+        "cl_client_type": "nimbus"
+      },
+      {
+        "el_client_type": "besu",
+        "cl_client_type": "lighthouse"
+      },
+      {
+        "el_client_type": "reth",
+        "cl_client_type": "lodestar"
+      },
+      {
+        "el_client_type": "ethereumjs",
+        "cl_client_type": "teku"
+      }
+    ],
+    "additional_services": [
+      "tx_spammer",
+      "blob_spammer",
+      "el_forkmon",
+      "beacon_metrics_gazer",
+      "dora",
+      "prometheus_grafana",
+      "goomy_blob",
+      "full_beaconchain_explorer",
+      "custom_flood"
+    ],
+    "mev_type": "full"
+  }
+  
\ No newline at end of file