feat: improve built-in assertoor tests (#488)

Improves the built-in assertoor tests to make them more resilient
against non-critical failures.

static_files/assertoor-config/tests/blob-transactions-test.yaml

@@ -12,16 +12,17 @@ tasks:
     minClientCount: 1
 
 # check if all client pairs propose blocks with blob transactions
-- name: run_tasks_concurrent
+- name: run_task_background
   title: "Check if all clients propose blocks with blob transactions"
   config:
-    succeedTaskCount: 1
-    tasks:
-    - name: generate_blob_transactions
+    onBackgroundComplete: fail
+    backgroundTask:
+      name: generate_blob_transactions
       title: "Generate 2 blob transactions per block"
       config:
         childWallets: 5
-        limitPending: 2
+        walletSeed: "blob-test"
+        limitPending: 4
         limitPerBlock: 2
         randomTarget: true
         amount: 1000000
@@ -29,10 +30,10 @@ tasks:
         blobSidecars: 1
       configVars:
         privateKey: "walletPrivkey"
-
-    - name: run_task_matrix
+    foregroundTask:
+      name: run_task_matrix
       title: "Check transaction inclusion with all client pairs"
-      timeout: 1h
+      timeout: 30m
       configVars:
         matrixValues: "validatorPairNames"
       config:
@@ -40,11 +41,11 @@ tasks:
         matrixVar: "validatorPairName"
         task:
           name: check_consensus_block_proposals
-          title: "Wait for block proposal with >= 2 blobs from ${validatorPairName}"
+          title: "Wait for block proposal with >= 1 blobs from ${validatorPairName}"
           configVars:
             validatorNamePattern: "validatorPairName"
           config:
-            minBlobCount: 2
+            minBlobCount: 1
 - name: "sleep"
   title: "Wait 30sec to clear up pending transactions"
   config:
@@ -63,16 +64,17 @@ tasks:
       title: "Check if blob transactions can be sent via ${clientPairName}"
       config:
         tasks:
-        - name: run_tasks_concurrent
+        - name: run_task_background
           title: "Check if blob transactions can be sent via ${clientPairName}"
           config:
-            succeedTaskCount: 1
-            tasks:
-            - name: generate_blob_transactions
+            onBackgroundComplete: fail
+            backgroundTask:
+              name: generate_blob_transactions
               title: "Generate 2 blob transactions per block and send via ${clientPairName}"
               config:
                 childWallets: 5
-                limitPending: 2
+                walletSeed: "blob-test"
+                limitPending: 4
                 limitPerBlock: 2
                 randomTarget: true
                 amount: 1000000
@@ -81,10 +83,11 @@ tasks:
               configVars:
                 privateKey: "walletPrivkey"
                 clientPattern: "clientPairName"
-            - name: check_consensus_block_proposals
-              title: "Wait for block proposal with >= 2 blobs"
+            foregroundTask:
+              name: check_consensus_block_proposals
+              title: "Wait for block proposal with >= 1 blobs"
               config:
-                minBlobCount: 2
+                minBlobCount: 1
         - name: "sleep"
           title: "Wait 30sec to clear up pending transactions"
           config:

static_files/assertoor-config/tests/eoa-transactions-test.yaml

@@ -12,12 +12,12 @@ tasks:
     minClientCount: 1
 
 # check if all client pairs propose blocks with legacy EOA transactions
-- name: run_tasks_concurrent
+- name: run_task_background
   title: "Check if all clients propose blocks with legacy EOA transactions"
   config:
-    succeedTaskCount: 1
-    tasks:
-    - name: generate_eoa_transactions
+    onBackgroundComplete: fail
+    backgroundTask:
+      name: generate_eoa_transactions
       title: "Generate 10 EOA transaction per block"
       config:
         childWallets: 10
@@ -30,9 +30,10 @@ tasks:
       configVars:
         privateKey: "walletPrivkey"
 
-    - name: run_task_matrix
+    foregroundTask:
+      name: run_task_matrix
       title: "Check transaction inclusion with all client pairs"
-      timeout: 1h
+      timeout: 30m
       configVars:
         matrixValues: "validatorPairNames"
       config:
@@ -63,12 +64,12 @@ tasks:
       title: "Check if legacy EOA transactions can be sent via ${clientPairName}"
       config:
         tasks:
-        - name: run_tasks_concurrent
+        - name: run_task_background
           title: "Check if legacy EOA transactions can be sent via ${clientPairName}"
           config:
-            succeedTaskCount: 1
-            tasks:
-            - name: generate_eoa_transactions
+            onBackgroundComplete: fail
+            backgroundTask:
+              name: generate_eoa_transactions
               title: "Generate 10 EOA transaction per block and send via ${clientPairName}"
               config:
                 childWallets: 10
@@ -81,7 +82,9 @@ tasks:
               configVars:
                 privateKey: "walletPrivkey"
                 clientPattern: "clientPairName"
-            - name: check_consensus_block_proposals
+
+            foregroundTask:
+              name: check_consensus_block_proposals
               title: "Wait for block proposal with >= 5 transactions"
               config:
                 minTransactionCount: 5
@@ -91,12 +94,12 @@ tasks:
             duration: 30s
 
 # check if all client pairs propose blocks with dynfee EOA transactions
-- name: run_tasks_concurrent
+- name: run_task_background
   title: "Check if all clients propose blocks with dynfee EOA transactions"
   config:
-    succeedTaskCount: 1
-    tasks:
-    - name: generate_eoa_transactions
+    onBackgroundComplete: fail
+    backgroundTask:
+      name: generate_eoa_transactions
       title: "Generate 10 EOA transaction per block"
       config:
         childWallets: 10
@@ -108,9 +111,10 @@ tasks:
       configVars:
         privateKey: "walletPrivkey"
 
-    - name: run_task_matrix
+    foregroundTask:
+      name: run_task_matrix
       title: "Check transaction inclusion with all client pairs"
-      timeout: 1h
+      timeout: 30m
       configVars:
         matrixValues: "validatorPairNames"
       config:
@@ -141,12 +145,12 @@ tasks:
       title: "Check if dynfee EOA transactions can be sent via ${clientPairName}"
       config:
         tasks:
-        - name: run_tasks_concurrent
+        - name: run_task_background
           title: "Check if dynfee EOA transactions can be sent via ${clientPairName}"
           config:
-            succeedTaskCount: 1
-            tasks:
-            - name: generate_eoa_transactions
+            onBackgroundComplete: fail
+            backgroundTask:
+              name: generate_eoa_transactions
               title: "Generate 10 EOA transaction per block and send via ${clientPairName}"
               config:
                 childWallets: 10
@@ -158,7 +162,8 @@ tasks:
               configVars:
                 privateKey: "walletPrivkey"
                 clientPattern: "clientPairName"
-            - name: check_consensus_block_proposals
+            foregroundTask:
+              name: check_consensus_block_proposals
               title: "Wait for block proposal with >= 5 transactions"
               config:
                 minTransactionCount: 5

static_files/assertoor-config/tests/validator-lifecycle-test.yaml

@@ -19,6 +19,7 @@ tasks:
       title: "Generate 300 deposits"
       config:
         limitTotal: 300
+        limitPerSlot: 20
         depositContract: "0x4242424242424242424242424242424242424242"
       configVars:
         walletPrivkey: "walletPrivkey"
@@ -62,11 +63,12 @@ tasks:
     minUnfinalizedEpochs: 5
 
 # check if all client pairs propose blocks with bls changes during un-finality
-- name: run_tasks_concurrent
+- name: run_task_background
   title: "Check if all clients propose blocks with BLS changes during un-finality"
   config:
-    tasks:
-    - name: generate_bls_changes
+    onBackgroundComplete: failOrIgnore
+    backgroundTask:
+      name: generate_bls_changes
       title: "Generate 50 BLS changes (1 bls change per slot)"
       config:
         limitTotal: 50
@@ -76,7 +78,8 @@ tasks:
       configVars:
         mnemonic: "validatorMnemonic"
 
-    - name: run_task_matrix
+    foregroundTask:
+      name: run_task_matrix
       title: "Check bls change inclusion with all client pairs"
       timeout: 1h
       configVars:
@@ -93,11 +96,12 @@ tasks:
             minBlsChangeCount: 1
 
 # check if all client pairs propose blocks with exits during un-finality
-- name: run_tasks_concurrent
+- name: run_task_background
   title: "Check if all clients propose blocks with exits during un-finality"
   config:
-    tasks:
-    - name: generate_exits
+    onBackgroundComplete: failOrIgnore
+    backgroundTask:
+      name: generate_exits
       title: "Generate 50 Voluntary Exits (1 exit per slot)"
       config:
         limitTotal: 50
@@ -106,7 +110,8 @@ tasks:
       configVars:
         mnemonic: "validatorMnemonic"
 
-    - name: run_task_matrix
+    foregroundTask:
+      name: run_task_matrix
       title: "Check exit inclusion with all client pairs"
       timeout: 1h
       configVars:
@@ -123,11 +128,12 @@ tasks:
             minExitCount: 1
 
 # check if all client pairs propose blocks with attester slashings
-- name: run_tasks_concurrent
+- name: run_task_background
   title: "Check if all clients propose blocks with attester slashings during un-finality"
   config:
-    tasks:
-    - name: generate_slashings
+    onBackgroundComplete: failOrIgnore
+    backgroundTask:
+      name: generate_slashings
       title: "Generate 1 attester slashing per slot"
       config:
         slashingType: "attester"
@@ -138,7 +144,8 @@ tasks:
       configVars:
         mnemonic: "validatorMnemonic"
 
-    - name: run_task_matrix
+    foregroundTask:
+      name: run_task_matrix
       title: "Check slashing inclusion with all client pairs"
       timeout: 1h
       configVars:
@@ -155,11 +162,12 @@ tasks:
             minAttesterSlashingCount: 1
 
 # check if all client pairs propose blocks with proposer slashings
-- name: run_tasks_concurrent
+- name: run_task_background
   title: "Check if all clients propose blocks with proposer slashings during un-finality"
   config:
-    tasks:
-    - name: generate_slashings
+    onBackgroundComplete: failOrIgnore
+    backgroundTask:
+      name: generate_slashings
       title: "Generate 1 proposer slashing per slot"
       config:
         slashingType: "proposer"
@@ -170,7 +178,8 @@ tasks:
       configVars:
         mnemonic: "validatorMnemonic"
 
-    - name: run_task_matrix
+    foregroundTask:
+      name: run_task_matrix
       title: "Check slashing inclusion with all client pairs"
       timeout: 1h
       configVars:
@@ -200,11 +209,12 @@ tasks:
     maxUnfinalizedEpochs: 4
 
 # check if all client pairs propose blocks with bls changes during finality
-- name: run_tasks_concurrent
+- name: run_task_background
   title: "Check if all clients propose blocks with BLS changes during finality"
   config:
-    tasks:
-    - name: generate_bls_changes
+    onBackgroundComplete: failOrIgnore
+    backgroundTask:
+      name: generate_bls_changes
       title: "Generate 1 bls change per slot"
       config:
         startIndex: 150
@@ -215,7 +225,8 @@ tasks:
       configVars:
         mnemonic: "validatorMnemonic"
 
-    - name: run_task_matrix
+    foregroundTask:
+      name: run_task_matrix
       title: "Check bls change inclusion with all client pairs"
       timeout: 1h
       configVars:
@@ -232,11 +243,12 @@ tasks:
             minBlsChangeCount: 1
 
 # check if all client pairs propose blocks with exits during finality
-- name: run_tasks_concurrent
+- name: run_task_background
   title: "Check if all clients propose blocks with exits during finality"
   config:
-    tasks:
-    - name: generate_exits
+    onBackgroundComplete: failOrIgnore
+    backgroundTask:
+      name: generate_exits
       title: "Generate 1 exit per slot"
       config:
         startIndex: 150
@@ -246,7 +258,8 @@ tasks:
       configVars:
         mnemonic: "validatorMnemonic"
 
-    - name: run_task_matrix
+    foregroundTask:
+      name: run_task_matrix
       title: "Check exit inclusion with all client pairs"
       timeout: 1h
       configVars:
@@ -263,11 +276,12 @@ tasks:
             minExitCount: 1
 
 # check if all client pairs propose blocks with attester slashings
-- name: run_tasks_concurrent
+- name: run_task_background
   title: "Check if all clients propose blocks with attester slashings during finality"
   config:
-    tasks:
-    - name: generate_slashings
+    onBackgroundComplete: failOrIgnore
+    backgroundTask:
+      name: generate_slashings
       title: "Generate 1 attester slashing per slot"
       config:
         slashingType: "attester"
@@ -278,7 +292,8 @@ tasks:
       configVars:
         mnemonic: "validatorMnemonic"
 
-    - name: run_task_matrix
+    foregroundTask:
+      name: run_task_matrix
       title: "Check slashing inclusion with all client pairs"
       timeout: 1h
       configVars:
@@ -295,11 +310,12 @@ tasks:
             minAttesterSlashingCount: 1
 
 # check if all client pairs propose blocks with proposer slashings
-- name: run_tasks_concurrent
+- name: run_task_background
   title: "Check if all clients propose blocks with proposer slashings during finality"
   config:
-    tasks:
-    - name: generate_slashings
+    onBackgroundComplete: failOrIgnore
+    backgroundTask:
+      name: generate_slashings
       title: "Generate 1 proposer slashing per slot"
       config:
         slashingType: "proposer"
@@ -310,7 +326,8 @@ tasks:
       configVars:
         mnemonic: "validatorMnemonic"
 
-    - name: run_task_matrix
+    foregroundTask:
+      name: run_task_matrix
       title: "Check slashing inclusion with all client pairs"
       timeout: 1h
       configVars: