Skip to content

E
ethereum-package
Commit d5966991 authored by pk910's avatar pk910 Committed by GitHub
Browse files
Options

feat: improve built-in assertoor tests (#488) 

Improves the built-in assertoor tests to make them more resilient
against non-critical failures.
parent b3418cf1
Expand all Hide whitespace changes
Inline Side-by-side
Showing with 308 additions and 73 deletions
static_files/assertoor-config/tests/blob-transactions-test.yaml
View file @ d5966991
...@@ -12,16 +12,17 @@ tasks: ...@@ -12,16 +12,17 @@ tasks:
minClientCount: 1 minClientCount: 1
# check if all client pairs propose blocks with blob transactions # check if all client pairs propose blocks with blob transactions
- name: run_tasks_concurrent - name: run_task_background
title: "Check if all clients propose blocks with blob transactions" title: "Check if all clients propose blocks with blob transactions"
config: config:
succeedTaskCount: 1 onBackgroundComplete: fail
tasks: backgroundTask:
- name: generate_blob_transactions name: generate_blob_transactions
title: "Generate 2 blob transactions per block" title: "Generate 2 blob transactions per block"
config: config:
childWallets: 5 childWallets: 5
limitPending: 2 walletSeed: "blob-test"
limitPending: 4
limitPerBlock: 2 limitPerBlock: 2
randomTarget: true randomTarget: true
amount: 1000000 amount: 1000000
...@@ -29,10 +30,10 @@ tasks: ...@@ -29,10 +30,10 @@ tasks:
blobSidecars: 1 blobSidecars: 1
configVars: configVars:
privateKey: "walletPrivkey" privateKey: "walletPrivkey"
foregroundTask:
- name: run_task_matrix name: run_task_matrix
title: "Check transaction inclusion with all client pairs" title: "Check transaction inclusion with all client pairs"
timeout: 1h timeout: 30m
configVars: configVars:
matrixValues: "validatorPairNames" matrixValues: "validatorPairNames"
config: config:
...@@ -40,11 +41,11 @@ tasks: ...@@ -40,11 +41,11 @@ tasks:
matrixVar: "validatorPairName" matrixVar: "validatorPairName"
task: task:
name: check_consensus_block_proposals name: check_consensus_block_proposals
title: "Wait for block proposal with >= 2 blobs from ${validatorPairName}" title: "Wait for block proposal with >= 1 blobs from ${validatorPairName}"
configVars: configVars:
validatorNamePattern: "validatorPairName" validatorNamePattern: "validatorPairName"
config: config:
minBlobCount: 2 minBlobCount: 1
- name: "sleep" - name: "sleep"
title: "Wait 30sec to clear up pending transactions" title: "Wait 30sec to clear up pending transactions"
config: config:
...@@ -63,16 +64,17 @@ tasks: ...@@ -63,16 +64,17 @@ tasks:
title: "Check if blob transactions can be sent via ${clientPairName}" title: "Check if blob transactions can be sent via ${clientPairName}"
config: config:
tasks: tasks:
- name: run_tasks_concurrent - name: run_task_background
title: "Check if blob transactions can be sent via ${clientPairName}" title: "Check if blob transactions can be sent via ${clientPairName}"
config: config:
succeedTaskCount: 1 onBackgroundComplete: fail
tasks: backgroundTask:
- name: generate_blob_transactions name: generate_blob_transactions
title: "Generate 2 blob transactions per block and send via ${clientPairName}" title: "Generate 2 blob transactions per block and send via ${clientPairName}"
config: config:
childWallets: 5 childWallets: 5
limitPending: 2 walletSeed: "blob-test"
limitPending: 4
limitPerBlock: 2 limitPerBlock: 2
randomTarget: true randomTarget: true
amount: 1000000 amount: 1000000
...@@ -81,10 +83,11 @@ tasks: ...@@ -81,10 +83,11 @@ tasks:
configVars: configVars:
privateKey: "walletPrivkey" privateKey: "walletPrivkey"
clientPattern: "clientPairName" clientPattern: "clientPairName"
- name: check_consensus_block_proposals foregroundTask:
title: "Wait for block proposal with >= 2 blobs" name: check_consensus_block_proposals
title: "Wait for block proposal with >= 1 blobs"
config: config:
minBlobCount: 2 minBlobCount: 1
- name: "sleep" - name: "sleep"
title: "Wait 30sec to clear up pending transactions" title: "Wait 30sec to clear up pending transactions"
config: config:
......
static_files/assertoor-config/tests/eoa-transactions-test.yaml
View file @ d5966991
...@@ -12,12 +12,12 @@ tasks: ...@@ -12,12 +12,12 @@ tasks:
minClientCount: 1 minClientCount: 1
# check if all client pairs propose blocks with legacy EOA transactions # check if all client pairs propose blocks with legacy EOA transactions
- name: run_tasks_concurrent - name: run_task_background
title: "Check if all clients propose blocks with legacy EOA transactions" title: "Check if all clients propose blocks with legacy EOA transactions"
config: config:
succeedTaskCount: 1 onBackgroundComplete: fail
tasks: backgroundTask:
- name: generate_eoa_transactions name: generate_eoa_transactions
title: "Generate 10 EOA transaction per block" title: "Generate 10 EOA transaction per block"
config: config:
childWallets: 10 childWallets: 10
...@@ -30,9 +30,10 @@ tasks: ...@@ -30,9 +30,10 @@ tasks:
configVars: configVars:
privateKey: "walletPrivkey" privateKey: "walletPrivkey"
- name: run_task_matrix foregroundTask:
name: run_task_matrix
title: "Check transaction inclusion with all client pairs" title: "Check transaction inclusion with all client pairs"
timeout: 1h timeout: 30m
configVars: configVars:
matrixValues: "validatorPairNames" matrixValues: "validatorPairNames"
config: config:
...@@ -63,12 +64,12 @@ tasks: ...@@ -63,12 +64,12 @@ tasks:
title: "Check if legacy EOA transactions can be sent via ${clientPairName}" title: "Check if legacy EOA transactions can be sent via ${clientPairName}"
config: config:
tasks: tasks:
- name: run_tasks_concurrent - name: run_task_background
title: "Check if legacy EOA transactions can be sent via ${clientPairName}" title: "Check if legacy EOA transactions can be sent via ${clientPairName}"
config: config:
succeedTaskCount: 1 onBackgroundComplete: fail
tasks: backgroundTask:
- name: generate_eoa_transactions name: generate_eoa_transactions
title: "Generate 10 EOA transaction per block and send via ${clientPairName}" title: "Generate 10 EOA transaction per block and send via ${clientPairName}"
config: config:
childWallets: 10 childWallets: 10
...@@ -81,7 +82,9 @@ tasks: ...@@ -81,7 +82,9 @@ tasks:
configVars: configVars:
privateKey: "walletPrivkey" privateKey: "walletPrivkey"
clientPattern: "clientPairName" clientPattern: "clientPairName"
- name: check_consensus_block_proposals
foregroundTask:
name: check_consensus_block_proposals
title: "Wait for block proposal with >= 5 transactions" title: "Wait for block proposal with >= 5 transactions"
config: config:
minTransactionCount: 5 minTransactionCount: 5
...@@ -91,12 +94,12 @@ tasks: ...@@ -91,12 +94,12 @@ tasks:
duration: 30s duration: 30s
# check if all client pairs propose blocks with dynfee EOA transactions # check if all client pairs propose blocks with dynfee EOA transactions
- name: run_tasks_concurrent - name: run_task_background
title: "Check if all clients propose blocks with dynfee EOA transactions" title: "Check if all clients propose blocks with dynfee EOA transactions"
config: config:
succeedTaskCount: 1 onBackgroundComplete: fail
tasks: backgroundTask:
- name: generate_eoa_transactions name: generate_eoa_transactions
title: "Generate 10 EOA transaction per block" title: "Generate 10 EOA transaction per block"
config: config:
childWallets: 10 childWallets: 10
...@@ -108,9 +111,10 @@ tasks: ...@@ -108,9 +111,10 @@ tasks:
configVars: configVars:
privateKey: "walletPrivkey" privateKey: "walletPrivkey"
- name: run_task_matrix foregroundTask:
name: run_task_matrix
title: "Check transaction inclusion with all client pairs" title: "Check transaction inclusion with all client pairs"
timeout: 1h timeout: 30m
configVars: configVars:
matrixValues: "validatorPairNames" matrixValues: "validatorPairNames"
config: config:
...@@ -141,12 +145,12 @@ tasks: ...@@ -141,12 +145,12 @@ tasks:
title: "Check if dynfee EOA transactions can be sent via ${clientPairName}" title: "Check if dynfee EOA transactions can be sent via ${clientPairName}"
config: config:
tasks: tasks:
- name: run_tasks_concurrent - name: run_task_background
title: "Check if dynfee EOA transactions can be sent via ${clientPairName}" title: "Check if dynfee EOA transactions can be sent via ${clientPairName}"
config: config:
succeedTaskCount: 1 onBackgroundComplete: fail
tasks: backgroundTask:
- name: generate_eoa_transactions name: generate_eoa_transactions
title: "Generate 10 EOA transaction per block and send via ${clientPairName}" title: "Generate 10 EOA transaction per block and send via ${clientPairName}"
config: config:
childWallets: 10 childWallets: 10
...@@ -158,7 +162,8 @@ tasks: ...@@ -158,7 +162,8 @@ tasks:
configVars: configVars:
privateKey: "walletPrivkey" privateKey: "walletPrivkey"
clientPattern: "clientPairName" clientPattern: "clientPairName"
- name: check_consensus_block_proposals foregroundTask:
name: check_consensus_block_proposals
title: "Wait for block proposal with >= 5 transactions" title: "Wait for block proposal with >= 5 transactions"
config: config:
minTransactionCount: 5 minTransactionCount: 5
......
static_files/assertoor-config/tests/validator-lifecycle-test.yaml
View file @ d5966991
...@@ -19,6 +19,7 @@ tasks: ...@@ -19,6 +19,7 @@ tasks:
title: "Generate 300 deposits" title: "Generate 300 deposits"
config: config:
limitTotal: 300 limitTotal: 300
limitPerSlot: 20
depositContract: "0x4242424242424242424242424242424242424242" depositContract: "0x4242424242424242424242424242424242424242"
configVars: configVars:
walletPrivkey: "walletPrivkey" walletPrivkey: "walletPrivkey"
...@@ -62,11 +63,12 @@ tasks: ...@@ -62,11 +63,12 @@ tasks:
minUnfinalizedEpochs: 5 minUnfinalizedEpochs: 5
# check if all client pairs propose blocks with bls changes during un-finality # check if all client pairs propose blocks with bls changes during un-finality
- name: run_tasks_concurrent - name: run_task_background
title: "Check if all clients propose blocks with BLS changes during un-finality" title: "Check if all clients propose blocks with BLS changes during un-finality"
config: config:
tasks: onBackgroundComplete: failOrIgnore
- name: generate_bls_changes backgroundTask:
name: generate_bls_changes
title: "Generate 50 BLS changes (1 bls change per slot)" title: "Generate 50 BLS changes (1 bls change per slot)"
config: config:
limitTotal: 50 limitTotal: 50
...@@ -76,7 +78,8 @@ tasks: ...@@ -76,7 +78,8 @@ tasks:
configVars: configVars:
mnemonic: "validatorMnemonic" mnemonic: "validatorMnemonic"
- name: run_task_matrix foregroundTask:
name: run_task_matrix
title: "Check bls change inclusion with all client pairs" title: "Check bls change inclusion with all client pairs"
timeout: 1h timeout: 1h
configVars: configVars:
...@@ -93,11 +96,12 @@ tasks: ...@@ -93,11 +96,12 @@ tasks:
minBlsChangeCount: 1 minBlsChangeCount: 1
# check if all client pairs propose blocks with exits during un-finality # check if all client pairs propose blocks with exits during un-finality
- name: run_tasks_concurrent - name: run_task_background
title: "Check if all clients propose blocks with exits during un-finality" title: "Check if all clients propose blocks with exits during un-finality"
config: config:
tasks: onBackgroundComplete: failOrIgnore
- name: generate_exits backgroundTask:
name: generate_exits
title: "Generate 50 Voluntary Exits (1 exit per slot)" title: "Generate 50 Voluntary Exits (1 exit per slot)"
config: config:
limitTotal: 50 limitTotal: 50
...@@ -106,7 +110,8 @@ tasks: ...@@ -106,7 +110,8 @@ tasks:
configVars: configVars:
mnemonic: "validatorMnemonic" mnemonic: "validatorMnemonic"
- name: run_task_matrix foregroundTask:
name: run_task_matrix
title: "Check exit inclusion with all client pairs" title: "Check exit inclusion with all client pairs"
timeout: 1h timeout: 1h
configVars: configVars:
...@@ -123,11 +128,12 @@ tasks: ...@@ -123,11 +128,12 @@ tasks:
minExitCount: 1 minExitCount: 1
# check if all client pairs propose blocks with attester slashings # check if all client pairs propose blocks with attester slashings
- name: run_tasks_concurrent - name: run_task_background
title: "Check if all clients propose blocks with attester slashings during un-finality" title: "Check if all clients propose blocks with attester slashings during un-finality"
config: config:
tasks: onBackgroundComplete: failOrIgnore
- name: generate_slashings backgroundTask:
name: generate_slashings
title: "Generate 1 attester slashing per slot" title: "Generate 1 attester slashing per slot"
config: config:
slashingType: "attester" slashingType: "attester"
...@@ -138,7 +144,8 @@ tasks: ...@@ -138,7 +144,8 @@ tasks:
configVars: configVars:
mnemonic: "validatorMnemonic" mnemonic: "validatorMnemonic"
- name: run_task_matrix foregroundTask:
name: run_task_matrix
title: "Check slashing inclusion with all client pairs" title: "Check slashing inclusion with all client pairs"
timeout: 1h timeout: 1h
configVars: configVars:
...@@ -155,11 +162,12 @@ tasks: ...@@ -155,11 +162,12 @@ tasks:
minAttesterSlashingCount: 1 minAttesterSlashingCount: 1
# check if all client pairs propose blocks with proposer slashings # check if all client pairs propose blocks with proposer slashings
- name: run_tasks_concurrent - name: run_task_background
title: "Check if all clients propose blocks with proposer slashings during un-finality" title: "Check if all clients propose blocks with proposer slashings during un-finality"
config: config:
tasks: onBackgroundComplete: failOrIgnore
- name: generate_slashings backgroundTask:
name: generate_slashings
title: "Generate 1 proposer slashing per slot" title: "Generate 1 proposer slashing per slot"
config: config:
slashingType: "proposer" slashingType: "proposer"
...@@ -170,7 +178,8 @@ tasks: ...@@ -170,7 +178,8 @@ tasks:
configVars: configVars:
mnemonic: "validatorMnemonic" mnemonic: "validatorMnemonic"
- name: run_task_matrix foregroundTask:
name: run_task_matrix
title: "Check slashing inclusion with all client pairs" title: "Check slashing inclusion with all client pairs"
timeout: 1h timeout: 1h
configVars: configVars:
...@@ -200,11 +209,12 @@ tasks: ...@@ -200,11 +209,12 @@ tasks:
maxUnfinalizedEpochs: 4 maxUnfinalizedEpochs: 4
# check if all client pairs propose blocks with bls changes during finality # check if all client pairs propose blocks with bls changes during finality
- name: run_tasks_concurrent - name: run_task_background
title: "Check if all clients propose blocks with BLS changes during finality" title: "Check if all clients propose blocks with BLS changes during finality"
config: config:
tasks: onBackgroundComplete: failOrIgnore
- name: generate_bls_changes backgroundTask:
name: generate_bls_changes
title: "Generate 1 bls change per slot" title: "Generate 1 bls change per slot"
config: config:
startIndex: 150 startIndex: 150
...@@ -215,7 +225,8 @@ tasks: ...@@ -215,7 +225,8 @@ tasks:
configVars: configVars:
mnemonic: "validatorMnemonic" mnemonic: "validatorMnemonic"
- name: run_task_matrix foregroundTask:
name: run_task_matrix
title: "Check bls change inclusion with all client pairs" title: "Check bls change inclusion with all client pairs"
timeout: 1h timeout: 1h
configVars: configVars:
...@@ -232,11 +243,12 @@ tasks: ...@@ -232,11 +243,12 @@ tasks:
minBlsChangeCount: 1 minBlsChangeCount: 1
# check if all client pairs propose blocks with exits during finality # check if all client pairs propose blocks with exits during finality
- name: run_tasks_concurrent - name: run_task_background
title: "Check if all clients propose blocks with exits during finality" title: "Check if all clients propose blocks with exits during finality"
config: config:
tasks: onBackgroundComplete: failOrIgnore
- name: generate_exits backgroundTask:
name: generate_exits
title: "Generate 1 exit per slot" title: "Generate 1 exit per slot"
config: config:
startIndex: 150 startIndex: 150
...@@ -246,7 +258,8 @@ tasks: ...@@ -246,7 +258,8 @@ tasks:
configVars: configVars:
mnemonic: "validatorMnemonic" mnemonic: "validatorMnemonic"
- name: run_task_matrix foregroundTask:
name: run_task_matrix
title: "Check exit inclusion with all client pairs" title: "Check exit inclusion with all client pairs"
timeout: 1h timeout: 1h
configVars: configVars:
...@@ -263,11 +276,12 @@ tasks: ...@@ -263,11 +276,12 @@ tasks:
minExitCount: 1 minExitCount: 1
# check if all client pairs propose blocks with attester slashings # check if all client pairs propose blocks with attester slashings
- name: run_tasks_concurrent - name: run_task_background
title: "Check if all clients propose blocks with attester slashings during finality" title: "Check if all clients propose blocks with attester slashings during finality"
config: config:
tasks: onBackgroundComplete: failOrIgnore
- name: generate_slashings backgroundTask:
name: generate_slashings
title: "Generate 1 attester slashing per slot" title: "Generate 1 attester slashing per slot"
config: config:
slashingType: "attester" slashingType: "attester"
...@@ -278,7 +292,8 @@ tasks: ...@@ -278,7 +292,8 @@ tasks:
configVars: configVars:
mnemonic: "validatorMnemonic" mnemonic: "validatorMnemonic"
- name: run_task_matrix foregroundTask:
name: run_task_matrix
title: "Check slashing inclusion with all client pairs" title: "Check slashing inclusion with all client pairs"
timeout: 1h timeout: 1h
configVars: configVars:
...@@ -295,11 +310,12 @@ tasks: ...@@ -295,11 +310,12 @@ tasks:
minAttesterSlashingCount: 1 minAttesterSlashingCount: 1
# check if all client pairs propose blocks with proposer slashings # check if all client pairs propose blocks with proposer slashings
- name: run_tasks_concurrent - name: run_task_background
title: "Check if all clients propose blocks with proposer slashings during finality" title: "Check if all clients propose blocks with proposer slashings during finality"
config: config:
tasks: onBackgroundComplete: failOrIgnore
- name: generate_slashings backgroundTask:
name: generate_slashings
title: "Generate 1 proposer slashing per slot" title: "Generate 1 proposer slashing per slot"
config: config:
slashingType: "proposer" slashingType: "proposer"
...@@ -310,7 +326,8 @@ tasks: ...@@ -310,7 +326,8 @@ tasks:
configVars: configVars:
mnemonic: "validatorMnemonic" mnemonic: "validatorMnemonic"
- name: run_task_matrix foregroundTask:
name: run_task_matrix
title: "Check slashing inclusion with all client pairs" title: "Check slashing inclusion with all client pairs"
timeout: 1h timeout: 1h
configVars: configVars:
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment