feat: enable shadowforking (#475)

Co-authored-by: pk910 <github@pk910.de>
Co-authored-by: parithosh <parithosh.jayanthi@ethereum.org>

.github/tests/holesky-shadowfork-verkle.yaml_norun

+participants:
+  - el_client_type: geth
+    el_client_image: ethpandaops/geth:transition-post-genesis-04b0304
+    cl_client_type: lighthouse
+    cl_client_image: ethpandaops/lighthouse:verkle-trees-capella-2ffb8a9
+  - el_client_type: geth
+    el_client_image: ethpandaops/geth:transition-post-genesis-04b0304
+    cl_client_type: lodestar
+    cl_client_image: ethpandaops/lodestar:g11tech-verge-815364b
+network_params:
+  electra_fork_epoch: 1
+  network: holesky-shadowfork-verkle
+  genesis_delay: 300
+additional_services:
+  - dora
+snooper_enabled: true
+persistent: true

.github/tests/holesky-shadowfork.yaml_norun

+participants:
+  - el_client_type: geth
+    el_client_image: ethereum/client-go:v1.13.11
+    cl_client_type: teku
+    cl_client_image: consensys/teku:24.1.1
+network_params:
+  dencun_fork_epoch: 1
+  network: holesky-shadowfork
+additional_services:
+  - dora
+snooper_enabled: true
+persistent: true

.github/tests/verkle-gen-devnet-3.yaml → .github/tests/verkle-gen-devnet-4.yaml

 participants:
   - el_client_type: geth
-    el_client_image: ethpandaops/geth:kaustinen-with-shapella-6d7b22c
+    el_client_image: ethpandaops/geth:kaustinen-with-shapella-0b110bd
     cl_client_type: lighthouse
     cl_client_image: ethpandaops/lighthouse:verkle-trees-capella-2ffb8a9
     count: 2
   - el_client_type: geth
-    el_client_image: ethpandaops/geth:kaustinen-with-shapella-6d7b22c
+    el_client_image: ethpandaops/geth:kaustinen-with-shapella-0b110bd
     cl_client_type: lodestar
     cl_client_image: ethpandaops/lodestar:g11tech-verge-815364b
 network_params:
-  network: verkle-gen-devnet-3
+  network: verkle-gen-devnet-4
 

.github/tests/verkle-gen.yaml

 participants:
   - el_client_type: geth
-    el_client_image: ethpandaops/geth:kaustinen-with-shapella-6d7b22c
+    el_client_image: ethpandaops/geth:kaustinen-with-shapella-0b110bd
     cl_client_type: lighthouse
     cl_client_image: ethpandaops/lighthouse:verkle-trees-capella-2ffb8a9
     count: 2
   - el_client_type: geth
-    el_client_image: ethpandaops/geth:kaustinen-with-shapella-6d7b22c
+    el_client_image: ethpandaops/geth:kaustinen-with-shapella-0b110bd
     cl_client_type: lodestar
     cl_client_image: ethpandaops/lodestar:g11tech-verge-815364b
     count: 2

.github/tests/verkle.yaml

 participants:
   - el_client_type: geth
-    el_client_image: ethpandaops/geth:transition-post-genesis-1d80ebd
+    el_client_image: ethpandaops/geth:transition-post-genesis-04b0304
     cl_client_type: lighthouse
     cl_client_image: ethpandaops/lighthouse:verkle-trees-capella-2ffb8a9
     count: 2
   - el_client_type: geth
-    el_client_image: ethpandaops/geth:transition-post-genesis-1d80ebd
+    el_client_image: ethpandaops/geth:transition-post-genesis-04b0304
     cl_client_type: lodestar
     cl_client_image: ethpandaops/lodestar:g11tech-verge-815364b
 network_params:

README.md

@@ -64,6 +64,27 @@ To mitigate these issues, you can use the `el_client_volume_size` and `cl_client
 
 For optimal performance, we recommend using a cloud provider that allows you to provision Kubernetes clusters with fast persistent storage or self hosting your own Kubernetes cluster with fast persistent storage.
 
+### Shadowforking
+In order to enable shadowfork capabilities, you can use the `network_params.network` flag. The expected value is the name of the network you want to shadowfork followed by `-shadowfork`. Please note that `persistent` configuration parameter has to be enabled for shadowforks to work! Current limitation on k8s is it is only working on a single node cluster. For example, to shadowfork the Holesky testnet, you can use the following command:
+```yaml
+...
+network_params:
+  network: "holesky-shadowfork"
+persistent: true
+...
+```
+
+##### Shadowforking custom verkle networks
+In order to enable shadowfork capabilities for verkle networks, you need to define electra and mention verkle in the network name after shadowfork.
+```yaml
+...
+network_params:
+  electra_fork_epoch: 1
+  network: "holesky-shadowfork-verkle"
+persistent: true
+...
+```
+
 #### Taints and tolerations
 It is possible to run the package on a Kubernetes cluster with taints and tolerations. This is done by adding the tolerations to the `tolerations` field in the `network_params.yaml` file. For example:
 ```yaml
@@ -218,7 +239,7 @@ participants:
   #   effect: "NoSchedule"
   #   toleration_seconds: 3600
   # Defaults to empty
-  el_tolerations: []
+  cl_tolerations: []
 
   # A list of tolerations that will be passed to the validator container
   # Only works with Kubernetes
@@ -362,7 +383,7 @@ network_params:
   # Defaults to 2048
   eth1_follow_distance: 2048
 
-  # The epoch at which the capella and deneb forks are set to occur.
+  # The epoch at which the capella/deneb/electra forks are set to occur.
   capella_fork_epoch: 0
   deneb_fork_epoch: 500
   electra_fork_epoch: null
@@ -373,6 +394,14 @@ network_params:
   # You can sync any devnet by setting this to the network name (e.g. "dencun-devnet-12", "verkle-gen-devnet-2")
   network: "kurtosis"
 
+  # The number of epochs to wait validators to be able to withdraw
+  # Defaults to 256 epochs ~27 hours
+  min_validator_withdrawability_delay: 256
+
+  # The period of the shard committee
+  # Defaults to 256 epoch ~27 hours
+  shard_committee_period: 256
+
 # Configuration place for transaction spammer - https:#github.com/MariusVanDerWijden/tx-fuzz
 tx_spammer_params:
   # A list of optional extra params that will be passed to the TX Spammer container for modifying its behaviour

network_params.yaml

@@ -4,9 +4,13 @@ participants:
     el_client_log_level: ""
     el_extra_params: []
     el_extra_labels: {}
+    el_tolerations: []
     cl_client_type: lighthouse
     cl_client_image: sigp/lighthouse:latest
     cl_client_log_level: ""
+    cl_tolerations: []
+    validator_tolerations: []
+    tolerations: []
     beacon_extra_params: []
     beacon_extra_labels: {}
     validator_extra_params: []
@@ -50,6 +54,10 @@ network_params:
   capella_fork_epoch: 0
   deneb_fork_epoch: 4
   electra_fork_epoch: null
+  network: kurtosis
+  min_validator_withdrawability_delay: 256
+  shard_committee_period: 256
+
 additional_services:
   - tx_spammer
   - blob_spammer
@@ -78,3 +86,4 @@ mev_params:
 grafana_additional_dashboards: []
 persistent: false
 xatu_sentry_enabled: false
+global_tolerations: []

src/cl/lighthouse/lighthouse_launcher.star

@@ -138,13 +138,7 @@ def launch(
         cl_tolerations, participant_tolerations, global_tolerations
     )
 
-    network_name = (
-        "devnets"
-        if launcher.network != "kurtosis"
-        and launcher.network != "ephemery"
-        and launcher.network not in constants.PUBLIC_NETWORKS
-        else launcher.network
-    )
+    network_name = shared_utils.get_network_name(launcher.network)
 
     bn_min_cpu = int(bn_min_cpu) if int(bn_min_cpu) > 0 else BEACON_MIN_CPU
     bn_max_cpu = (
@@ -381,7 +375,10 @@ def get_beacon_config(
 
     if network not in constants.PUBLIC_NETWORKS:
         cmd.append("--testnet-dir=" + constants.GENESIS_CONFIG_MOUNT_PATH_ON_CONTAINER)
-        if network == constants.NETWORK_NAME.kurtosis:
+        if (
+            network == constants.NETWORK_NAME.kurtosis
+            or constants.NETWORK_NAME.shadowfork in network
+        ):
             if boot_cl_client_ctxs != None:
                 cmd.append(
                     "--boot-nodes="

src/cl/lodestar/lodestar_launcher.star

@@ -116,13 +116,7 @@ def launch(
         cl_tolerations, participant_tolerations, global_tolerations
     )
 
-    network_name = (
-        "devnets"
-        if launcher.network != "kurtosis"
-        and launcher.network != "ephemery"
-        and launcher.network not in constants.PUBLIC_NETWORKS
-        else launcher.network
-    )
+    network_name = shared_utils.get_network_name(launcher.network)
 
     bn_min_cpu = int(bn_min_cpu) if int(bn_min_cpu) > 0 else BEACON_MIN_CPU
     bn_max_cpu = (
@@ -349,7 +343,10 @@ def get_beacon_config(
             + constants.GENESIS_CONFIG_MOUNT_PATH_ON_CONTAINER
             + "/genesis.ssz"
         )
-        if network == constants.NETWORK_NAME.kurtosis:
+        if (
+            network == constants.NETWORK_NAME.kurtosis
+            or constants.NETWORK_NAME.shadowfork in network
+        ):
             if bootnode_contexts != None:
                 cmd.append(
                     "--bootnodes="

src/cl/nimbus/nimbus_launcher.star

@@ -152,13 +152,7 @@ def launch(
         cl_tolerations, participant_tolerations, global_tolerations
     )
 
-    network_name = (
-        "devnets"
-        if launcher.network != "kurtosis"
-        and launcher.network != "ephemery"
-        and launcher.network not in constants.PUBLIC_NETWORKS
-        else launcher.network
-    )
+    network_name = shared_utils.get_network_name(launcher.network)
 
     bn_min_cpu = int(bn_min_cpu) if int(bn_min_cpu) > 0 else BEACON_MIN_CPU
     bn_max_cpu = (
@@ -394,7 +388,10 @@ def get_beacon_config(
             + constants.GENESIS_CONFIG_MOUNT_PATH_ON_CONTAINER
             + "/bootstrap_nodes.txt"
         )
-        if network == constants.NETWORK_NAME.kurtosis:
+        if (
+            network == constants.NETWORK_NAME.kurtosis
+            or constants.NETWORK_NAME.shadowfork in network
+        ):
             if bootnode_contexts == None:
                 cmd.append("--subscribe-all-subnets")
             else:

src/cl/prysm/prysm_launcher.star

@@ -143,13 +143,7 @@ def launch(
         cl_tolerations, participant_tolerations, global_tolerations
     )
 
-    network_name = (
-        "devnets"
-        if launcher.network != "kurtosis"
-        and launcher.network != "ephemery"
-        and launcher.network not in constants.PUBLIC_NETWORKS
-        else launcher.network
-    )
+    network_name = shared_utils.get_network_name(launcher.network)
 
     bn_min_cpu = int(bn_min_cpu) if int(bn_min_cpu) > 0 else BEACON_MIN_CPU
     bn_max_cpu = (
@@ -360,7 +354,10 @@ def get_beacon_config(
             + constants.GENESIS_CONFIG_MOUNT_PATH_ON_CONTAINER
             + "/genesis.ssz",
         )
-        if network == constants.NETWORK_NAME.kurtosis:
+        if (
+            network == constants.NETWORK_NAME.kurtosis
+            or constants.NETWORK_NAME.shadowfork in network
+        ):
             if bootnode_contexts != None:
                 for ctx in bootnode_contexts[: constants.MAX_ENR_ENTRIES]:
                     cmd.append("--peer=" + ctx.multiaddr)

src/cl/teku/teku_launcher.star

@@ -152,13 +152,7 @@ def launch(
             int(bn_max_mem) if int(bn_max_mem) > 0 else holesky_beacon_memory_limit
         )
 
-    network_name = (
-        "devnets"
-        if launcher.network != "kurtosis"
-        and launcher.network != "ephemery"
-        and launcher.network not in constants.PUBLIC_NETWORKS
-        else launcher.network
-    )
+    network_name = shared_utils.get_network_name(launcher.network)
 
     bn_min_cpu = int(bn_min_cpu) if int(bn_min_cpu) > 0 else BEACON_MIN_CPU
     bn_max_cpu = (
@@ -400,7 +394,10 @@ def get_beacon_config(
             + constants.GENESIS_CONFIG_MOUNT_PATH_ON_CONTAINER
             + "/genesis.ssz"
         )
-        if network == constants.NETWORK_NAME.kurtosis:
+        if (
+            network == constants.NETWORK_NAME.kurtosis
+            or constants.NETWORK_NAME.shadowfork in network
+        ):
             if bootnode_contexts != None:
                 cmd.append(
                     "--p2p-discovery-bootnodes="
@@ -430,6 +427,13 @@ def get_beacon_config(
                     plan, el_cl_genesis_data.files_artifact_uuid
                 )
             )
+        elif constants.NETWORK_NAME.shadowfork in network:
+            cmd.append(
+                "--p2p-discovery-bootnodes="
+                + shared_utils.get_devnet_enrs_list(
+                    plan, el_cl_genesis_data.files_artifact_uuid
+                )
+            )
         else:  # Devnets
             # TODO Remove once checkpoint sync is working for verkle
             if constants.NETWORK_NAME.verkle not in network:

src/el/besu/besu_launcher.star

@@ -87,13 +87,7 @@ def launch(
         el_tolerations, participant_tolerations, global_tolerations
     )
 
-    network_name = (
-        "devnets"
-        if launcher.network != "kurtosis"
-        and launcher.network != "ephemery"
-        and launcher.network not in constants.PUBLIC_NETWORKS
-        else launcher.network
-    )
+    network_name = shared_utils.get_network_name(launcher.network)
 
     el_min_cpu = int(el_min_cpu) if int(el_min_cpu) > 0 else EXECUTION_MIN_CPU
     el_max_cpu = (
@@ -208,7 +202,10 @@ def get_config(
         "--metrics-host=0.0.0.0",
         "--metrics-port={0}".format(METRICS_PORT_NUM),
     ]
-    if network not in constants.PUBLIC_NETWORKS:
+    if (
+        network not in constants.PUBLIC_NETWORKS
+        or constants.NETWORK_NAME.shadowfork in network
+    ):
         cmd.append(
             "--genesis-file="
             + constants.GENESIS_CONFIG_MOUNT_PATH_ON_CONTAINER

src/el/erigon/erigon_launcher.star

@@ -87,13 +87,7 @@ def launch(
         el_tolerations, participant_tolerations, global_tolerations
     )
 
-    network_name = (
-        "devnets"
-        if launcher.network != "kurtosis"
-        and launcher.network != "ephemery"
-        and launcher.network not in constants.PUBLIC_NETWORKS
-        else launcher.network
-    )
+    network_name = shared_utils.get_network_name(launcher.network)
 
     el_min_cpu = int(el_min_cpu) if int(el_min_cpu) > 0 else EXECUTION_MIN_CPU
     el_max_cpu = (
@@ -134,6 +128,7 @@ def launch(
         extra_params,
         extra_env_vars,
         extra_labels,
+        launcher.cancun_time,
         persistent,
         el_volume_size,
         tolerations,
@@ -181,6 +176,7 @@ def get_config(
     extra_params,
     extra_env_vars,
     extra_labels,
+    cancun_time,
     persistent,
     el_volume_size,
     tolerations,
@@ -195,6 +191,11 @@ def get_config(
         "--chain={0}".format(
             network if network in constants.PUBLIC_NETWORKS else "dev"
         ),
+        "{0}".format(
+            "--override.cancun=" + str(cancun_time)
+            if constants.NETWORK_NAME.shadowfork in network
+            else ""
+        ),
         "--networkid={0}".format(networkid),
         "--log.console.verbosity=" + verbosity_level,
         "--datadir=" + EXECUTION_DATA_DIRPATH_ON_CLIENT_CONTAINER,
@@ -296,10 +297,11 @@ def get_config(
     )
 
 
-def new_erigon_launcher(el_cl_genesis_data, jwt_file, network, networkid):
+def new_erigon_launcher(el_cl_genesis_data, jwt_file, network, networkid, cancun_time):
     return struct(
         el_cl_genesis_data=el_cl_genesis_data,
         jwt_file=jwt_file,
         network=network,
         networkid=networkid,
+        cancun_time=cancun_time,
     )

src/el/ethereumjs/ethereumjs_launcher.star

@@ -89,13 +89,7 @@ def launch(
         el_tolerations, participant_tolerations, global_tolerations
     )
 
-    network_name = (
-        "devnets"
-        if launcher.network != "kurtosis"
-        and launcher.network != "ephemery"
-        and launcher.network not in constants.PUBLIC_NETWORKS
-        else launcher.network
-    )
+    network_name = shared_utils.get_network_name(launcher.network)
 
     el_min_cpu = int(el_min_cpu) if int(el_min_cpu) > 0 else EXECUTION_MIN_CPU
     el_max_cpu = (

src/el/geth/geth_launcher.star

@@ -98,13 +98,7 @@ def launch(
         el_tolerations, participant_tolerations, global_tolerations
     )
 
-    network_name = (
-        "devnets"
-        if launcher.network != "kurtosis"
-        and launcher.network != "ephemery"
-        and launcher.network not in constants.PUBLIC_NETWORKS
-        else launcher.network
-    )
+    network_name = shared_utils.get_network_name(launcher.network)
 
     el_min_cpu = int(el_min_cpu) if int(el_min_cpu) > 0 else EXECUTION_MIN_CPU
     el_max_cpu = (
@@ -147,7 +141,8 @@ def launch(
         extra_labels,
         launcher.capella_fork_epoch,
         launcher.electra_fork_epoch,
-        launcher.final_genesis_timestamp,
+        launcher.cancun_time,
+        launcher.prague_time,
         persistent,
         el_volume_size,
         tolerations,
@@ -197,19 +192,22 @@ def get_config(
     extra_labels,
     capella_fork_epoch,
     electra_fork_epoch,
-    final_genesis_timestamp,
+    cancun_time,
+    prague_time,
     persistent,
     el_volume_size,
     tolerations,
 ):
     # TODO: Remove this once electra fork has path based storage scheme implemented
-    if electra_fork_epoch != None or constants.NETWORK_NAME.verkle in network:
+    if (
+        electra_fork_epoch != None or constants.NETWORK_NAME.verkle in network
+    ) and constants.NETWORK_NAME.shadowfork not in network:
         if (
             electra_fork_epoch == 0 or constants.NETWORK_NAME.verkle + "-gen" in network
         ):  # verkle-gen
             init_datadir_cmd_str = "geth --datadir={0} --cache.preimages --override.prague={1} init {2}".format(
                 EXECUTION_DATA_DIRPATH_ON_CLIENT_CONTAINER,
-                final_genesis_timestamp,
+                prague_time,
                 constants.GENESIS_CONFIG_MOUNT_PATH_ON_CONTAINER + "/genesis.json",
             )
         else:  # verkle
@@ -224,6 +222,8 @@ def get_config(
             EXECUTION_DATA_DIRPATH_ON_CLIENT_CONTAINER,
             constants.GENESIS_CONFIG_MOUNT_PATH_ON_CONTAINER + "/genesis.json",
         )
+    elif constants.NETWORK_NAME.shadowfork in network:
+        init_datadir_cmd_str = "echo shadowfork"
     else:
         init_datadir_cmd_str = "geth init --state.scheme=path --datadir={0} {1}".format(
             EXECUTION_DATA_DIRPATH_ON_CLIENT_CONTAINER,
@@ -239,6 +239,7 @@ def get_config(
             "--state.scheme=path"
             if electra_fork_epoch == None
             and "verkle" not in network
+            and constants.NETWORK_NAME.shadowfork not in network  # for now
             and "--builder" not in extra_params
             and capella_fork_epoch == 0
             else ""
@@ -251,13 +252,18 @@ def get_config(
         ),
         # Override prague fork timestamp if electra_fork_epoch == 0
         "{0}".format(
-            "--override.prague=" + final_genesis_timestamp
+            "--override.prague=" + str(prague_time)
             if electra_fork_epoch == 0 or "verkle-gen" in network
             else ""
         ),
         "{0}".format(
             "--{}".format(network) if network in constants.PUBLIC_NETWORKS else ""
         ),
+        "{0}".format(
+            "--override.cancun=" + str(cancun_time)
+            if constants.NETWORK_NAME.shadowfork in network
+            else ""
+        ),
         "--networkid={0}".format(networkid),
         "--verbosity=" + verbosity_level,
         "--datadir=" + EXECUTION_DATA_DIRPATH_ON_CLIENT_CONTAINER,
@@ -301,7 +307,10 @@ def get_config(
             if "--ws.api" in arg:
                 cmd[index] = "--ws.api=admin,engine,net,eth,web3,debug,suavex"
 
-    if network == constants.NETWORK_NAME.kurtosis:
+    if (
+        network == constants.NETWORK_NAME.kurtosis
+        or constants.NETWORK_NAME.shadowfork in network
+    ):
         if len(existing_el_clients) > 0:
             cmd.append(
                 "--bootnodes="
@@ -312,6 +321,13 @@ def get_config(
                     ]
                 )
             )
+        if (
+            constants.NETWORK_NAME.shadowfork in network and "verkle" in network
+        ):  # verkle shadowfork
+            cmd.append("--override.prague=" + str(prague_time))
+            cmd.append("--override.overlay-stride=10000")
+            cmd.append("--override.blockproof=true")
+            cmd.append("--clear.verkle.costs=true")
     elif network not in constants.PUBLIC_NETWORKS:
         cmd.append(
             "--bootnodes="
@@ -371,8 +387,9 @@ def new_geth_launcher(
     jwt_file,
     network,
     networkid,
-    final_genesis_timestamp,
     capella_fork_epoch,
+    cancun_time,
+    prague_time,
     electra_fork_epoch=None,
 ):
     return struct(
@@ -380,7 +397,8 @@ def new_geth_launcher(
         jwt_file=jwt_file,
         network=network,
         networkid=networkid,
-        final_genesis_timestamp=final_genesis_timestamp,
         capella_fork_epoch=capella_fork_epoch,
+        cancun_time=cancun_time,
+        prague_time=prague_time,
         electra_fork_epoch=electra_fork_epoch,
     )

src/el/nethermind/nethermind_launcher.star

@@ -85,13 +85,7 @@ def launch(
         el_tolerations, participant_tolerations, global_tolerations
     )
 
-    network_name = (
-        "devnets"
-        if launcher.network != "kurtosis"
-        and launcher.network != "ephemery"
-        and launcher.network not in constants.PUBLIC_NETWORKS
-        else launcher.network
-    )
+    network_name = shared_utils.get_network_name(launcher.network)
 
     el_min_cpu = int(el_min_cpu) if int(el_min_cpu) > 0 else EXECUTION_MIN_CPU
     el_max_cpu = (
@@ -208,10 +202,20 @@ def get_config(
             + constants.GENESIS_CONFIG_MOUNT_PATH_ON_CONTAINER
             + "/chainspec.json"
         )
+    elif constants.NETWORK_NAME.shadowfork in network:
+        cmd.append(
+            "--Init.ChainSpecPath="
+            + constants.GENESIS_CONFIG_MOUNT_PATH_ON_CONTAINER
+            + "/chainspec.json"
+        )
+        cmd.append("--config=" + network)
     else:
         cmd.append("--config=" + network)
 
-    if network == constants.NETWORK_NAME.kurtosis:
+    if (
+        network == constants.NETWORK_NAME.kurtosis
+        or constants.NETWORK_NAME.shadowfork in network
+    ):
         if len(existing_el_clients) > 0:
             cmd.append(
                 "--Network.StaticPeers="

src/el/reth/reth_launcher.star

@@ -87,13 +87,8 @@ def launch(
     tolerations = input_parser.get_client_tolerations(
         el_tolerations, participant_tolerations, global_tolerations
     )
-    network_name = (
-        "devnets"
-        if launcher.network != "kurtosis"
-        and launcher.network != "ephemery"
-        and launcher.network not in constants.PUBLIC_NETWORKS
-        else launcher.network
-    )
+
+    network_name = shared_utils.get_network_name(launcher.network)
 
     el_min_cpu = int(el_min_cpu) if int(el_min_cpu) > 0 else EXECUTION_MIN_CPU
     el_max_cpu = (

src/package_io/constants.star

@@ -53,6 +53,13 @@ CAPELLA_FORK_VERSION = "0x40000038"
 DENEB_FORK_VERSION = "0x50000038"
 ELECTRA_FORK_VERSION = "0x60000038"
 
+ETHEREUM_GENESIS_GENERATOR = struct(
+    bellatrix_genesis="ethpandaops/ethereum-genesis-generator:1.3.15",  # EOL
+    capella_genesis="ethpandaops/ethereum-genesis-generator:2.0.12",  # Default
+    verkle_support_genesis="ethpandaops/ethereum-genesis-generator:3.0.0-rc.19",  # soon to be deneb genesis
+    verkle_genesis="ethpandaops/ethereum-genesis-generator:4.0.0-rc.6",
+)
+
 NETWORK_NAME = struct(
     mainnet="mainnet",
     goerli="goerli",
@@ -61,6 +68,7 @@ NETWORK_NAME = struct(
     ephemery="ephemery",
     kurtosis="kurtosis",
     verkle="verkle",
+    shadowfork="shadowfork",
 )
 
 PUBLIC_NETWORKS = (
@@ -92,6 +100,14 @@ GENESIS_VALIDATORS_ROOT = {
     "holesky": "0x9143aa7c615a7f7115e2b6aac319c03529df8242ae705fba9df39b79c59fa8b1",
 }
 
+DEPOSIT_CONTRACT_ADDRESS = {
+    "mainnet": "0x00000000219ab540356cBB839Cbe05303d7705Fa",
+    "goerli": "0xff50ed3d0ec03aC01D4C79aAd74928BFF48a7b2b",
+    "sepolia": "0x7f02C3E3c98b133055B8B348B2Ac625669Ed295D",
+    "holesky": "0x4242424242424242424242424242424242424242",
+    "ephemery": "0x4242424242424242424242424242424242424242",
+}
+
 GENESIS_TIME = {
     "mainnet": 1606824023,
     "goerli": 1616508000,

src/package_io/input_parser.star

@@ -30,7 +30,7 @@ NIMBUS_NODE_NAME = "nimbus"
 
 # Placeholder value for the deneb fork epoch if electra is being run
 # TODO: This is a hack, and should be removed once we electra is rebased on deneb
-HIGH_DENEB_VALUE_FORK_VERKLE = 20000
+HIGH_DENEB_VALUE_FORK_VERKLE = 2000000000
 
 # MEV Params
 FLASHBOTS_MEV_BOOST_PORT = 18550
@@ -66,7 +66,10 @@ def input_parser(plan, input_args):
     # add default eth2 input params
     result["mev_type"] = None
     result["mev_params"] = get_default_mev_params()
-    if result["network_params"]["network"] == "kurtosis":
+    if (
+        result["network_params"]["network"] == constants.NETWORK_NAME.kurtosis
+        or constants.NETWORK_NAME.shadowfork in result["network_params"]["network"]
+    ):
         result["additional_services"] = DEFAULT_ADDITIONAL_SERVICES
     else:
         result["additional_services"] = []
@@ -80,6 +83,12 @@ def input_parser(plan, input_args):
     result["persistent"] = False
     result["global_tolerations"] = []
 
+    if constants.NETWORK_NAME.shadowfork in result["network_params"]["network"]:
+        shadow_base = result["network_params"]["network"].split("-shadowfork")[0]
+        result["network_params"][
+            "deposit_contract_address"
+        ] = constants.DEPOSIT_CONTRACT_ADDRESS[shadow_base]
+
     for attr in input_args:
         value = input_args[attr]
         # if its inserted we use the value inserted
@@ -209,6 +218,10 @@ def input_parser(plan, input_args):
             deneb_fork_epoch=result["network_params"]["deneb_fork_epoch"],
             electra_fork_epoch=result["network_params"]["electra_fork_epoch"],
             network=result["network_params"]["network"],
+            min_validator_withdrawability_delay=result["network_params"][
+                "min_validator_withdrawability_delay"
+            ],
+            shard_committee_period=result["network_params"]["shard_committee_period"],
         ),
         mev_params=struct(
             mev_relay_image=result["mev_params"]["mev_relay_image"],
@@ -406,7 +419,10 @@ def parse_network_params(input_args):
             "deposit_contract_address is empty or spaces it needs to be of non zero length"
         )
 
-    if result["network_params"]["network"] == "kurtosis":
+    if (
+        result["network_params"]["network"] == "kurtosis"
+        or constants.NETWORK_NAME.shadowfork in result["network_params"]["network"]
+    ):
         if (
             result["network_params"]["preregistered_validator_keys_mnemonic"].strip()
             == ""
@@ -431,7 +447,10 @@ def parse_network_params(input_args):
     ):
         fail("electra can only happen with capella genesis not bellatrix")
 
-    if result["network_params"]["network"] == "kurtosis":
+    if (
+        result["network_params"]["network"] == constants.NETWORK_NAME.kurtosis
+        or constants.NETWORK_NAME.shadowfork in result["network_params"]["network"]
+    ):
         if MIN_VALIDATORS > actual_num_validators:
             fail(
                 "We require at least {0} validators but got {1}".format(
@@ -533,6 +552,8 @@ def default_network_params():
         "deneb_fork_epoch": 500,
         "electra_fork_epoch": None,
         "network": "kurtosis",
+        "min_validator_withdrawability_delay": 256,
+        "shard_committee_period": 256,
     }
 
 

src/participant_network.star

@@ -8,6 +8,9 @@ el_cl_genesis_data_generator = import_module(
 el_cl_genesis_data = import_module(
     "./prelaunch_data_generator/el_cl_genesis/el_cl_genesis_data.star"
 )
+
+input_parser = import_module("./package_io/input_parser.star")
+
 shared_utils = import_module("./shared_utils/shared_utils.star")
 
 static_files = import_module("./static_files/static_files.star")
@@ -67,9 +70,127 @@ def launch_participant_network(
     global_tolerations,
     parallel_keystore_generation=False,
 ):
+    network_id = network_params.network_id
     num_participants = len(participants)
-    if network_params.network == constants.NETWORK_NAME.kurtosis:
-        # We are running a kurtosis network
+    latest_block = ""
+    cancun_time = 0
+    prague_time = 0
+    shadowfork_block = "latest"
+    if (
+        constants.NETWORK_NAME.shadowfork in network_params.network
+        and ("verkle" in network_params.network)
+        and ("holesky" in network_params.network)
+    ):
+        shadowfork_block = "793312"  # Hardcodes verkle shadowfork block for holesky
+
+    if (
+        network_params.network == constants.NETWORK_NAME.kurtosis
+        or constants.NETWORK_NAME.shadowfork in network_params.network
+    ):
+        if (
+            constants.NETWORK_NAME.shadowfork in network_params.network
+        ):  # shadowfork requires some preparation
+            base_network = shared_utils.get_network_name(network_params.network)
+            # overload the network name to remove the shadowfork suffix
+            if constants.NETWORK_NAME.ephemery in base_network:
+                chain_id = plan.run_sh(
+                    run="curl -s https://ephemery.dev/latest/config.yaml | yq .DEPOSIT_CHAIN_ID | tr -d '\n'",
+                    image="linuxserver/yq",
+                )
+                network_id = chain_id.output
+            else:
+                network_id = constants.NETWORK_ID[
+                    base_network
+                ]  # overload the network id to match the network name
+            latest_block = plan.run_sh(  # fetch the latest block
+                run="mkdir -p /shadowfork && \
+                    curl -o /shadowfork/latest_block.json https://ethpandaops-ethereum-node-snapshots.ams3.digitaloceanspaces.com/"
+                + base_network
+                + "/geth/"
+                + shadowfork_block
+                + "/_snapshot_eth_getBlockByNumber.json",
+                image="badouralix/curl-jq",
+                store=[StoreSpec(src="/shadowfork", name="latest_blocks")],
+            )
+
+            # maybe we can do the copy in the same step as the fetch?
+            for index, participant in enumerate(participants):
+                tolerations = input_parser.get_client_tolerations(
+                    participant.el_tolerations,
+                    participant.tolerations,
+                    global_tolerations,
+                )
+                cl_client_type = participant.cl_client_type
+                el_client_type = participant.el_client_type
+
+                # Zero-pad the index using the calculated zfill value
+                index_str = shared_utils.zfill_custom(
+                    index + 1, len(str(len(participants)))
+                )
+
+                el_service_name = "el-{0}-{1}-{2}".format(
+                    index_str, el_client_type, cl_client_type
+                )
+                shadowfork_data = plan.add_service(
+                    name="shadowfork-{0}".format(el_service_name),
+                    config=ServiceConfig(
+                        image="alpine:3.19.1",
+                        cmd=[
+                            "apk add --no-cache curl tar zstd && curl -s -L https://ethpandaops-ethereum-node-snapshots.ams3.digitaloceanspaces.com/"
+                            + base_network
+                            + "/"
+                            + el_client_type
+                            + "/"
+                            + shadowfork_block
+                            + "/snapshot.tar.zst"
+                            + " | tar -I zstd -xvf - -C /data/"
+                            + el_client_type
+                            + "/execution-data"
+                            + " && touch /tmp/finished"
+                            + " && tail -f /dev/null"
+                        ],
+                        entrypoint=["/bin/sh", "-c"],
+                        files={
+                            "/data/"
+                            + el_client_type
+                            + "/execution-data": Directory(
+                                persistent_key="data-{0}".format(el_service_name),
+                                size=constants.VOLUME_SIZE[base_network][
+                                    el_client_type + "_volume_size"
+                                ],
+                            ),
+                        },
+                        env_vars={
+                            "RCLONE_CONFIG_MYS3_TYPE": "s3",
+                            "RCLONE_CONFIG_MYS3_PROVIDER": "DigitalOcean",
+                            "RCLONE_CONFIG_MYS3_ENDPOINT": "https://ams3.digitaloceanspaces.com",
+                        },
+                        tolerations=tolerations,
+                    ),
+                )
+            for index, participant in enumerate(participants):
+                cl_client_type = participant.cl_client_type
+                el_client_type = participant.el_client_type
+
+                # Zero-pad the index using the calculated zfill value
+                index_str = shared_utils.zfill_custom(
+                    index + 1, len(str(len(participants)))
+                )
+
+                el_service_name = "el-{0}-{1}-{2}".format(
+                    index_str, el_client_type, cl_client_type
+                )
+                plan.wait(
+                    service_name="shadowfork-{0}".format(el_service_name),
+                    recipe=ExecRecipe(command=["cat", "/tmp/finished"]),
+                    field="code",
+                    assertion="==",
+                    target_value=0,
+                    interval="1s",
+                    timeout="6h",  # 6 hours should be enough for the biggest network
+                )
+
+        # We are running a kurtosis or shadowfork network
         plan.print("Generating cl validator key stores")
         validator_data = None
         if not parallel_keystore_generation:
@@ -87,8 +208,6 @@ def launch_participant_network(
 
         plan.print(json.indent(json.encode(validator_data)))
 
-        network_id = network_params.network_id
-
         # We need to send the same genesis time to both the EL and the CL to ensure that timestamp based forking works as expected
         final_genesis_timestamp = get_final_genesis_timestamp(
             plan,
@@ -105,13 +224,14 @@ def launch_participant_network(
                 total_number_of_validator_keys += participant.validator_count
 
         plan.print("Generating EL CL data")
+
         # we are running bellatrix genesis (deprecated) - will be removed in the future
         if (
             network_params.capella_fork_epoch > 0
             and network_params.electra_fork_epoch == None
         ):
             ethereum_genesis_generator_image = (
-                "ethpandaops/ethereum-genesis-generator:1.3.15"
+                constants.ETHEREUM_GENESIS_GENERATOR.bellatrix_genesis
             )
         # we are running capella genesis - default behavior
         elif (
@@ -119,17 +239,17 @@ def launch_participant_network(
             and network_params.electra_fork_epoch == None
         ):
             ethereum_genesis_generator_image = (
-                "ethpandaops/ethereum-genesis-generator:2.0.11"
+                constants.ETHEREUM_GENESIS_GENERATOR.capella_genesis
             )
         # we are running electra - experimental
         elif network_params.electra_fork_epoch != None:
             if network_params.electra_fork_epoch == 0:
                 ethereum_genesis_generator_image = (
-                    "ethpandaops/ethereum-genesis-generator:4.0.0-rc.6"
+                    constants.ETHEREUM_GENESIS_GENERATOR.verkle_genesis
                 )
             else:
                 ethereum_genesis_generator_image = (
-                    "ethpandaops/ethereum-genesis-generator:3.0.0-rc.18"
+                    constants.ETHEREUM_GENESIS_GENERATOR.verkle_support_genesis
                 )
         else:
             fail(
@@ -145,7 +265,7 @@ def launch_participant_network(
             ethereum_genesis_generator_image,
             el_cl_genesis_config_template,
             final_genesis_timestamp,
-            network_params.network_id,
+            network_id,
             network_params.deposit_contract_address,
             network_params.seconds_per_slot,
             network_params.preregistered_validator_keys_mnemonic,
@@ -157,6 +277,9 @@ def launch_participant_network(
             network_params.capella_fork_epoch,
             network_params.deneb_fork_epoch,
             network_params.electra_fork_epoch,
+            latest_block.files_artifacts[0] if latest_block != "" else "",
+            network_params.min_validator_withdrawability_delay,
+            network_params.shard_committee_period,
         )
     elif network_params.network in constants.PUBLIC_NETWORKS:
         # We are running a public network
@@ -167,6 +290,8 @@ def launch_participant_network(
         el_cl_data = el_cl_genesis_data.new_el_cl_genesis_data(
             dummy.files_artifacts[0],
             constants.GENESIS_VALIDATORS_ROOT[network_params.network],
+            cancun_time,
+            prague_time,
         )
         final_genesis_timestamp = constants.GENESIS_TIME[network_params.network]
         network_id = constants.NETWORK_ID[network_params.network]
@@ -184,6 +309,8 @@ def launch_participant_network(
         el_cl_data = el_cl_genesis_data.new_el_cl_genesis_data(
             el_cl_genesis_data_uuid.files_artifacts[0],
             genesis_validators_root,
+            cancun_time,
+            prague_time,
         )
         final_genesis_timestamp = shared_utils.read_genesis_timestamp_from_config(
             plan, el_cl_genesis_data_uuid.files_artifacts[0]
@@ -209,6 +336,8 @@ def launch_participant_network(
         el_cl_data = el_cl_genesis_data.new_el_cl_genesis_data(
             el_cl_genesis_data_uuid.files_artifacts[0],
             genesis_validators_root,
+            cancun_time,
+            prague_time,
         )
         final_genesis_timestamp = shared_utils.read_genesis_timestamp_from_config(
             plan, el_cl_genesis_data_uuid.files_artifacts[0]
@@ -225,8 +354,9 @@ def launch_participant_network(
                 jwt_file,
                 network_params.network,
                 network_id,
-                final_genesis_timestamp,
                 network_params.capella_fork_epoch,
+                el_cl_data.cancun_time,
+                el_cl_data.prague_time,
                 network_params.electra_fork_epoch,
             ),
             "launch_method": geth.launch,
@@ -237,8 +367,9 @@ def launch_participant_network(
                 jwt_file,
                 network_params.network,
                 network_id,
-                final_genesis_timestamp,
                 network_params.capella_fork_epoch,
+                el_cl_data.cancun_time,
+                el_cl_data.prague_time,
                 network_params.electra_fork_epoch,
             ),
             "launch_method": geth.launch,
@@ -257,6 +388,7 @@ def launch_participant_network(
                 jwt_file,
                 network_params.network,
                 network_id,
+                el_cl_data.cancun_time,
             ),
             "launch_method": erigon.launch,
         },
@@ -399,6 +531,7 @@ def launch_participant_network(
     preregistered_validator_keys_for_nodes = (
         validator_data.per_node_keystores
         if network_params.network == constants.NETWORK_NAME.kurtosis
+        or constants.NETWORK_NAME.shadowfork in network_params.network
         else None
     )
 

src/prelaunch_data_generator/el_cl_genesis/el_cl_genesis_data.star

 def new_el_cl_genesis_data(
     files_artifact_uuid,
     genesis_validators_root,
+    cancun_time,
+    prague_time,
 ):
     return struct(
         files_artifact_uuid=files_artifact_uuid,
         genesis_validators_root=genesis_validators_root,
+        cancun_time=cancun_time,
+        prague_time=prague_time,
     )

src/prelaunch_data_generator/el_cl_genesis/el_cl_genesis_generator.star

@@ -6,6 +6,7 @@ constants = import_module("../../package_io/constants.star")
 
 GENESIS_VALUES_PATH = "/opt"
 GENESIS_VALUES_FILENAME = "values.env"
+SHADOWFORK_FILEPATH = "/shadowfork"
 
 
 def generate_el_cl_genesis_data(
@@ -25,7 +26,16 @@ def generate_el_cl_genesis_data(
     capella_fork_epoch,
     deneb_fork_epoch,
     electra_fork_epoch,
+    latest_block,
+    min_validator_withdrawability_delay,
+    shard_committee_period,
 ):
+    files = {}
+    shadowfork_file = ""
+    if latest_block != "":
+        files[SHADOWFORK_FILEPATH] = latest_block
+        shadowfork_file = SHADOWFORK_FILEPATH + "/shadowfork/latest_block.json"
+
     template_data = new_env_file_for_el_cl_genesis_data(
         genesis_unix_timestamp,
         network_id,
@@ -40,6 +50,9 @@ def generate_el_cl_genesis_data(
         capella_fork_epoch,
         deneb_fork_epoch,
         electra_fork_epoch,
+        shadowfork_file,
+        min_validator_withdrawability_delay,
+        shard_committee_period,
     )
     genesis_generation_template = shared_utils.new_template_and_data(
         genesis_generation_config_yml_template, template_data
@@ -55,10 +68,12 @@ def generate_el_cl_genesis_data(
         genesis_values_and_dest_filepath, "genesis-el-cl-env-file"
     )
 
+    files[GENESIS_VALUES_PATH] = genesis_generation_config_artifact_name
+
     genesis = plan.run_sh(
         run="cp /opt/values.env /config/values.env && ./entrypoint.sh all && mkdir /network-configs && mv /data/custom_config_data/* /network-configs/",
         image=image,
-        files={GENESIS_VALUES_PATH: genesis_generation_config_artifact_name},
+        files=files,
         store=[
             StoreSpec(src="/network-configs/", name="el_cl_genesis_data"),
             StoreSpec(
@@ -75,8 +90,23 @@ def generate_el_cl_genesis_data(
         wait=None,
     )
 
+    cancun_time = plan.run_sh(
+        run="jq .config.cancunTime /data/network-configs/genesis.json | tr -d '\n'",
+        image="badouralix/curl-jq",
+        files={"/data": genesis.files_artifacts[0]},
+    )
+
+    prague_time = plan.run_sh(
+        run="jq .config.pragueTime /data/network-configs/genesis.json | tr -d '\n'",
+        image="badouralix/curl-jq",
+        files={"/data": genesis.files_artifacts[0]},
+    )
+
     result = el_cl_genesis_data.new_el_cl_genesis_data(
-        genesis.files_artifacts[0], genesis_validators_root.output
+        genesis.files_artifacts[0],
+        genesis_validators_root.output,
+        cancun_time.output,
+        prague_time.output,
     )
 
     return result
@@ -96,6 +126,9 @@ def new_env_file_for_el_cl_genesis_data(
     capella_fork_epoch,
     deneb_fork_epoch,
     electra_fork_epoch,
+    shadowfork_file,
+    min_validator_withdrawability_delay,
+    shard_committee_period,
 ):
     return {
         "UnixTimestamp": genesis_unix_timestamp,
@@ -116,4 +149,7 @@ def new_env_file_for_el_cl_genesis_data(
         "CapellaForkVersion": constants.CAPELLA_FORK_VERSION,
         "DenebForkVersion": constants.DENEB_FORK_VERSION,
         "ElectraForkVersion": constants.ELECTRA_FORK_VERSION,
+        "ShadowForkFile": shadowfork_file,
+        "MinValidatorWithdrawabilityDelay": min_validator_withdrawability_delay,
+        "ShardCommitteePeriod": shard_committee_period,
     }

src/shared_utils/shared_utils.star

@@ -139,3 +139,19 @@ print(network_id, end="")
         """,
     )
     return value.output
+
+
+def get_network_name(network):
+    network_name = network
+    if (
+        network != constants.NETWORK_NAME.kurtosis
+        and network != constants.NETWORK_NAME.ephemery
+        and constants.NETWORK_NAME.shadowfork not in network
+        and network not in constants.PUBLIC_NETWORKS
+    ):
+        network_name = "devnets"
+
+    if constants.NETWORK_NAME.shadowfork in network:
+        network_name = network.split("-shadowfork")[0]
+
+    return network_name

src/static_files/static_files.star

@@ -68,3 +68,5 @@ CL_GENESIS_GENERATION_MNEMONICS_TEMPLATE_FILEPATH = (
 )
 
 JWT_PATH_FILEPATH = STATIC_FILES_DIRPATH + "/jwt/jwtsecret"
+
+SHADOWFORK_FILEPATH = "/network-configs/latest_block.json"

static_files/genesis-generation-config/el-cl/values.env.tmpl

@@ -21,3 +21,6 @@ export GENESIS_DELAY={{ .GenesisDelay }}
 export MAX_CHURN={{ .MaxChurn }}
 export EJECTION_BALANCE={{ .EjectionBalance }}
 export ETH1_FOLLOW_DISTANCE={{ .Eth1FollowDistance }}
+export SHADOW_FORK_FILE={{ .ShadowForkFile }}
+export MIN_VALIDATOR_WITHDRAWABILITY_DELAY={{ .MinValidatorWithdrawabilityDelay }}
+export SHARD_COMMITTEE_PERIOD={{ .ShardCommitteePeriod }}