feat: enable persistence (#422)

.circleci/config.yml

@@ -109,6 +109,13 @@ jobs:
       - checkout
       - run: kurtosis run ${PWD} "$(cat ./.github/tests/mix-with-tools-mev.yaml)"
 
+  mix_persistence_k8s:
+    resource_class: xlarge
+    executor: ubuntu_vm
+    steps:
+      - checkout
+      - run: kurtosis run ${PWD} "$(cat ./.github/tests/mix-persistence-k8s.yaml)"
+
 workflows:
   nightly_tests:
     when: << pipeline.parameters.should-enable-check-latest-version-workflow >>
@@ -155,3 +162,10 @@ workflows:
             branches:
               ignore:
                 - main
+
+      - mix_persistence_k8s:
+          <<: *setup_kurtosis_k3s
+          filters:
+            branches:
+              ignore:
+                - main

.github/tests/besu-persistence.yaml

+participants:
+  - el_client_type: besu
+    cl_client_type: lighthouse
+additional_services: []
+persistent: true

.github/tests/erigon-persistence.yaml

+participants:
+  - el_client_type: erigon
+    cl_client_type: lighthouse
+additional_services: []
+persistent: true

.github/tests/mix-persistence-k8s.yaml

+participants:
+  - el_client_type: geth
+    cl_client_type: prysm
+  - el_client_type: nethermind
+    cl_client_type: nimbus
+    cl_split_mode_enabled: true
+  - el_client_type: besu
+    cl_client_type: lighthouse
+  - el_client_type: reth
+    cl_client_type: lodestar
+  - el_client_type: ethereumjs
+    cl_client_type: teku
+    cl_split_mode_enabled: true
+  - el_client_type: erigon
+    cl_client_type: teku
+  - el_client_type: besu
+    cl_client_type: nimbus
+additional_services: []
+persistent: true

.github/tests/mix-persistence.yaml

+participants:
+  - el_client_type: geth
+    cl_client_type: prysm
+  - el_client_type: geth
+    cl_client_type: nimbus
+    cl_split_mode_enabled: true
+  - el_client_type: nethermind
+    cl_client_type: lighthouse
+  - el_client_type: reth
+    cl_client_type: lodestar
+  - el_client_type: ethereumjs
+    cl_client_type: lighthouse
+additional_services:
+  - dora
+persistent: true

.github/tests/teku-persistence.yaml

+participants:
+  - el_client_type: geth
+    cl_client_type: teku
+additional_services: []
+persistent: true

.github/workflows/per-pr.yml

@@ -32,6 +32,7 @@ jobs:
             "./.github/tests/mev.yaml",
             "./.github/tests/mev-mock.yaml",
             "./.github/tests/mix-with-tools.yaml",
+            "./.github/tests/mix-persistence.yaml",
             "./network_params.yaml"
           ]
     runs-on: ubuntu-latest

README.md

@@ -331,6 +331,7 @@ grafana_additional_dashboards: []
 
 # Whether the environment should be persistent; this is WIP and is slowly being rolled out accross services
 # Note this requires Kurtosis greater than 0.85.49 to work
+# Note Erigon, Besu, Teku persistence is not currently supported with docker.
 # Defaults to False
 persistent: False
 

main.star

@@ -86,6 +86,7 @@ def run(plan, args={}):
         args_with_right_defaults.participants,
         network_params,
         args_with_right_defaults.global_client_log_level,
+        persistent,
         parallel_keystore_generation,
     )
 

src/cl/lighthouse/lighthouse_launcher.star

@@ -10,13 +10,11 @@ blobber_launcher = import_module("../../blobber/blobber_launcher.star")
 
 LIGHTHOUSE_BINARY_COMMAND = "lighthouse"
 
-VALIDATOR_KEYS_MOUNTPOINT_ON_CLIENTS = "/validator-keys"
-
 RUST_BACKTRACE_ENVVAR_NAME = "RUST_BACKTRACE"
 RUST_FULL_BACKTRACE_KEYWORD = "full"
 
 #  ---------------------------------- Beacon client -------------------------------------
-CONSENSUS_DATA_DIRPATH_ON_BEACON_SERVICE_CONTAINER = "/consensus-data"
+BEACON_DATA_DIRPATH_ON_BEACON_SERVICE_CONTAINER = "/data/lighthouse/beacon-data"
 
 # Port IDs
 BEACON_TCP_DISCOVERY_PORT_ID = "tcp-discovery"
@@ -36,6 +34,7 @@ BEACON_MIN_MEMORY = 256
 BEACON_MAX_MEMORY = 1024
 
 #  ---------------------------------- Validator client -------------------------------------
+VALIDATOR_KEYS_MOUNTPOINT_ON_CLIENTS = "/data/lighthouse/validator-keys"
 VALIDATOR_HTTP_PORT_ID = "http"
 VALIDATOR_METRICS_PORT_ID = "metrics"
 VALIDATOR_HTTP_PORT_NUM = 5042
@@ -121,10 +120,11 @@ def launch(
     extra_validator_params,
     extra_beacon_labels,
     extra_validator_labels,
+    persistent,
     split_mode_enabled=False,
 ):
-    beacon_node_service_name = "{0}".format(service_name)
-    validator_node_service_name = "{0}-{1}".format(
+    beacon_service_name = "{0}".format(service_name)
+    validator_service_name = "{0}-{1}".format(
         service_name, VALIDATOR_SUFFIX_SERVICE_NAME
     )
 
@@ -141,6 +141,7 @@ def launch(
     beacon_config = get_beacon_config(
         launcher.el_cl_genesis_data,
         image,
+        beacon_service_name,
         bootnode_contexts,
         el_client_context,
         log_level,
@@ -152,9 +153,10 @@ def launch(
         snooper_engine_context,
         extra_beacon_params,
         extra_beacon_labels,
+        persistent,
     )
 
-    beacon_service = plan.add_service(beacon_node_service_name, beacon_config)
+    beacon_service = plan.add_service(beacon_service_name, beacon_config)
     beacon_http_port = beacon_service.ports[BEACON_HTTP_PORT_ID]
     beacon_http_url = "http://{0}:{1}".format(
         beacon_service.ip_address, beacon_http_port.number
@@ -162,7 +164,7 @@ def launch(
 
     # Blobber config
     if blobber_enabled:
-        blobber_service_name = "{0}-{1}".format("blobber", beacon_node_service_name)
+        blobber_service_name = "{0}-{1}".format("blobber", beacon_service_name)
         blobber_config = blobber_launcher.get_config(
             blobber_service_name,
             node_keystore_files,
@@ -190,6 +192,7 @@ def launch(
         validator_config = get_validator_config(
             launcher.el_cl_genesis_data,
             image,
+            validator_service_name,
             log_level,
             beacon_http_url,
             el_client_context,
@@ -198,14 +201,12 @@ def launch(
             v_max_cpu,
             v_min_mem,
             v_max_mem,
-            validator_node_service_name,
             extra_validator_params,
             extra_validator_labels,
+            persistent,
         )
 
-        validator_service = plan.add_service(
-            validator_node_service_name, validator_config
-        )
+        validator_service = plan.add_service(validator_service_name, validator_config)
 
     # TODO(old) add validator availability using the validator API: https://ethereum.github.io/beacon-APIs/?urls.primaryName=v1#/ValidatorRequiredApi | from eth2-merge-kurtosis-module
     beacon_node_identity_recipe = GetHttpRequestRecipe(
@@ -218,7 +219,7 @@ def launch(
         },
     )
     response = plan.request(
-        recipe=beacon_node_identity_recipe, service_name=beacon_node_service_name
+        recipe=beacon_node_identity_recipe, service_name=beacon_service_name
     )
     beacon_node_enr = response["extract.enr"]
     beacon_multiaddr = response["extract.multiaddr"]
@@ -229,7 +230,7 @@ def launch(
         beacon_service.ip_address, beacon_metrics_port.number
     )
     beacon_node_metrics_info = node_metrics.new_node_metrics_info(
-        beacon_node_service_name, METRICS_PATH, beacon_metrics_url
+        beacon_service_name, METRICS_PATH, beacon_metrics_url
     )
     nodes_metrics_info = [beacon_node_metrics_info]
 
@@ -239,7 +240,7 @@ def launch(
             validator_service.ip_address, validator_metrics_port.number
         )
         validator_node_metrics_info = node_metrics.new_node_metrics_info(
-            validator_node_service_name, METRICS_PATH, validator_metrics_url
+            validator_service_name, METRICS_PATH, validator_metrics_url
         )
         nodes_metrics_info.append(validator_node_metrics_info)
 
@@ -249,8 +250,8 @@ def launch(
         beacon_service.ip_address,
         BEACON_HTTP_PORT_NUM,
         nodes_metrics_info,
-        beacon_node_service_name,
-        validator_node_service_name,
+        beacon_service_name,
+        validator_service_name,
         beacon_multiaddr,
         beacon_peer_id,
         snooper_enabled,
@@ -264,6 +265,7 @@ def launch(
 def get_beacon_config(
     el_cl_genesis_data,
     image,
+    service_name,
     boot_cl_client_ctxs,
     el_client_context,
     log_level,
@@ -275,6 +277,7 @@ def get_beacon_config(
     snooper_engine_context,
     extra_params,
     extra_labels,
+    persistent,
 ):
     # If snooper is enabled use the snooper engine context, otherwise use the execution client context
     if snooper_enabled:
@@ -300,7 +303,7 @@ def get_beacon_config(
         LIGHTHOUSE_BINARY_COMMAND,
         "beacon_node",
         "--debug-level=" + log_level,
-        "--datadir=" + CONSENSUS_DATA_DIRPATH_ON_BEACON_SERVICE_CONTAINER,
+        "--datadir=" + BEACON_DATA_DIRPATH_ON_BEACON_SERVICE_CONTAINER,
         "--testnet-dir=" + constants.GENESIS_CONFIG_MOUNT_PATH_ON_CONTAINER,
         # vvvvvvvvvvvvvvvvvvv REMOVE THESE WHEN CONNECTING TO EXTERNAL NET vvvvvvvvvvvvvvvvvvvvv
         "--disable-enr-auto-update",
@@ -358,14 +361,19 @@ def get_beacon_config(
     recipe = GetHttpRequestRecipe(
         endpoint="/eth/v1/node/identity", port_id=BEACON_HTTP_PORT_ID
     )
+    files = {
+        constants.GENESIS_DATA_MOUNTPOINT_ON_CLIENTS: el_cl_genesis_data.files_artifact_uuid
+    }
 
+    if persistent:
+        files[BEACON_DATA_DIRPATH_ON_BEACON_SERVICE_CONTAINER] = Directory(
+            persistent_key="data-{0}".format(service_name),
+        )
     return ServiceConfig(
         image=image,
         ports=BEACON_USED_PORTS,
         cmd=cmd,
-        files={
-            constants.GENESIS_DATA_MOUNTPOINT_ON_CLIENTS: el_cl_genesis_data.files_artifact_uuid
-        },
+        files=files,
         env_vars={RUST_BACKTRACE_ENVVAR_NAME: RUST_FULL_BACKTRACE_KEYWORD},
         private_ip_address_placeholder=PRIVATE_IP_ADDRESS_PLACEHOLDER,
         ready_conditions=cl_node_ready_conditions.get_ready_conditions(
@@ -388,6 +396,7 @@ def get_beacon_config(
 def get_validator_config(
     el_cl_genesis_data,
     image,
+    service_name,
     log_level,
     beacon_client_http_url,
     el_client_context,
@@ -396,9 +405,9 @@ def get_validator_config(
     v_max_cpu,
     v_min_mem,
     v_max_mem,
-    validator_node_service_name,
     extra_params,
     extra_labels,
+    persistent,
 ):
     validator_keys_dirpath = shared_utils.path_join(
         VALIDATOR_KEYS_MOUNTPOINT_ON_CLIENTS,
@@ -442,14 +451,16 @@ def get_validator_config(
     if len(extra_params):
         cmd.extend([param for param in extra_params])
 
+    files = {
+        constants.GENESIS_DATA_MOUNTPOINT_ON_CLIENTS: el_cl_genesis_data.files_artifact_uuid,
+        VALIDATOR_KEYS_MOUNTPOINT_ON_CLIENTS: node_keystore_files.files_artifact_uuid,
+    }
+
     return ServiceConfig(
         image=image,
         ports=VALIDATOR_USED_PORTS,
         cmd=cmd,
-        files={
-            constants.GENESIS_DATA_MOUNTPOINT_ON_CLIENTS: el_cl_genesis_data.files_artifact_uuid,
-            VALIDATOR_KEYS_MOUNTPOINT_ON_CLIENTS: node_keystore_files.files_artifact_uuid,
-        },
+        files=files,
         env_vars={RUST_BACKTRACE_ENVVAR_NAME: RUST_FULL_BACKTRACE_KEYWORD},
         min_cpu=v_min_cpu,
         max_cpu=v_max_cpu,

src/cl/lodestar/lodestar_launcher.star

@@ -7,7 +7,7 @@ blobber_launcher = import_module("../../blobber/blobber_launcher.star")
 constants = import_module("../../package_io/constants.star")
 
 #  ---------------------------------- Beacon client -------------------------------------
-CONSENSUS_DATA_DIRPATH_ON_SERVICE_CONTAINER = "/consensus-data"
+BEACON_DATA_DIRPATH_ON_SERVICE_CONTAINER = "/data/lodestar/beacon-data"
 # Port IDs
 TCP_DISCOVERY_PORT_ID = "tcp-discovery"
 UDP_DISCOVERY_PORT_ID = "udp-discovery"
@@ -28,6 +28,7 @@ BEACON_MAX_MEMORY = 1024
 
 #  ---------------------------------- Validator client -------------------------------------
 VALIDATOR_KEYS_MOUNT_DIRPATH_ON_SERVICE_CONTAINER = "/validator-keys"
+VALIDATOR_DATA_DIRPATH_ON_SERVICE_CONTAINER = "/data/lodestar/validator-data"
 # The min/max CPU/memory that the validator node can use
 VALIDATOR_MIN_CPU = 50
 VALIDATOR_MAX_CPU = 300
@@ -97,10 +98,11 @@ def launch(
     extra_validator_params,
     extra_beacon_labels,
     extra_validator_labels,
+    persistent,
     split_mode_enabled=False,
 ):
-    beacon_node_service_name = "{0}".format(service_name)
-    validator_node_service_name = "{0}-{1}".format(
+    beacon_service_name = "{0}".format(service_name)
+    validator_service_name = "{0}-{1}".format(
         service_name, VALIDATOR_SUFFIX_SERVICE_NAME
     )
     log_level = input_parser.get_client_log_level_or_default(
@@ -116,6 +118,7 @@ def launch(
     beacon_config = get_beacon_config(
         launcher.el_cl_genesis_data,
         image,
+        beacon_service_name,
         bootnode_contexts,
         el_client_context,
         log_level,
@@ -127,9 +130,10 @@ def launch(
         snooper_engine_context,
         extra_beacon_params,
         extra_beacon_labels,
+        persistent,
     )
 
-    beacon_service = plan.add_service(beacon_node_service_name, beacon_config)
+    beacon_service = plan.add_service(beacon_service_name, beacon_config)
 
     beacon_http_port = beacon_service.ports[BEACON_HTTP_PORT_ID]
 
@@ -139,7 +143,7 @@ def launch(
 
     # Blobber config
     if blobber_enabled:
-        blobber_service_name = "{0}-{1}".format("blobber", beacon_node_service_name)
+        blobber_service_name = "{0}-{1}".format("blobber", beacon_service_name)
         blobber_config = blobber_launcher.get_config(
             blobber_service_name,
             node_keystore_files,
@@ -163,9 +167,9 @@ def launch(
         v_min_mem = int(v_min_mem) if int(v_min_mem) > 0 else VALIDATOR_MIN_MEMORY
         v_max_mem = int(v_max_mem) if int(v_max_mem) > 0 else VALIDATOR_MAX_MEMORY
         validator_config = get_validator_config(
-            validator_node_service_name,
             launcher.el_cl_genesis_data,
             image,
+            validator_service_name,
             log_level,
             beacon_http_url,
             el_client_context,
@@ -174,12 +178,12 @@ def launch(
             v_max_cpu,
             v_min_mem,
             v_max_mem,
-            validator_node_service_name,
             extra_validator_params,
             extra_validator_labels,
+            persistent,
         )
 
-        plan.add_service(validator_node_service_name, validator_config)
+        plan.add_service(validator_service_name, validator_config)
 
     # TODO(old) add validator availability using the validator API: https://ethereum.github.io/beacon-APIs/?urls.primaryName=v1#/ValidatorRequiredApi | from eth2-merge-kurtosis-module
 
@@ -193,7 +197,7 @@ def launch(
         },
     )
     response = plan.request(
-        recipe=beacon_node_identity_recipe, service_name=beacon_node_service_name
+        recipe=beacon_node_identity_recipe, service_name=beacon_service_name
     )
     beacon_node_enr = response["extract.enr"]
     beacon_multiaddr = response["extract.multiaddr"]
@@ -215,8 +219,8 @@ def launch(
         beacon_service.ip_address,
         HTTP_PORT_NUM,
         nodes_metrics_info,
-        beacon_node_service_name,
-        validator_node_service_name,
+        beacon_service_name,
+        validator_service_name,
         beacon_multiaddr,
         beacon_peer_id,
         snooper_enabled,
@@ -230,6 +234,7 @@ def launch(
 def get_beacon_config(
     el_cl_genesis_data,
     image,
+    service_name,
     bootnode_contexts,
     el_client_context,
     log_level,
@@ -241,6 +246,7 @@ def get_beacon_config(
     snooper_engine_context,
     extra_params,
     extra_labels,
+    persistent,
 ):
     el_client_rpc_url_str = "http://{0}:{1}".format(
         el_client_context.ip_addr,
@@ -264,7 +270,7 @@ def get_beacon_config(
         "--logLevel=" + log_level,
         "--port={0}".format(DISCOVERY_PORT_NUM),
         "--discoveryPort={0}".format(DISCOVERY_PORT_NUM),
-        "--dataDir=" + CONSENSUS_DATA_DIRPATH_ON_SERVICE_CONTAINER,
+        "--dataDir=" + BEACON_DATA_DIRPATH_ON_SERVICE_CONTAINER,
         "--paramsFile="
         + constants.GENESIS_CONFIG_MOUNT_PATH_ON_CONTAINER
         + "/config.yaml",
@@ -306,14 +312,19 @@ def get_beacon_config(
     if len(extra_params) > 0:
         # this is a repeated<proto type>, we convert it into Starlark
         cmd.extend([param for param in extra_params])
+    files = {
+        constants.GENESIS_DATA_MOUNTPOINT_ON_CLIENTS: el_cl_genesis_data.files_artifact_uuid
+    }
 
+    if persistent:
+        files[BEACON_DATA_DIRPATH_ON_SERVICE_CONTAINER] = Directory(
+            persistent_key="data-{0}".format(service_name)
+        )
     return ServiceConfig(
         image=image,
         ports=BEACON_USED_PORTS,
         cmd=cmd,
-        files={
-            constants.GENESIS_DATA_MOUNTPOINT_ON_CLIENTS: el_cl_genesis_data.files_artifact_uuid
-        },
+        files=files,
         private_ip_address_placeholder=PRIVATE_IP_ADDRESS_PLACEHOLDER,
         ready_conditions=cl_node_ready_conditions.get_ready_conditions(
             BEACON_HTTP_PORT_ID
@@ -333,9 +344,9 @@ def get_beacon_config(
 
 
 def get_validator_config(
-    service_name,
     el_cl_genesis_data,
     image,
+    service_name,
     log_level,
     beacon_client_http_url,
     el_client_context,
@@ -344,12 +355,12 @@ def get_validator_config(
     v_max_cpu,
     v_min_mem,
     v_max_mem,
-    validator_node_service_name,
     extra_params,
     extra_labels,
+    persistent,
 ):
     root_dirpath = shared_utils.path_join(
-        CONSENSUS_DATA_DIRPATH_ON_SERVICE_CONTAINER, service_name
+        VALIDATOR_DATA_DIRPATH_ON_SERVICE_CONTAINER, service_name
     )
 
     validator_keys_dirpath = shared_utils.path_join(
@@ -365,7 +376,7 @@ def get_validator_config(
     cmd = [
         "validator",
         "--logLevel=" + log_level,
-        "--dataDir=" + root_dirpath,
+        # "--dataDir=" + root_dirpath,
         "--paramsFile="
         + constants.GENESIS_CONFIG_MOUNT_PATH_ON_CONTAINER
         + "/config.yaml",
@@ -388,14 +399,16 @@ def get_validator_config(
         # this is a repeated<proto type>, we convert it into Starlark
         cmd.extend([param for param in extra_params])
 
+    files = {
+        constants.GENESIS_DATA_MOUNTPOINT_ON_CLIENTS: el_cl_genesis_data.files_artifact_uuid,
+        VALIDATOR_KEYS_MOUNT_DIRPATH_ON_SERVICE_CONTAINER: node_keystore_files.files_artifact_uuid,
+    }
+
     return ServiceConfig(
         image=image,
         ports=VALIDATOR_USED_PORTS,
         cmd=cmd,
-        files={
-            constants.GENESIS_DATA_MOUNTPOINT_ON_CLIENTS: el_cl_genesis_data.files_artifact_uuid,
-            VALIDATOR_KEYS_MOUNT_DIRPATH_ON_SERVICE_CONTAINER: node_keystore_files.files_artifact_uuid,
-        },
+        files=files,
         private_ip_address_placeholder=PRIVATE_IP_ADDRESS_PLACEHOLDER,
         min_cpu=v_min_cpu,
         max_cpu=v_max_cpu,

src/cl/nimbus/nimbus_launcher.star

@@ -51,7 +51,7 @@ VALIDATOR_METRICS_PATH = "/metrics"
 
 # Nimbus requires that its data directory already exists (because it expects you to bind-mount it), so we
 #  have to to create it
-CONSENSUS_DATA_DIRPATH_IN_SERVICE_CONTAINER = "$HOME/consensus-data"
+BEACON_DATA_DIRPATH_ON_SERVICE_CONTAINER = "/data/beacon-data"
 # Nimbus wants the data dir to have these perms
 CONSENSUS_DATA_DIR_PERMS_STR = "0700"
 
@@ -137,10 +137,11 @@ def launch(
     extra_validator_params,
     extra_beacon_labels,
     extra_validator_labels,
+    persistent,
     split_mode_enabled,
 ):
-    beacon_node_service_name = "{0}".format(service_name)
-    validator_node_service_name = "{0}-{1}".format(
+    beacon_service_name = "{0}".format(service_name)
+    validator_service_name = "{0}-{1}".format(
         service_name, VALIDATOR_SUFFIX_SERVICE_NAME
     )
 
@@ -156,6 +157,7 @@ def launch(
     beacon_config = get_beacon_config(
         launcher.el_cl_genesis_data,
         image,
+        beacon_service_name,
         bootnode_contexts,
         el_client_context,
         log_level,
@@ -164,15 +166,15 @@ def launch(
         bn_max_cpu,
         bn_min_mem,
         bn_max_mem,
-        beacon_node_service_name,
         snooper_enabled,
         snooper_engine_context,
         extra_beacon_params,
         extra_beacon_labels,
         split_mode_enabled,
+        persistent,
     )
 
-    beacon_service = plan.add_service(beacon_node_service_name, beacon_config)
+    beacon_service = plan.add_service(beacon_service_name, beacon_config)
     beacon_http_port = beacon_service.ports[BEACON_HTTP_PORT_ID]
     beacon_metrics_port = beacon_service.ports[BEACON_METRICS_PORT_ID]
     beacon_http_url = "http://{0}:{1}".format(
@@ -214,6 +216,7 @@ def launch(
         validator_config = get_validator_config(
             launcher.el_cl_genesis_data,
             image,
+            validator_service_name,
             log_level,
             beacon_http_url,
             el_client_context,
@@ -222,14 +225,12 @@ def launch(
             v_max_cpu,
             v_min_mem,
             v_max_mem,
-            validator_node_service_name,
             extra_validator_params,
             extra_validator_labels,
+            persistent,
         )
 
-        validator_service = plan.add_service(
-            validator_node_service_name, validator_config
-        )
+        validator_service = plan.add_service(validator_service_name, validator_config)
 
     if validator_service:
         validator_metrics_port = validator_service.ports[VALIDATOR_METRICS_PORT_ID]
@@ -237,7 +238,7 @@ def launch(
             validator_service.ip_address, validator_metrics_port.number
         )
         validator_node_metrics_info = node_metrics.new_node_metrics_info(
-            validator_node_service_name, VALIDATOR_METRICS_PATH, validator_metrics_url
+            validator_service_name, VALIDATOR_METRICS_PATH, validator_metrics_url
         )
         nodes_metrics_info.append(validator_node_metrics_info)
 
@@ -247,8 +248,8 @@ def launch(
         beacon_service.ip_address,
         BEACON_HTTP_PORT_NUM,
         nodes_metrics_info,
-        beacon_node_service_name,
-        validator_node_service_name,
+        beacon_service_name,
+        validator_service_name,
         beacon_multiaddr,
         beacon_peer_id,
         snooper_enabled,
@@ -262,6 +263,7 @@ def launch(
 def get_beacon_config(
     el_cl_genesis_data,
     image,
+    service_name,
     bootnode_contexts,
     el_client_context,
     log_level,
@@ -270,12 +272,12 @@ def get_beacon_config(
     bn_max_cpu,
     bn_min_mem,
     bn_max_mem,
-    beacon_node_service_name,
     snooper_enabled,
     snooper_engine_context,
     extra_params,
     extra_labels,
     split_mode_enabled,
+    persistent,
 ):
     validator_keys_dirpath = ""
     validator_secrets_dirpath = ""
@@ -306,7 +308,7 @@ def get_beacon_config(
         "--udp-port={0}".format(BEACON_DISCOVERY_PORT_NUM),
         "--tcp-port={0}".format(BEACON_DISCOVERY_PORT_NUM),
         "--network=" + constants.GENESIS_CONFIG_MOUNT_PATH_ON_CONTAINER,
-        "--data-dir=" + CONSENSUS_DATA_DIRPATH_IN_SERVICE_CONTAINER,
+        "--data-dir=" + BEACON_DATA_DIRPATH_ON_SERVICE_CONTAINER,
         "--web3-url=" + EXECUTION_ENGINE_ENDPOINT,
         "--nat=extip:" + PRIVATE_IP_ADDRESS_PLACEHOLDER,
         "--enr-auto-update=false",
@@ -364,6 +366,11 @@ def get_beacon_config(
             VALIDATOR_KEYS_MOUNTPOINT_ON_CLIENTS
         ] = node_keystore_files.files_artifact_uuid
 
+    if persistent:
+        files[BEACON_DATA_DIRPATH_ON_SERVICE_CONTAINER] = Directory(
+            persistent_key="data-{0}".format(service_name)
+        )
+
     return ServiceConfig(
         image=image,
         ports=BEACON_USED_PORTS,
@@ -390,6 +397,7 @@ def get_beacon_config(
 def get_validator_config(
     el_cl_genesis_data,
     image,
+    service_name,
     log_level,
     beacon_http_url,
     el_client_context,
@@ -398,9 +406,9 @@ def get_validator_config(
     v_max_cpu,
     v_min_mem,
     v_max_mem,
-    validator_node_service_name,
     extra_params,
     extra_labels,
+    persistent,
 ):
     validator_keys_dirpath = ""
     validator_secrets_dirpath = ""
@@ -432,14 +440,16 @@ def get_validator_config(
     if len(extra_params) > 0:
         cmd.extend([param for param in extra_params if param != "--split=true"])
 
+    files = {
+        VALIDATOR_KEYS_MOUNTPOINT_ON_CLIENTS: node_keystore_files.files_artifact_uuid,
+    }
+
     return ServiceConfig(
         image=image,
         ports=VALIDATOR_USED_PORTS,
         cmd=cmd,
         entrypoint=DEFAULT_VALIDATOR_IMAGE_ENTRYPOINT,
-        files={
-            VALIDATOR_KEYS_MOUNTPOINT_ON_CLIENTS: node_keystore_files.files_artifact_uuid
-        },
+        files=files,
         private_ip_address_placeholder=PRIVATE_IP_ADDRESS_PLACEHOLDER,
         min_cpu=v_min_cpu,
         max_cpu=v_max_cpu,

src/cl/prysm/prysm_launcher.star

@@ -8,7 +8,7 @@ IMAGE_SEPARATOR_DELIMITER = ","
 EXPECTED_NUM_IMAGES = 2
 
 #  ---------------------------------- Beacon client -------------------------------------
-CONSENSUS_DATA_DIRPATH_ON_SERVICE_CONTAINER = "/consensus-data"
+BEACON_DATA_DIRPATH_ON_SERVICE_CONTAINER = "/data/prysm/beacon-data/"
 
 # Port IDs
 TCP_DISCOVERY_PORT_ID = "tcp-discovery"
@@ -31,6 +31,7 @@ BEACON_MIN_MEMORY = 256
 BEACON_MAX_MEMORY = 1024
 
 #  ---------------------------------- Validator client -------------------------------------
+VALIDATOR_DATA_DIRPATH_ON_SERVICE_CONTAINER = "/data/prysm/validator-data/"
 VALIDATOR_KEYS_MOUNT_DIRPATH_ON_SERVICE_CONTAINER = "/validator-keys"
 PRYSM_PASSWORD_MOUNT_DIRPATH_ON_SERVICE_CONTAINER = "/prysm-password"
 
@@ -109,6 +110,7 @@ def launch(
     extra_validator_params,
     extra_beacon_labels,
     extra_validator_labels,
+    persistent,
     split_mode_enabled=False,
 ):
     split_images = images.split(IMAGE_SEPARATOR_DELIMITER)
@@ -126,8 +128,8 @@ def launch(
     if validator_image.strip() == "":
         fail("An empty validator image was provided")
 
-    beacon_node_service_name = "{0}".format(service_name)
-    validator_node_service_name = "{0}-{1}".format(
+    beacon_service_name = "{0}".format(service_name)
+    validator_service_name = "{0}-{1}".format(
         service_name, VALIDATOR_SUFFIX_SERVICE_NAME
     )
     log_level = input_parser.get_client_log_level_or_default(
@@ -142,6 +144,7 @@ def launch(
     beacon_config = get_beacon_config(
         launcher.el_cl_genesis_data,
         beacon_image,
+        beacon_service_name,
         bootnode_contexts,
         el_client_context,
         log_level,
@@ -153,9 +156,10 @@ def launch(
         snooper_engine_context,
         extra_beacon_params,
         extra_beacon_labels,
+        persistent,
     )
 
-    beacon_service = plan.add_service(beacon_node_service_name, beacon_config)
+    beacon_service = plan.add_service(beacon_service_name, beacon_config)
 
     beacon_http_port = beacon_service.ports[BEACON_HTTP_PORT_ID]
 
@@ -172,7 +176,7 @@ def launch(
         validator_config = get_validator_config(
             launcher.el_cl_genesis_data,
             validator_image,
-            validator_node_service_name,
+            validator_service_name,
             log_level,
             beacon_rpc_endpoint,
             beacon_http_endpoint,
@@ -186,11 +190,10 @@ def launch(
             extra_validator_labels,
             launcher.prysm_password_relative_filepath,
             launcher.prysm_password_artifact_uuid,
+            persistent,
         )
 
-        validator_service = plan.add_service(
-            validator_node_service_name, validator_config
-        )
+        validator_service = plan.add_service(validator_service_name, validator_config)
 
     # TODO(old) add validator availability using the validator API: https://ethereum.github.io/beacon-APIs/?urls.primaryName=v1#/ValidatorRequiredApi | from eth2-merge-kurtosis-module
     beacon_node_identity_recipe = GetHttpRequestRecipe(
@@ -203,7 +206,7 @@ def launch(
         },
     )
     response = plan.request(
-        recipe=beacon_node_identity_recipe, service_name=beacon_node_service_name
+        recipe=beacon_node_identity_recipe, service_name=beacon_service_name
     )
     beacon_node_enr = response["extract.enr"]
     beacon_multiaddr = response["extract.multiaddr"]
@@ -214,7 +217,7 @@ def launch(
         beacon_service.ip_address, beacon_metrics_port.number
     )
     beacon_node_metrics_info = node_metrics.new_node_metrics_info(
-        beacon_node_service_name, METRICS_PATH, beacon_metrics_url
+        beacon_service_name, METRICS_PATH, beacon_metrics_url
     )
     nodes_metrics_info = [beacon_node_metrics_info]
 
@@ -224,7 +227,7 @@ def launch(
             validator_service.ip_address, validator_metrics_port.number
         )
         validator_node_metrics_info = node_metrics.new_node_metrics_info(
-            validator_node_service_name, METRICS_PATH, validator_metrics_url
+            validator_service_name, METRICS_PATH, validator_metrics_url
         )
         nodes_metrics_info.append(validator_node_metrics_info)
 
@@ -234,8 +237,8 @@ def launch(
         beacon_service.ip_address,
         HTTP_PORT_NUM,
         nodes_metrics_info,
-        beacon_node_service_name,
-        validator_node_service_name,
+        beacon_service_name,
+        validator_service_name,
         beacon_multiaddr,
         beacon_peer_id,
         snooper_enabled,
@@ -249,6 +252,7 @@ def launch(
 def get_beacon_config(
     el_cl_genesis_data,
     beacon_image,
+    service_name,
     bootnode_contexts,
     el_client_context,
     log_level,
@@ -260,6 +264,7 @@ def get_beacon_config(
     snooper_engine_context,
     extra_params,
     extra_labels,
+    persistent,
 ):
     # If snooper is enabled use the snooper engine context, otherwise use the execution client context
     if snooper_enabled:
@@ -275,7 +280,7 @@ def get_beacon_config(
 
     cmd = [
         "--accept-terms-of-use=true",  # it's mandatory in order to run the node
-        "--datadir=" + CONSENSUS_DATA_DIRPATH_ON_SERVICE_CONTAINER,
+        "--datadir=" + BEACON_DATA_DIRPATH_ON_SERVICE_CONTAINER,
         "--chain-config-file="
         + constants.GENESIS_CONFIG_MOUNT_PATH_ON_CONTAINER
         + "/config.yaml",
@@ -316,13 +321,20 @@ def get_beacon_config(
         # we do the for loop as otherwise its a proto repeated array
         cmd.extend([param for param in extra_params])
 
+    files = {
+        constants.GENESIS_DATA_MOUNTPOINT_ON_CLIENTS: el_cl_genesis_data.files_artifact_uuid
+    }
+
+    if persistent:
+        files[BEACON_DATA_DIRPATH_ON_SERVICE_CONTAINER] = Directory(
+            persistent_key="data-{0}".format(service_name)
+        )
+
     return ServiceConfig(
         image=beacon_image,
         ports=BEACON_NODE_USED_PORTS,
         cmd=cmd,
-        files={
-            constants.GENESIS_DATA_MOUNTPOINT_ON_CLIENTS: el_cl_genesis_data.files_artifact_uuid,
-        },
+        files=files,
         private_ip_address_placeholder=PRIVATE_IP_ADDRESS_PLACEHOLDER,
         ready_conditions=cl_node_ready_conditions.get_ready_conditions(
             BEACON_HTTP_PORT_ID
@@ -358,6 +370,7 @@ def get_validator_config(
     extra_labels,
     prysm_password_relative_filepath,
     prysm_password_artifact_uuid,
+    persistent,
 ):
     validator_keys_dirpath = shared_utils.path_join(
         VALIDATOR_KEYS_MOUNT_DIRPATH_ON_SERVICE_CONTAINER,
@@ -377,7 +390,7 @@ def get_validator_config(
         "--beacon-rpc-provider=" + beacon_rpc_endpoint,
         "--wallet-dir=" + validator_keys_dirpath,
         "--wallet-password-file=" + validator_secrets_dirpath,
-        "--datadir=" + CONSENSUS_DATA_DIRPATH_ON_SERVICE_CONTAINER,
+        # "--datadir=" + VALIDATOR_DATA_DIRPATH_ON_SERVICE_CONTAINER,
         "--monitoring-port={0}".format(VALIDATOR_MONITORING_PORT_NUM),
         "--verbosity=" + log_level,
         "--suggested-fee-recipient=" + constants.VALIDATING_REWARDS_ACCOUNT,
@@ -396,16 +409,21 @@ def get_validator_config(
     if len(extra_params) > 0:
         # we do the for loop as otherwise its a proto repeated array
         cmd.extend([param for param in extra_params])
+    files = {
+        constants.GENESIS_DATA_MOUNTPOINT_ON_CLIENTS: el_cl_genesis_data.files_artifact_uuid,
+        VALIDATOR_KEYS_MOUNT_DIRPATH_ON_SERVICE_CONTAINER: node_keystore_files.files_artifact_uuid,
+        PRYSM_PASSWORD_MOUNT_DIRPATH_ON_SERVICE_CONTAINER: prysm_password_artifact_uuid,
+    }
+    if persistent:
+        files[VALIDATOR_DATA_DIRPATH_ON_SERVICE_CONTAINER] = Directory(
+            persistent_key="data-{0}".format(service_name)
+        )
 
     return ServiceConfig(
         image=validator_image,
         ports=VALIDATOR_NODE_USED_PORTS,
         cmd=cmd,
-        files={
-            constants.GENESIS_DATA_MOUNTPOINT_ON_CLIENTS: el_cl_genesis_data.files_artifact_uuid,
-            VALIDATOR_KEYS_MOUNT_DIRPATH_ON_SERVICE_CONTAINER: node_keystore_files.files_artifact_uuid,
-            PRYSM_PASSWORD_MOUNT_DIRPATH_ON_SERVICE_CONTAINER: prysm_password_artifact_uuid,
-        },
+        files=files,
         private_ip_address_placeholder=PRIVATE_IP_ADDRESS_PLACEHOLDER,
         min_cpu=v_min_cpu,
         max_cpu=v_max_cpu,

src/cl/teku/teku_launcher.star

@@ -7,14 +7,10 @@ cl_node_ready_conditions = import_module("../../cl/cl_node_ready_conditions.star
 constants = import_module("../../package_io/constants.star")
 TEKU_BINARY_FILEPATH_IN_IMAGE = "/opt/teku/bin/teku"
 
+#  ---------------------------------- Beacon client -------------------------------------
 # The Docker container runs as the "teku" user so we can't write to root
-CONSENSUS_DATA_DIRPATH_ON_SERVICE_CONTAINER = "/opt/teku/consensus-data"
+BEACON_DATA_DIRPATH_ON_SERVICE_CONTAINER = "/opt/teku/teku-beacon-data"
 
-# These will get mounted as root and Teku needs directory write permissions, so we'll copy this
-#  into the Teku user's home directory to get around it
-VALIDATOR_KEYS_DIRPATH_ON_SERVICE_CONTAINER = "/validator-keys"
-
-#  ---------------------------------- Beacon client -------------------------------------
 # Port IDs
 BEACON_TCP_DISCOVERY_PORT_ID = "tcp-discovery"
 BEACON_UDP_DISCOVERY_PORT_ID = "udp-discovery"
@@ -34,6 +30,12 @@ BEACON_MAX_MEMORY = 2048
 
 BEACON_METRICS_PATH = "/metrics"
 #  ---------------------------------- Validator client -------------------------------------
+# These will get mounted as root and Teku needs directory write permissions, so we'll copy this
+#  into the Teku user's home directory to get around it
+VALIDATOR_DATA_DIRPATH_ON_SERVICE_CONTAINER = "/opt/teku/teku-validator-data"
+
+VALIDATOR_KEYS_DIRPATH_ON_SERVICE_CONTAINER = "/validator-keys"
+
 VALIDATOR_KEYS_MOUNTPOINT_ON_CLIENTS = "/validator-keys"
 VALIDATOR_HTTP_PORT_ID = "http"
 VALIDATOR_METRICS_PORT_ID = "metrics"
@@ -122,6 +124,7 @@ def launch(
     extra_validator_params,
     extra_beacon_labels,
     extra_validator_labels,
+    persistent,
     split_mode_enabled,
 ):
     beacon_service_name = "{0}".format(service_name)
@@ -144,6 +147,7 @@ def launch(
     config = get_beacon_config(
         launcher.el_cl_genesis_data,
         image,
+        beacon_service_name,
         bootnode_context,
         el_client_context,
         log_level,
@@ -154,10 +158,10 @@ def launch(
         bn_max_mem,
         snooper_enabled,
         snooper_engine_context,
-        beacon_service_name,
         extra_beacon_params,
         extra_beacon_labels,
         split_mode_enabled,
+        persistent,
     )
 
     beacon_service = plan.add_service(service_name, config)
@@ -204,6 +208,7 @@ def launch(
         validator_config = get_validator_config(
             launcher.el_cl_genesis_data,
             image,
+            validator_service_name,
             log_level,
             beacon_http_url,
             el_client_context,
@@ -215,6 +220,7 @@ def launch(
             validator_service_name,
             extra_validator_params,
             extra_validator_labels,
+            persistent,
         )
 
         validator_service = plan.add_service(validator_service_name, validator_config)
@@ -250,6 +256,7 @@ def launch(
 def get_beacon_config(
     el_cl_genesis_data,
     image,
+    service_name,
     bootnode_contexts,
     el_client_context,
     log_level,
@@ -260,10 +267,10 @@ def get_beacon_config(
     bn_max_mem,
     snooper_enabled,
     snooper_engine_context,
-    service_name,
     extra_params,
     extra_labels,
     split_mode_enabled,
+    persistent,
 ):
     validator_keys_dirpath = ""
     validator_secrets_dirpath = ""
@@ -296,7 +303,7 @@ def get_beacon_config(
         "--initial-state="
         + constants.GENESIS_CONFIG_MOUNT_PATH_ON_CONTAINER
         + "/genesis.ssz",
-        "--data-path=" + CONSENSUS_DATA_DIRPATH_ON_SERVICE_CONTAINER,
+        "--data-path=" + BEACON_DATA_DIRPATH_ON_SERVICE_CONTAINER,
         "--data-storage-mode={0}".format(
             "ARCHIVE" if constants.ARCHIVE_MODE else "PRUNE"
         ),
@@ -366,7 +373,11 @@ def get_beacon_config(
         files[
             VALIDATOR_KEYS_DIRPATH_ON_SERVICE_CONTAINER
         ] = node_keystore_files.files_artifact_uuid
-    cmd_str = " ".join(cmd)
+
+    if persistent:
+        files[BEACON_DATA_DIRPATH_ON_SERVICE_CONTAINER] = Directory(
+            persistent_key="data-{0}".format(service_name)
+        )
     return ServiceConfig(
         image=image,
         ports=BEACON_USED_PORTS,
@@ -394,6 +405,7 @@ def get_beacon_config(
 def get_validator_config(
     el_cl_genesis_data,
     image,
+    service_name,
     log_level,
     beacon_http_url,
     el_client_context,
@@ -405,6 +417,7 @@ def get_validator_config(
     validator_service_name,
     extra_params,
     extra_labels,
+    persistent,
 ):
     validator_keys_dirpath = ""
     validator_secrets_dirpath = ""
@@ -424,8 +437,8 @@ def get_validator_config(
         "--network="
         + constants.GENESIS_CONFIG_MOUNT_PATH_ON_CONTAINER
         + "/config.yaml",
-        "--data-path=" + CONSENSUS_DATA_DIRPATH_ON_SERVICE_CONTAINER,
-        "--data-validator-path=" + CONSENSUS_DATA_DIRPATH_ON_SERVICE_CONTAINER,
+        # "--data-path=" + VALIDATOR_DATA_DIRPATH_ON_SERVICE_CONTAINER,
+        # "--data-validator-path=" + VALIDATOR_DATA_DIRPATH_ON_SERVICE_CONTAINER,
         "--beacon-node-api-endpoint=" + beacon_http_url,
         "--validator-keys={0}:{1}".format(
             validator_keys_dirpath,
@@ -447,14 +460,16 @@ def get_validator_config(
     if len(extra_params) > 0:
         cmd.extend([param for param in extra_params if param != "--split=true"])
 
+    files = {
+        constants.GENESIS_DATA_MOUNTPOINT_ON_CLIENTS: el_cl_genesis_data.files_artifact_uuid,
+        VALIDATOR_KEYS_MOUNTPOINT_ON_CLIENTS: node_keystore_files.files_artifact_uuid,
+    }
+
     return ServiceConfig(
         image=image,
         ports=VALIDATOR_USED_PORTS,
         cmd=cmd,
-        files={
-            constants.GENESIS_DATA_MOUNTPOINT_ON_CLIENTS: el_cl_genesis_data.files_artifact_uuid,
-            VALIDATOR_KEYS_MOUNTPOINT_ON_CLIENTS: node_keystore_files.files_artifact_uuid,
-        },
+        files=files,
         private_ip_address_placeholder=PRIVATE_IP_ADDRESS_PLACEHOLDER,
         min_cpu=v_min_cpu,
         max_cpu=v_max_cpu,

src/el/besu/besu_launcher.star

@@ -75,6 +75,7 @@ def launch(
     extra_params,
     extra_env_vars,
     extra_labels,
+    persistent,
 ):
     log_level = input_parser.get_client_log_level_or_default(
         participant_log_level, global_log_level, BESU_LOG_LEVELS
@@ -91,6 +92,7 @@ def launch(
         launcher.network_id,
         launcher.el_cl_genesis_data,
         image,
+        service_name,
         existing_el_clients,
         cl_client_name,
         log_level,
@@ -101,6 +103,7 @@ def launch(
         extra_params,
         extra_env_vars,
         extra_labels,
+        persistent,
     )
 
     service = plan.add_service(service_name, config)
@@ -129,6 +132,7 @@ def get_config(
     network_id,
     el_cl_genesis_data,
     image,
+    service_name,
     existing_el_clients,
     cl_client_name,
     log_level,
@@ -139,6 +143,7 @@ def get_config(
     extra_params,
     extra_env_vars,
     extra_labels,
+    persistent,
 ):
     cmd = [
         "besu",
@@ -190,13 +195,20 @@ def get_config(
     cmd_str = " ".join(cmd)
 
     extra_env_vars = extra_env_vars | JAVA_OPTS
+
+    files = {
+        constants.GENESIS_DATA_MOUNTPOINT_ON_CLIENTS: el_cl_genesis_data.files_artifact_uuid,
+    }
+
+    if persistent:
+        files[EXECUTION_DATA_DIRPATH_ON_CLIENT_CONTAINER] = Directory(
+            persistent_key="data-{0}".format(service_name),
+        )
     return ServiceConfig(
         image=image,
         ports=USED_PORTS,
         cmd=[cmd_str],
-        files={
-            constants.GENESIS_DATA_MOUNTPOINT_ON_CLIENTS: el_cl_genesis_data.files_artifact_uuid,
-        },
+        files=files,
         env_vars=extra_env_vars,
         entrypoint=ENTRYPOINT_ARGS,
         private_ip_address_placeholder=PRIVATE_IP_ADDRESS_PLACEHOLDER,

src/el/erigon/erigon_launcher.star

@@ -76,6 +76,7 @@ def launch(
     extra_params,
     extra_env_vars,
     extra_labels,
+    persistent,
 ):
     log_level = input_parser.get_client_log_level_or_default(
         participant_log_level, global_log_level, ERIGON_LOG_LEVELS
@@ -92,6 +93,7 @@ def launch(
         launcher.network_id,
         launcher.el_cl_genesis_data,
         image,
+        service_name,
         existing_el_clients,
         cl_client_name,
         log_level,
@@ -102,6 +104,7 @@ def launch(
         extra_params,
         extra_env_vars,
         extra_labels,
+        persistent,
     )
 
     service = plan.add_service(service_name, config)
@@ -132,6 +135,7 @@ def get_config(
     network_id,
     el_cl_genesis_data,
     image,
+    service_name,
     existing_el_clients,
     cl_client_name,
     verbosity_level,
@@ -142,6 +146,7 @@ def get_config(
     extra_params,
     extra_env_vars,
     extra_labels,
+    persistent,
 ):
     network_id = network_id
 
@@ -201,14 +206,19 @@ def get_config(
     command_arg = [init_datadir_cmd_str, " ".join(cmd)]
 
     command_arg_str = " && ".join(command_arg)
+    files = {
+        constants.GENESIS_DATA_MOUNTPOINT_ON_CLIENTS: el_cl_genesis_data.files_artifact_uuid,
+    }
 
+    if persistent:
+        files[EXECUTION_DATA_DIRPATH_ON_CLIENT_CONTAINER] = Directory(
+            persistent_key="data-{0}".format(service_name),
+        )
     return ServiceConfig(
         image=image,
         ports=USED_PORTS,
         cmd=[command_arg_str],
-        files={
-            constants.GENESIS_DATA_MOUNTPOINT_ON_CLIENTS: el_cl_genesis_data.files_artifact_uuid,
-        },
+        files=files,
         entrypoint=ENTRYPOINT_ARGS,
         private_ip_address_placeholder=PRIVATE_IP_ADDRESS_PLACEHOLDER,
         min_cpu=el_min_cpu,

src/el/ethereumjs/ethereumjs_launcher.star

@@ -31,7 +31,7 @@ METRICS_PORT_ID = "metrics"
 METRICS_PATH = "/metrics"
 
 # The dirpath of the execution data directory on the client container
-EXECUTION_DATA_DIRPATH_ON_CLIENT_CONTAINER = "/execution-data"
+EXECUTION_DATA_DIRPATH_ON_CLIENT_CONTAINER = "/data/ethereumjs/execution-data"
 
 PRIVATE_IP_ADDRESS_PLACEHOLDER = "KURTOSIS_IP_ADDR_PLACEHOLDER"
 
@@ -80,6 +80,7 @@ def launch(
     extra_params,
     extra_env_vars,
     extra_labels,
+    persistent,
 ):
     log_level = input_parser.get_client_log_level_or_default(
         participant_log_level, global_log_level, VERBOSITY_LEVELS
@@ -95,6 +96,7 @@ def launch(
     config = get_config(
         launcher.el_cl_genesis_data,
         image,
+        service_name,
         existing_el_clients,
         cl_client_name,
         log_level,
@@ -105,6 +107,7 @@ def launch(
         extra_params,
         extra_env_vars,
         extra_labels,
+        persistent,
     )
 
     service = plan.add_service(service_name, config)
@@ -131,6 +134,7 @@ def launch(
 def get_config(
     el_cl_genesis_data,
     image,
+    service_name,
     existing_el_clients,
     cl_client_name,
     verbosity_level,
@@ -141,6 +145,7 @@ def get_config(
     extra_params,
     extra_env_vars,
     extra_labels,
+    persistent,
 ):
     cmd = [
         "--gethGenesis="
@@ -182,13 +187,19 @@ def get_config(
         # this is a repeated<proto type>, we convert it into Starlark
         cmd.extend([param for param in extra_params])
 
+    files = {
+        constants.GENESIS_DATA_MOUNTPOINT_ON_CLIENTS: el_cl_genesis_data.files_artifact_uuid,
+    }
+
+    if persistent:
+        files[EXECUTION_DATA_DIRPATH_ON_CLIENT_CONTAINER] = Directory(
+            persistent_key="data-{0}".format(service_name),
+        )
     return ServiceConfig(
         image=image,
         ports=USED_PORTS,
         cmd=cmd,
-        files={
-            constants.GENESIS_DATA_MOUNTPOINT_ON_CLIENTS: el_cl_genesis_data.files_artifact_uuid,
-        },
+        files=files,
         entrypoint=ENTRYPOINT_ARGS,
         private_ip_address_placeholder=PRIVATE_IP_ADDRESS_PLACEHOLDER,
         min_cpu=el_min_cpu,

src/el/geth/geth_launcher.star

@@ -36,7 +36,7 @@ NUM_MINING_THREADS = 1
 METRICS_PATH = "/debug/metrics/prometheus"
 
 # The dirpath of the execution data directory on the client container
-EXECUTION_DATA_DIRPATH_ON_CLIENT_CONTAINER = "/execution-data"
+EXECUTION_DATA_DIRPATH_ON_CLIENT_CONTAINER = "/data/geth/execution-data"
 
 PRIVATE_IP_ADDRESS_PLACEHOLDER = "KURTOSIS_IP_ADDR_PLACEHOLDER"
 
@@ -86,6 +86,7 @@ def launch(
     extra_params,
     extra_env_vars,
     extra_labels,
+    persistent,
 ):
     log_level = input_parser.get_client_log_level_or_default(
         participant_log_level, global_log_level, VERBOSITY_LEVELS
@@ -101,6 +102,7 @@ def launch(
         launcher.network_id,
         launcher.el_cl_genesis_data,
         image,
+        service_name,
         existing_el_clients,
         cl_client_name,
         log_level,
@@ -114,6 +116,7 @@ def launch(
         launcher.capella_fork_epoch,
         launcher.electra_fork_epoch,
         launcher.final_genesis_timestamp,
+        persistent,
     )
 
     service = plan.add_service(service_name, config)
@@ -144,6 +147,7 @@ def get_config(
     network_id,
     el_cl_genesis_data,
     image,
+    service_name,
     existing_el_clients,
     cl_client_name,
     verbosity_level,
@@ -157,6 +161,7 @@ def get_config(
     capella_fork_epoch,
     electra_fork_epoch,
     final_genesis_timestamp,
+    persistent,
 ):
     # TODO: Remove this once electra fork has path based storage scheme implemented
     if electra_fork_epoch != None:
@@ -263,13 +268,18 @@ def get_config(
     ]
     command_str = " && ".join(subcommand_strs)
 
+    files = {
+        constants.GENESIS_DATA_MOUNTPOINT_ON_CLIENTS: el_cl_genesis_data.files_artifact_uuid
+    }
+    if persistent:
+        files[EXECUTION_DATA_DIRPATH_ON_CLIENT_CONTAINER] = Directory(
+            persistent_key="data-{0}".format(service_name),
+        )
     return ServiceConfig(
         image=image,
         ports=USED_PORTS,
         cmd=[command_str],
-        files={
-            constants.GENESIS_DATA_MOUNTPOINT_ON_CLIENTS: el_cl_genesis_data.files_artifact_uuid
-        },
+        files=files,
         entrypoint=ENTRYPOINT_ARGS,
         private_ip_address_placeholder=PRIVATE_IP_ADDRESS_PLACEHOLDER,
         min_cpu=el_min_cpu,

src/el/nethermind/nethermind_launcher.star

@@ -7,7 +7,7 @@ node_metrics = import_module("../../node_metrics_info.star")
 constants = import_module("../../package_io/constants.star")
 
 # The dirpath of the execution data directory on the client container
-EXECUTION_DATA_DIRPATH_ON_CLIENT_CONTAINER = "/execution-data"
+EXECUTION_DATA_DIRPATH_ON_CLIENT_CONTAINER = "/data/nethermind/execution-data"
 
 METRICS_PATH = "/metrics"
 
@@ -74,6 +74,7 @@ def launch(
     extra_params,
     extra_env_vars,
     extra_labels,
+    persistent,
 ):
     log_level = input_parser.get_client_log_level_or_default(
         participant_log_level, global_log_level, NETHERMIND_LOG_LEVELS
@@ -89,6 +90,7 @@ def launch(
     config = get_config(
         launcher.el_cl_genesis_data,
         image,
+        service_name,
         existing_el_clients,
         cl_client_name,
         log_level,
@@ -99,6 +101,7 @@ def launch(
         extra_params,
         extra_env_vars,
         extra_labels,
+        persistent,
     )
 
     service = plan.add_service(service_name, config)
@@ -127,6 +130,7 @@ def launch(
 def get_config(
     el_cl_genesis_data,
     image,
+    service_name,
     existing_el_clients,
     cl_client_name,
     log_level,
@@ -137,6 +141,7 @@ def get_config(
     extra_params,
     extra_env_vars,
     extra_labels,
+    persistent,
 ):
     cmd = [
         "--log=" + log_level,
@@ -177,13 +182,20 @@ def get_config(
         # this is a repeated<proto type>, we convert it into Starlark
         cmd.extend([param for param in extra_params])
 
+    files = {
+        constants.GENESIS_DATA_MOUNTPOINT_ON_CLIENTS: el_cl_genesis_data.files_artifact_uuid,
+    }
+
+    if persistent:
+        files[EXECUTION_DATA_DIRPATH_ON_CLIENT_CONTAINER] = Directory(
+            persistent_key="data-{0}".format(service_name),
+        )
+
     return ServiceConfig(
         image=image,
         ports=USED_PORTS,
         cmd=cmd,
-        files={
-            constants.GENESIS_DATA_MOUNTPOINT_ON_CLIENTS: el_cl_genesis_data.files_artifact_uuid,
-        },
+        files=files,
         private_ip_address_placeholder=PRIVATE_IP_ADDRESS_PLACEHOLDER,
         min_cpu=el_min_cpu,
         max_cpu=el_max_cpu,

src/el/reth/reth_launcher.star

@@ -30,7 +30,7 @@ METRICS_PORT_ID = "metrics"
 METRICS_PATH = "/metrics"
 
 # The dirpath of the execution data directory on the client container
-EXECUTION_DATA_DIRPATH_ON_CLIENT_CONTAINER = "/execution-data"
+EXECUTION_DATA_DIRPATH_ON_CLIENT_CONTAINER = "/data/reth/execution-data"
 
 PRIVATE_IP_ADDRESS_PLACEHOLDER = "KURTOSIS_IP_ADDR_PLACEHOLDER"
 
@@ -78,6 +78,7 @@ def launch(
     extra_params,
     extra_env_vars,
     extra_labels,
+    persistent,
 ):
     log_level = input_parser.get_client_log_level_or_default(
         participant_log_level, global_log_level, VERBOSITY_LEVELS
@@ -93,6 +94,7 @@ def launch(
     config = get_config(
         launcher.el_cl_genesis_data,
         image,
+        service_name,
         existing_el_clients,
         cl_client_name,
         log_level,
@@ -103,6 +105,7 @@ def launch(
         extra_params,
         extra_env_vars,
         extra_labels,
+        persistent,
     )
 
     service = plan.add_service(service_name, config)
@@ -130,6 +133,7 @@ def launch(
 def get_config(
     el_cl_genesis_data,
     image,
+    service_name,
     existing_el_clients,
     cl_client_name,
     verbosity_level,
@@ -140,6 +144,7 @@ def get_config(
     extra_params,
     extra_env_vars,
     extra_labels,
+    persistent,
 ):
     init_datadir_cmd_str = "reth init --datadir={0} --chain={1}".format(
         EXECUTION_DATA_DIRPATH_ON_CLIENT_CONTAINER,
@@ -194,13 +199,19 @@ def get_config(
     ]
     command_str = " && ".join(subcommand_strs)
 
+    files = {
+        constants.GENESIS_DATA_MOUNTPOINT_ON_CLIENTS: el_cl_genesis_data.files_artifact_uuid,
+    }
+
+    if persistent:
+        files[EXECUTION_DATA_DIRPATH_ON_CLIENT_CONTAINER] = Directory(
+            persistent_key="data-{0}".format(service_name),
+        )
     return ServiceConfig(
         image=image,
         ports=USED_PORTS,
         cmd=[command_str],
-        files={
-            constants.GENESIS_DATA_MOUNTPOINT_ON_CLIENTS: el_cl_genesis_data.files_artifact_uuid,
-        },
+        files=files,
         entrypoint=ENTRYPOINT_ARGS,
         private_ip_address_placeholder=PRIVATE_IP_ADDRESS_PLACEHOLDER,
         min_cpu=el_min_cpu,

src/participant_network.star

@@ -56,6 +56,7 @@ def launch_participant_network(
     participants,
     network_params,
     global_log_level,
+    persistent,
     parallel_keystore_generation=False,
 ):
     num_participants = len(participants)
@@ -225,6 +226,7 @@ def launch_participant_network(
             participant.el_extra_params,
             participant.el_extra_env_vars,
             participant.el_extra_labels,
+            persistent,
         )
 
         # Add participant el additional prometheus metrics
@@ -344,6 +346,7 @@ def launch_participant_network(
                 participant.validator_extra_params,
                 participant.beacon_extra_labels,
                 participant.validator_extra_labels,
+                persistent,
                 participant.cl_split_mode_enabled,
             )
         else:
@@ -374,6 +377,7 @@ def launch_participant_network(
                 participant.validator_extra_params,
                 participant.beacon_extra_labels,
                 participant.validator_extra_labels,
+                persistent,
                 participant.cl_split_mode_enabled,
             )