feat: add vc_count to increase the number of validators per participant (#633)

README.md

@@ -344,6 +344,10 @@ participants:
     # - teku: consensys/teku:latest
     vc_image: ""
 
+    # The number of validator clients to run for this participant
+    # Defaults to 1
+    vc_count: 1
+
     # The log level string that this participant's CL client should log at
     # If this is emptystring then the global `logLevel` parameter's value will be translated into a string appropriate for the client (e.g. if
     # global `logLevel` = `info` then Teku would receive `INFO`, Prysm would receive `info`, etc.)

network_params.yaml

@@ -30,6 +30,7 @@ participants:
     vc_type: lighthouse
     vc_image: sigp/lighthouse:latest
     vc_log_level: ""
+    vc_count: 1
     vc_extra_env_vars: {}
     vc_extra_labels: {}
     vc_extra_params: []

src/cl/cl_launcher.star

@@ -119,7 +119,7 @@ def launch(
 
         cl_service_name = "cl-{0}-{1}-{2}".format(index_str, cl_type, el_type)
         new_cl_node_validator_keystores = None
-        if participant.validator_count != 0:
+        if participant.validator_count != 0 and participant.vc_count != 0:
             new_cl_node_validator_keystores = preregistered_validator_keys_for_nodes[
                 index
             ]

src/package_io/input_parser.star

@@ -214,6 +214,7 @@ def input_parser(plan, input_args):
                 vc_type=participant["vc_type"],
                 vc_image=participant["vc_image"],
                 vc_log_level=participant["vc_log_level"],
+                vc_count=participant["vc_count"],
                 vc_tolerations=participant["vc_tolerations"],
                 cl_extra_params=participant["cl_extra_params"],
                 cl_extra_labels=participant["cl_extra_labels"],
@@ -537,6 +538,30 @@ def parse_network_params(plan, input_args):
                 )
             participant["vc_image"] = default_image
 
+        if result["parallel_keystore_generation"] and participant["vc_count"] != 1:
+            fail(
+                "parallel_keystore_generation is only supported for 1 validator client per participant (for now)"
+            )
+
+        # If the num validator keys per node is not divisible by vc_count of a participant, fail
+        if (
+            participant["vc_count"] > 0
+            and result["network_params"]["num_validator_keys_per_node"]
+            % participant["vc_count"]
+            != 0
+        ):
+            fail(
+                "num_validator_keys_per_node: {0} is not divisible by vc_count: {1} for participant: {2}".format(
+                    result["network_params"]["num_validator_keys_per_node"],
+                    participant["vc_count"],
+                    str(index + 1)
+                    + "-"
+                    + participant["el_type"]
+                    + "-"
+                    + participant["cl_type"],
+                )
+            )
+
         snooper_enabled = participant["snooper_enabled"]
         if snooper_enabled == None:
             participant["snooper_enabled"] = result["snooper_enabled"]
@@ -826,6 +851,7 @@ def default_participant():
         "vc_type": "",
         "vc_image": "",
         "vc_log_level": "",
+        "vc_count": 1,
         "vc_extra_env_vars": {},
         "vc_extra_labels": {},
         "vc_extra_params": [],

src/participant_network.star

@@ -210,6 +210,7 @@ def launch_participant_network(
         cl_type = participant.cl_type
         vc_type = participant.vc_type
         index_str = shared_utils.zfill_custom(index + 1, len(str(len(participants))))
+        for sub_index in range(participant.vc_count):
             el_context = all_el_contexts[index]
             cl_context = all_cl_contexts[index]
 
@@ -238,7 +239,9 @@ def launch_participant_network(
                     )
                 )
 
-        all_ethereum_metrics_exporter_contexts.append(ethereum_metrics_exporter_context)
+            all_ethereum_metrics_exporter_contexts.append(
+                ethereum_metrics_exporter_context
+            )
 
             xatu_sentry_context = None
 
@@ -264,7 +267,9 @@ def launch_participant_network(
 
             all_xatu_sentry_contexts.append(xatu_sentry_context)
 
-        plan.print("Successfully added {0} CL participants".format(num_participants))
+            plan.print(
+                "Successfully added {0} CL participants".format(num_participants)
+            )
 
             plan.print("Start adding validators for participant #{0}".format(index_str))
             if participant.use_separate_vc == None:
@@ -274,7 +279,10 @@ def launch_participant_network(
                 all_snooper_beacon_contexts.append(None)
                 continue
 
-        if cl_type in _cls_that_need_separate_vc and not participant.use_separate_vc:
+            if (
+                cl_type in _cls_that_need_separate_vc
+                and not participant.use_separate_vc
+            ):
                 fail("{0} needs a separate validator client!".format(cl_type))
 
             if not participant.use_separate_vc:
@@ -288,14 +296,22 @@ def launch_participant_network(
 
             vc_keystores = None
             if participant.validator_count != 0:
+                if participant.vc_count == 1:
                     vc_keystores = preregistered_validator_keys_for_nodes[index]
+                else:
+                    vc_keystores = preregistered_validator_keys_for_nodes[
+                        index + sub_index
+                    ]
 
             vc_context = None
             snooper_beacon_context = None
 
             if participant.snooper_enabled:
-            snooper_service_name = "snooper-beacon-{0}-{1}-{2}".format(
-                index_str, cl_type, vc_type
+                snooper_service_name = "snooper-beacon-{0}-{1}-{2}{3}".format(
+                    index_str,
+                    cl_type,
+                    vc_type,
+                    "-" + str(sub_index) if participant.vc_count != 1 else "",
                 )
                 snooper_beacon_context = beacon_snooper.launch(
                     plan,
@@ -310,9 +326,20 @@ def launch_participant_network(
                 )
             all_snooper_beacon_contexts.append(snooper_beacon_context)
             full_name = (
-            "{0}-{1}-{2}-{3}".format(index_str, el_type, cl_type, vc_type)
+                "{0}-{1}-{2}-{3}{4}".format(
+                    index_str,
+                    el_type,
+                    cl_type,
+                    vc_type,
+                    "-" + str(sub_index) if participant.vc_count != 1 else "",
+                )
                 if participant.cl_type != participant.vc_type
-            else "{0}-{1}-{2}".format(index_str, el_type, cl_type)
+                else "{0}-{1}-{2}{3}".format(
+                    index_str,
+                    el_type,
+                    cl_type,
+                    "-" + str(sub_index) if participant.vc_count != 1 else "",
+                )
             )
 
             vc_context = vc.launch(
@@ -364,7 +391,10 @@ def launch_participant_network(
 
         el_context = all_el_contexts[index]
         cl_context = all_cl_contexts[index]
+        if participant.vc_count != 0:
             vc_context = all_vc_contexts[index]
+        else:
+            vc_context = None
 
         if participant.snooper_enabled:
             snooper_engine_context = all_snooper_engine_contexts[index]

src/prelaunch_data_generator/validator_keystores/validator_keystore_generator.star

@@ -2,7 +2,7 @@ shared_utils = import_module("../../shared_utils/shared_utils.star")
 keystore_files_module = import_module("./keystore_files.star")
 keystores_result = import_module("./generate_keystores_result.star")
 
-NODE_KEYSTORES_OUTPUT_DIRPATH_FORMAT_STR = "/node-{0}-keystores/"
+NODE_KEYSTORES_OUTPUT_DIRPATH_FORMAT_STR = "/node-{0}-keystores{1}/"
 
 # Prysm keystores are encrypted with a password
 PRYSM_PASSWORD = "password"
@@ -85,14 +85,30 @@ def generate_validator_keystores(plan, mnemonic, participants):
     all_output_dirpaths = []
     all_sub_command_strs = []
     running_total_validator_count = 0
+
     for idx, participant in enumerate(participants):
-        output_dirpath = NODE_KEYSTORES_OUTPUT_DIRPATH_FORMAT_STR.format(idx)
+        output_dirpath = NODE_KEYSTORES_OUTPUT_DIRPATH_FORMAT_STR.format(idx, "")
         if participant.validator_count == 0:
             all_output_dirpaths.append(output_dirpath)
             continue
-        start_index = running_total_validator_count
-        running_total_validator_count += participant.validator_count
-        stop_index = start_index + participant.validator_count
+
+        for i in range(participant.vc_count):
+            output_dirpath = (
+                NODE_KEYSTORES_OUTPUT_DIRPATH_FORMAT_STR.format(idx, "-" + str(i))
+                if participant.vc_count != 1
+                else NODE_KEYSTORES_OUTPUT_DIRPATH_FORMAT_STR.format(idx, "")
+            )
+
+            start_index = running_total_validator_count + i * (
+                participant.validator_count // participant.vc_count
+            )
+            stop_index = start_index + (
+                participant.validator_count // participant.vc_count
+            )
+
+            # Adjust stop_index for the last partition to include all remaining validators
+            if i == participant.vc_count - 1:
+                stop_index = running_total_validator_count + participant.validator_count
 
             generate_keystores_cmd = '{0} keystores --insecure --prysm-pass {1} --out-loc {2} --source-mnemonic "{3}" --source-min {4} --source-max {5}'.format(
                 KEYSTORES_GENERATION_TOOL_NAME,
@@ -102,14 +118,17 @@ def generate_validator_keystores(plan, mnemonic, participants):
                 start_index,
                 stop_index,
             )
+            all_output_dirpaths.append(output_dirpath)
+            all_sub_command_strs.append(generate_keystores_cmd)
+
             teku_permissions_cmd = "chmod 0777 -R " + output_dirpath + TEKU_KEYS_DIRNAME
             raw_secret_permissions_cmd = (
                 "chmod 0600 -R " + output_dirpath + RAW_SECRETS_DIRNAME
             )
-        all_sub_command_strs.append(generate_keystores_cmd)
             all_sub_command_strs.append(teku_permissions_cmd)
             all_sub_command_strs.append(raw_secret_permissions_cmd)
-        all_output_dirpaths.append(output_dirpath)
+
+        running_total_validator_count += participant.validator_count
 
     command_str = " && ".join(all_sub_command_strs)
 
@@ -124,26 +143,42 @@ def generate_validator_keystores(plan, mnemonic, participants):
     keystore_files = []
     running_total_validator_count = 0
     for idx, participant in enumerate(participants):
-        output_dirpath = all_output_dirpaths[idx]
         if participant.validator_count == 0:
             keystore_files.append(None)
             continue
+
+        for i in range(participant.vc_count):
+            output_dirpath = (
+                NODE_KEYSTORES_OUTPUT_DIRPATH_FORMAT_STR.format(idx, "-" + str(i))
+                if participant.vc_count != 1
+                else NODE_KEYSTORES_OUTPUT_DIRPATH_FORMAT_STR.format(idx, "")
+            )
             padded_idx = shared_utils.zfill_custom(idx + 1, len(str(len(participants))))
-        keystore_start_index = running_total_validator_count
-        running_total_validator_count += participant.validator_count
-        keystore_stop_index = (keystore_start_index + participant.validator_count) - 1
-        artifact_name = "{0}-{1}-{2}-{3}-{4}".format(
+
+            keystore_start_index = running_total_validator_count + i * (
+                participant.validator_count // participant.vc_count
+            )
+            keystore_stop_index = keystore_start_index + (
+                participant.validator_count // participant.vc_count
+            )
+
+            if i == participant.vc_count - 1:
+                keystore_stop_index = (
+                    running_total_validator_count + participant.validator_count
+                )
+
+            artifact_name = "{0}-{1}-{2}-{3}-{4}-{5}".format(
                 padded_idx,
                 participant.cl_type,
                 participant.el_type,
                 keystore_start_index,
-            keystore_stop_index,
+                keystore_stop_index - 1,
+                i,
             )
             artifact_name = plan.store_service_files(
                 service_name, output_dirpath, name=artifact_name
             )
 
-        # This is necessary because the way Kurtosis currently implements artifact-storing is
             base_dirname_in_artifact = shared_utils.path_base(output_dirpath)
             to_add = keystore_files_module.new_keystore_files(
                 artifact_name,
@@ -158,6 +193,8 @@ def generate_validator_keystores(plan, mnemonic, participants):
 
             keystore_files.append(to_add)
 
+        running_total_validator_count += participant.validator_count
+
     write_prysm_password_file_cmd = [
         "sh",
         "-c",
@@ -187,8 +224,6 @@ def generate_validator_keystores(plan, mnemonic, participants):
         keystore_files,
     )
 
-    # TODO replace this with a task so that we can get the container removed
-    # we are removing  a call to remove_service for idempotency
     return result
 
 
@@ -204,7 +239,7 @@ def generate_valdiator_keystores_in_parallel(plan, mnemonic, participants):
     finished_files_to_verify = []
     running_total_validator_count = 0
     for idx, participant in enumerate(participants):
-        output_dirpath = NODE_KEYSTORES_OUTPUT_DIRPATH_FORMAT_STR.format(idx)
+        output_dirpath = NODE_KEYSTORES_OUTPUT_DIRPATH_FORMAT_STR.format(idx, "")
         if participant.validator_count == 0:
             all_generation_commands.append(None)
             all_output_dirpaths.append(None)