lgiht house seems to be in a good spot

README.md

@@ -46,7 +46,7 @@ This is the Startosis version of the popular [eth2-merge-kurtosis-module](https:
 - [ ] participant_network/cl (requires facts and waits)
   - [ ] lighthouse 
     - [ ] facts and waits
-    - [ ] framework
+    - [x] framework
   - [ ] loadstar
     - [ ] facts and waits
     - [ ] framework

src/participant_network/cl/cl_client_context.star

 # differs from kurtosis-tech/eth2-merge-kurtosis-module in the sense it dosen't have the rest_client
+# broader use of the rest client allows for waiting for the first cl context to be heahty in module.go
+# TODO remove the above comment when things are working
 def new_cl_client_context(client_name, enr, ip_addr, http_port_num, cl_nodes_metrics_info):
 	return struct(
 		client_name = client_name,

src/participant_network/cl/lighthouse/lighthouse_launcher.star

+load("github.com/kurtosis-tech/eth2-module/src/shared_utils/shared_utils.star", "new_port_spec", "path_join", "path_dir")
+load("github.com/kurtosis-tech/eth2-module/src/module_io/parse_input.star", "get_client_log_level_or_default")
+load("github.com/kurtosis-tech/eth2-module/src/cl/cl_client_context.star", "new_cl_client_context")
+load("github.com/kurtosis-tech/eth2-module/src/cl/cl_node_metrics_info.star", "new_cl_node_metrics_info")
+load("github.com/kurtosis-tech/eth2-module/src/mev_boost/mev_boost_context.star", "mev_boost_endpoint")
+
+
+LIGHTHOUSE_BINARY_COMMAND = "lighthouse"
+
+GENESIS_DATA_MOUNTPOINT_ON_CLIENTS = "/genesis"
+
+VALIDATOR_KEYS_MOUNTPOINT_ON_CLIENTS = "/validator-keys"
+
+RUST_BACKTRACE_ENVVAR_NAME  = "RUST_BACKTRACE"
+RUST_FULL_BACKTRACE_KEYWORD = "full"
+
+#  ---------------------------------- Beacon client -------------------------------------
+CONSENSUS_DATA_DIRPATH_ON_BEACON_SERVICE_CONTAINER = "/consensus-data"
+
+# Port IDs
+BEACON_TCP_DISCOVERY_PORT_ID = "tcp-discovery"
+BEACON_UDP_DISCOVERY_PORT_ID = "udp-discovery"
+BEACON_HTTP_PORT_ID         = "http"
+BEACON_METRICS_PORT_ID      = "metrics"
+
+# Port nums
+BEACON_DISCOVERY_PORT_NUM = 9000
+BEACON_HTTP_PORT_NUM      = 4000
+BEACON_METRICS_PORT_NUM   = 5054
+
+# TODO remove if facts & waits doesn't need this
+MAX_NUM_HEALTHCHECK_RETRIES      = 10
+TIME_BETWEEN_HEALTHCHECK_RETRIES = 1 * time.second
+
+#  ---------------------------------- Validator client -------------------------------------
+VALIDATING_REWARDS_ACCOUNT = "0x0000000000000000000000000000000000000000"
+
+VALIDATOR_HTTP_PORT_ID     = "http"
+VALIDATOR_METRICS_PORT_ID  = "metrics"
+VALIDATOR_HTTP_PORT_NUM    = 5042
+VALIDATOR_METRICS_PORT_NUM = 5064
+
+METRICS_PATH = "/metrics"
+
+BEACON_SUFFIX_SERVICE_ID    = "beacon"
+VALIDATOR_SUFFIX_SERVICE_ID = "validator"
+
+PRIVATE_IP_ADDRESS_PLACEHOLDER = "KURTOSIS_IP_ADDR_PLACEHOLDER"
+
+# TODO push this into shared_utils
+TCP_PROTOCOL = "TCP"
+UDP_PROTOCOL = "UDP"
+
+BEACON_USED_PORTS = {
+	BEACON_TCP_DISCOVERY_PORT_ID: new_port_spec(BEACON_DISCOVERY_PORT_NUM, TCP_PROTOCOL),
+	BEACON_UDP_DISCOVERY_PORT_ID: new_port_spec(BEACON_DISCOVERY_PORT_NUM, UDP_PROTOCOL),
+	BEACON_HTTP_PORT_ID:         new_port_spec(BEACON_HTTP_PORT_NUM, TCP_PROTOCOL),
+	BEACON_METRICS_PORT_ID:      new_port_spec(BEACON_METRICS_PORT_NUM, TCP_PROTOCOL),
+}
+
+VALIDATOR_USED_PORTS = {
+	VALIDATOR_HTTP_PORT_ID:    new_port_spec(VALIDATOR_HTTP_PORT_NUM, TCP_PROTOCOL),
+	VALIDATOR_METRICS_PORT_ID: new_port_spec(VALIDATOR_METRICS_PORT_NUM, TCP_PROTOCOL),
+}
+
+LIGHTHOUSE_LOG_LEVELS = {
+	module_io.GlobalClientLogLevel.error: "error",
+	module_io.GlobalClientLogLeve.warn:  "warn",
+	module_io.GlobalClientLogLeve.info:  "info",
+	module_io.GlobalClientLogLevel.debug: "debug",
+	module_io.GlobalClientLogLevel.trace: "trace",
+}
+
+def launch(
+	launcher,
+	service_id,
+	image,
+	participant_log_level,
+	global_log_level,
+	bootnode_context,
+	el_client_context,
+	mev_boost_context,
+	node_keystore_files,
+	extra_beacon_params,
+	extra_validator_params):
+
+	beacon_node_service_id = "{0}-{1}".format(service_id, BEACON_SUFFIX_SERVICE_ID)
+	validator_node_service_id = "{0}-{1}".format(service_id, VALIDATOR_SUFFIX_SERVICE_ID)
+
+	log_level = get_client_log_level_or_default(participant_log_level, global_log_level, LIGHTHOUSE_LOG_LEVELS)
+
+	# Launch Beacon node
+	beacon_service_config = get_beacon_service_config(
+		launcher.genesis_data,
+		image,
+		bootnode_context,
+		el_client_context,
+		mev_boost_context,
+		log_level,
+		extra_beacon_params,
+	)
+
+	beacon_service = add_service(beacon_node_service_id, beacon_service_config)
+
+	beacon_http_port = beacon_service.ports[BEACON_HTTP_PORT_ID]
+
+	# TODO add facts & waits
+
+	# Launch validator node
+	beacon_http_url = "http://{0}:{1}".format(beacon_service.ip_address, beacon_http_port.number)
+
+	validator_service_config = get_validator_service_config(
+		launcher.genesis_data,
+		image,
+		log_level,
+		beacon_http_url,
+		node_keystore_files,
+		mev_boost_context,
+		extra_validator_params,
+	)
+
+	validtor_service = add_service(validator_node_service_id, validator_service_config)
+
+	# TODO add validator availability using the validator API: https://ethereum.github.io/beacon-APIs/?urls.primaryName=v1#/ValidatorRequiredApi | from eth2-merge-kurtosis-module
+
+	# TODO get node identity using facts and waits
+	beacon_node_enr = ""
+
+	beacon_metrics_port = beacon_service.ports[BEACON_METRICS_PORT_ID]
+	beacon_metrics_url = "{0}:{1}".format(beacon_service.ip_address, beacon_metrics_port.number)
+
+	validator_metrics_port = validtor_service.ports[VALIDATOR_METRICS_PORT_ID]
+	validator_metrics_url = "{00}:{1}".format(validtor_service.ip_address, validator_metrics_port.number)
+
+	beacon_node_metrics_info = new_cl_node_metrics_info(beacon_node_service_id, METRICS_PATH, beacon_metrics_url)
+	validator_node_metrics_info = new_cl_node_metrics_info(validator_node_service_id, METRICS_PATH, validator_metrics_url)
+	nodes_metrics_info = [beacon_node_metrics_info, validatorNodeMetricsInfo]
+
+	result = new_cl_client_context(
+		"lighthouse",
+		beacon_node_enr,
+		beacon_service.ip_address,
+		BEACON_HTTP_PORT_NUM,
+		nodes_metrics_info,
+	)
+
+	return result
+
+def get_beacon_service_config(
+	genesis_data,
+	image
+	boot_cl_client_ctx,
+	el_client_ctx,
+	mev_boost_context,
+	log_level
+	extra_params):
+
+	el_client_engine_rpc_url_str = "http://%v:%v".format(
+		el_client_ctx.ip_address,
+		el_client_ctx.engine_rpc_port_num,
+	)
+
+	# For some reason, Lighthouse takes in the parent directory of the config file (rather than the path to the config file itself)
+	genesis_config_parent_dirpath_on_client = path_join(GENESIS_DATA_MOUNTPOINT_ON_CLIENTS, path_dir(genesis_data.config_yml_rel_filepath))
+	jwt_secret_filepath = path_join(GENESIS_DATA_MOUNTPOINT_ON_CLIENTS, genesis_data.jwt_secret_relative_filepath)
+
+	# NOTE: If connecting to the merge devnet remotely we DON'T want the following flags; when they're not set, the node's external IP address is auto-detected
+	#  from the peers it communicates with but when they're set they basically say "override the autodetection and
+	#  use what I specify instead." This requires having a know external IP address and port, which we definitely won't
+	#  have with a network running in Kurtosis.
+	#    "--disable-enr-auto-update",
+	#    "--enr-address=" + externalIpAddress,
+	#    fmt.Sprintf("--enr-udp-port=%v", BEACON_DISCOVERY_PORT_NUM),
+	#    fmt.Sprintf("--enr-tcp-port=%v", beaconDiscoveryPortNum),
+	cmd_args = [
+		LIGHTHOUSE_BINARY_COMMAND,
+		"beacon_node",
+		"--debug-level=" + log_level,
+		"--datadir=" + CONSENSUS_DATA_DIRPATH_ON_BEACON_SERVICE_CONTAINER,
+		"--testnet-dir=" + genesis_config_parent_dirpath_on_client,
+		# vvvvvvvvvvvvvvvvvvv REMOVE THESE WHEN CONNECTING TO EXTERNAL NET vvvvvvvvvvvvvvvvvvvvv
+		"--disable-enr-auto-update",
+		"--enr-address=" + PRIVATE_IP_ADDRESS_PLACEHOLDER,
+		"--enr-udp-port={0}".format(BEACON_DISCOVERY_PORT_NUM),
+		"--enr-tcp-port={0}".format(BEACON_DISCOVERY_PORT_NUM),
+		# ^^^^^^^^^^^^^^^^^^^ REMOVE THESE WHEN CONNECTING TO EXTERNAL NET ^^^^^^^^^^^^^^^^^^^^^
+		"--listen-address=0.0.0.0",
+		"--port={0}".format(BEACON_DISCOVERY_PORT_NUM), # NOTE: Remove for connecting to external net!
+		"--http",
+		"--http-address=0.0.0.0",
+		"--http-port={0}".format(BEACON_HTTP_PORT_NUM),
+		"--http-allow-sync-stalled",
+		# NOTE: This comes from:
+		#   https://github.com/sigp/lighthouse/blob/7c88f582d955537f7ffff9b2c879dcf5bf80ce13/scripts/local_testnet/beacon_node.sh
+		# and the option says it's "useful for testing in smaller networks" (unclear what happens in larger networks)
+		"--disable-packet-filter",
+		"--execution-endpoints=" + el_client_engine_rpc_url_str,
+		"--jwt-secrets=" + jwt_secret_filepath,
+		"--suggested-fee-recipient=" + VALIDATING_REWARDS_ACCOUNT,
+		# Set per Paris' recommendation to reduce noise in the logs
+		"--subscribe-all-subnets",
+		# vvvvvvvvvvvvvvvvvvv METRICS CONFIG vvvvvvvvvvvvvvvvvvvvv
+		"--metrics",
+		"--metrics-address=0.0.0.0",
+		"--metrics-allow-origin=*",
+		"--metrics-port=%v".format(BEACON_METRICS_PORT_NUM),
+		# ^^^^^^^^^^^^^^^^^^^ METRICS CONFIG ^^^^^^^^^^^^^^^^^^^^^
+	]
+
+	if boot_cl_client_ctx != None:
+		cmd_args.append("--boot-nodes="+boot_cl_client_ctx.enode)
+
+	if mev_boost_context != nil {
+		cmd_args.append("--builder")
+		cmd_args.append(mev_boost_endpoint(mev_boost_context))
+	}
+
+	if len(extra_params) > 0 {
+		cmd_args.extend(extra_params)
+	}
+
+	return struct(
+		container_image_name = image,
+		used_ports = BEACON_USED_PORTS,
+		cmd_args = cmd_args,
+		files_artifact_mount_dirpaths = {
+			genesis_data.files_artifact_uuid: GENESIS_DATA_MOUNTPOINT_ON_CLIENTS
+		},
+		env_vars = {
+			RUST_BACKTRACE_ENVVAR_NAME: RUST_FULL_BACKTRACE_KEYWORD
+		},
+		privaite_ip_address_placeholder = PRIVATE_IP_ADDRESS_PLACEHOLDER
+	)
+
+
+def get_validator_service_config(
+	genesis_data,
+	image,
+	log_level,
+	beacon_client_http_url,
+	node_keystore_files,
+	mev_boost_context,
+	extra_params):
+
+	# For some reason, Lighthouse takes in the parent directory of the config file (rather than the path to the config file itself)
+	genesis_config_parent_dirpath_on_client = path_join(GENESIS_DATA_MOUNTPOINT_ON_CLIENTS, path_dir(launcher.genesisData.GetConfigYMLRelativeFilepath()))
+	validator_keys_dirpath = path_join(VALIDATOR_KEYS_MOUNTPOINT_ON_CLIENTS, node_keystore_files.RawKeysRelativeDirpath)
+	validator_secrets_dirpath = path_join(VALIDATOR_KEYS_MOUNTPOINT_ON_CLIENTS, node_keystore_files.RawSecretsRelativeDirpath)
+	
+	cmd_args = [
+		"lighthouse",
+		"validator_client",
+		"--debug-level=" + log_level,
+		"--testnet-dir=" + genesis_config_parent_dirpath_on_client,
+		"--validators-dir=" + validator_keys_dirpath,
+		# NOTE: When secrets-dir is specified, we can't add the --data-dir flag
+		"--secrets-dir=" + validator_secrets_dirpath,
+		# The node won't have a slashing protection database and will fail to start otherwise
+		"--init-slashing-protection",
+		"--http",
+		"--unencrypted-http-transport",
+		"--http-address=0.0.0.0",
+		"--http-port={0}".format(VALIDATOR_HTTP_PORT_NUM),
+		"--beacon-nodes=" + beacon_client_http_url,
+		#"--enable-doppelganger-protection", // Disabled to not have to wait 2 epochs before validator can start
+		# burn address - If unset, the validator will scream in its logs
+		"--suggested-fee-recipient=0x0000000000000000000000000000000000000000",
+		# vvvvvvvvvvvvvvvvvvv PROMETHEUS CONFIG vvvvvvvvvvvvvvvvvvvvv
+		"--metrics",
+		"--metrics-address=0.0.0.0",
+		"--metrics-allow-origin=*",
+		"--metrics-port={0}".format(VALIDATOR_METRICS_PORT_NUM),
+		# ^^^^^^^^^^^^^^^^^^^ PROMETHEUS CONFIG ^^^^^^^^^^^^^^^^^^^^^
+	]
+
+	if mev_boost_context != None:
+		cmd_args.append("--builder-proposals")
+
+	if len(extra_params):
+		cmd_args.extend(extra_params)
+
+
+	return struct(
+		container_image_name = image,
+		used_ports = VALIDATOR_USED_PORTS,
+		cmd_args = cmd_args,
+		files_artifact_mount_dirpaths = {
+			genesis_data.files_artifact_uuid: GENESIS_DATA_MOUNTPOINT_ON_CLIENTS,
+			node_keystore_files.files_artifact_uuid: VALIDATOR_KEYS_MOUNTPOINT_ON_CLIENTS,
+		},
+		env_vars = {
+			RUST_BACKTRACE_ENVVAR_NAME: RUST_FULL_BACKTRACE_KEYWORD
+		},
+	)
+
+
+
+def new_lighthouses_launcher(cl_genesi_data):
+	return struct(
+		cl_genesi_data = cl_genesi_data,
+	)
\ No newline at end of file

src/participant_network/prelaunch_data_generator/cl_validator_keystores/generate_keystores_result.star

@@ -3,11 +3,11 @@
 def new_generate_keystores_result(prysm_password_artifact_uuid, prysm_password_relative_filepath, per_node_keystores):
 	return struct(
 		#Files artifact UUID where the Prysm password is stored
-		PrysmPasswordArtifactUUid = prysm_password_artifact_uuid,
+		prysm_password_artifact_uuid = prysm_password_artifact_uuid,
 
 		# Relative to root of files artifact
-		PrysmPasswordRelativeFilepath = prysm_password_relative_filepath,
+		prysm_password_relative_filepath = prysm_password_relative_filepath,
 
 		# Contains keystores-per-client-type for each node in the network
-		PerNodeKeystores = per_node_keystores
+		per_node_keystores = per_node_keystores
 	)

src/participant_network/prelaunch_data_generator/cl_validator_keystores/keystore_files.star

 # One of these will be created per node we're trying to start
 def new_keystore_files(files_artifact_uuid, raw_keys_relative_dirpath, raw_secrets_relative_dirpath, nimbus_keys_relative_dirpath, prysm_relative_dirpath, teku_keys_relative_dirpath, teku_secrets_relative_dirpath):
 	return struct(
-		FilesArtifactUUID =  files_artifact_uuid,
+		files_artifact_uuid =  files_artifact_uuid,
 		# ------------ All directories below are relative to the root of the files artifact ----------------
-		RawKeysRelativeDirpath =  raw_keys_relative_dirpath,
-		RawSecretsRelativeDirpath =  raw_secrets_relative_dirpath,
-		NimbusKeysRelativeDirpath =  nimbus_keys_relative_dirpath,
-		PrysmRelativeDirpath =  prysm_relative_dirpath,
-		TekuKeysRelativeDirpath =  teku_keys_relative_dirpath,
-		TekuSecretsRelativeDirpath =  teku_secrets_relative_dirpath 
+		raw_keys_relative_dirpath =  raw_keys_relative_dirpath,
+		raw_secrets_relative_dirpath =  raw_secrets_relative_dirpath,
+		nimbus_keys_relative_dirpath =  nimbus_keys_relative_dirpath,
+		prysm_relative_dirpath =  prysm_relative_dirpath,
+		teku_keys_relative_dirpath =  teku_keys_relative_dirpath,
+		teku_secrets_relative_dirpath =  teku_secrets_relative_dirpath 
 	)

src/shared_utils/shared_utils.star

@@ -12,5 +12,13 @@ def path_base(path):
 	return split_path[-1]
 
 
+def path_dir(path):
+	split_path = path.split("/")
+	if len(split_path) <= 1:
+		return "."
+	split_path = split_path[:-1]
+	return "/".join(split_path) or "/"
+
+
 def new_port_spec(number, protocol):
 	return struct(number = number, protocol = protocol)