Merge pull request #5 from kurtosis-tech/gyani/monitoring-stuff

added the post el/cl client monitoring pieces like forkmon, grafana etc

README.md

@@ -7,21 +7,23 @@ This is the Startosis version of the popular [eth2-merge-kurtosis-module](https:
 ### Parity Missing Tasks
 
 - [ ] Module IO (this is blocked on Startosis Args working)
-- [ ] forkmon (this is blocked on CL clients running)
-- [ ] prometheus (this is blocked on CL clients running)
-- [ ] grafana (this is blocked on prometheus running)
-- [ ] testnet_verifier (this is blocked on CL/EL clients running)
-- [ ] transaction_spammer (this is blocked on EL clients running)
+- [x] forkmon (this is blocked on CL clients running)
+- [x] prometheus (this is blocked on CL clients running)
+- [x] grafana (this is blocked on prometheus running)
+- [ ] grafana needs an upload files endpoint in Startosis
+- [x] testnet_verifier (this is blocked on CL/EL clients running)
+- [x] transaction_spammer (this is blocked on EL clients running)
 - [ ] participant_network/participant_network
   - [ ] has most data generation things, needs to start EL/CL clients
+  - [ ] needs upload files to be implemented
 - [x] participant_network/participant
   - [x] pure POJO should be quick to implement NO BLOCKERS
 - [x] mev_boost participant_network/mev_boost NO BLOCKERS - removed some attributes that aren't used
   - [x] mev_boost_context pure POJO NO BLOCKERS
   - [x] mev_boost_launcher NO BLOCKERS
-- [ ] participant_network/pre_launch_data_generator (the only missing piece here is remove_service)
+- [x] participant_network/pre_launch_data_generator (the only missing piece here is remove_service)
   - [x] data generation
-  - [ ] remove services post generation
+  - [x] remove services post generation
 - [ ] participant_network/el (requires facts and waits)
   - [ ] besu - facts and waits
   - [ ] erigon - facts and waits

docs/changelog.md

@@ -5,6 +5,7 @@
 - Added genesis_constants
 - Added a lot of participant_network/pre_launch_data_generator
 - Added a lot of simple objects that just keep data
+- Added monitoring on top of the repo
 
 # 0.0.0
 * Initial commit

src/forkmon/forkmon_launcher.star

+load("github.com/kurtosis-tech/eth2-module/src/shared_utils/shared_utils.star", "new_port_spec", "new_template_and_data", "path_join")
+
+
+SERVICE_ID = "forkmon"
+IMAGE_NAME = "ralexstokes/ethereum_consensus_monitor:latest"
+
+HTTP_PORT_ID     = "http"
+HTTP_PORT_NUMBER = uint16(80)
+HTTP_PROTOCOL = "TCP"
+
+FORKMON_CONFIG_FILENAME = "forkmon-config.toml"
+
+FORKMON_CONFIG_MOUNT_DIRPATH_ON_SERVICE = "/config"
+
+USED_PORTS = {
+	HTTP_PORT_ID: new_port_spec(HTTP_PORT_NUMBER, HTTP_PROTOCOL)
+}
+
+
+def launch_forkmon(
+		config_template,
+		cl_client_contexts,
+		genesis_unix_timestamp,
+		seconds_per_slot,
+		slots_per_epoch
+	):
+
+	all_cl_client_info = []
+	for client in cl_client_contexts:
+		client_info = new_cl_client_info(client.ip_addr, client.http_port_num)
+		all_cl_client_info.append(client_info)
+
+	template_data = new_config_template_data(HTTP_PORT_NUMBER, all_cl_client_info, seconds_per_slot, slots_per_epoch, genesis_unix_timestamp)
+	template_data_json = json.encode(template_data)
+
+	template_and_data = new_template_and_data(config_template, template_data_json)
+	template_and_data_by_rel_dest_filepath = {}
+	template_and_data_by_rel_dest_filepath[FORKMON_CONFIG_FILENAME] = template_and_data
+
+	config_files_artifact_uuid = render_templates(template_and_data_by_rel_dest_filepath)
+
+	service_config = get_service_config(config_files_artifact_uuid)
+
+	add_service(SERVICE_ID, service_config)
+
+
+def get_service_config(config_files_artifact_uuid):
+	config_file_path = path_join(FORKMON_CONFIG_MOUNT_DIRPATH_ON_SERVICE, FORKMON_CONFIG_FILENAME)
+	return struct(
+		container_image_name = IMAGE_NAME,
+		used_ports = USED_PORTS,
+		files_artifact_mount_dirpaths = {
+			config_files_artifact_uuid: FORKMON_CONFIG_MOUNT_DIRPATH_ON_SERVICE,
+		}
+		cmd_args = ["--config-path", config_file_path]
+	)
+
+
+def new_config_template_data():
+	return {
+		"ListenPortNum": listen_port_num,
+		"CLClientInfo": cl_client_info,
+		"SecondsPerSlot": seconds_per_slot,
+		"SlotsPerEpoch": slots_per_epoch,
+		"GenesisUnixTimestamp": genesis_unix_timestamp,
+	}
+
+
+def new_cl_client_info(ip_addr, port_num):
+	return {
+		"IPAddr": ip_addr,
+		"PortNum": port_num
+	}

src/grafana/grafana_launcher.star

+load("github.com/kurtosis-tech/eth2-module/src/shared_utils/shared_utils.star", "new_port_spec", "new_template_and_data", "path_join")
+
+
+SERVICE_ID = "grafana"
+
+IMAGE_NAME = "grafana/grafana-enterprise:9.2.3"
+
+HTTP_PORT_ID = "http"
+HTTP_PORT_NUMBER_UINT16 = 3000
+HTTP_PORT_PROTOCOL= "TCP"
+
+DATASOURCE_CONFIG_REL_FILEPATH = "datasources/datasource.yml"
+
+# this is relative to the files artifact root
+DASHBOARD_PROVIDERS_CONFIG_REL_FILEPATH = "dashboards/dashboard-providers.yml"
+
+CONFIG_DIRPATH_ENV_VAR = "GF_PATHS_PROVISIONING"
+
+GRAFANA_CONFIG_DIRPATH_ON_SERVICE = "/config"
+GRAFANA_DASHBOARDS_DIRPATH_ON_SERVICE = "/dashboards"
+GRAFANA_DASHBOARDS_FILEPATH_ON_SERVICE = GRAFANA_DASHBOARDS_DIRPATH_ON_SERVICE + "/dashboard.json"
+
+USED_PORTS = {
+	HTTP_PORT_ID: new_port_spec(HTTP_PORT_NUMBER_UINT16, HTTP_PORT_PROTOCOL)
+}
+
+
+def launch_grafana(datasource_config_template, dashboard_providers_config_template, prometheus_private_url):	
+	grafana_config_artifacts_uuid, grafana_dashboards_uuid = get_grafana_config_dir_artifact_uuid(datasource_config_template, dashboard_providers_config_template, prometheus_private_url)
+
+	service_config = get_service_config(grafana_config_artifacts_uuid, grafana_dashboards_artifacts_uuid)
+
+	add_service(SERVICE_ID, service_config)
+
+
+def get_grafana_config_dir_artifact_uuid(datasource_config_template, dashboard_providers_config_template, prometheus_private_url):
+	datasource_data = new_datasource_config_template_data(prometheus_private_url)
+	datasource_data_as_json = json.encode(datasource_data)
+	datasource_template_and_data = new_template_and_data(datasource_config_template, datasource_data_as_json)
+
+	dashboard_providers_data = new_dashboard_providers_config_template_data(GRAFANA_DASHBOARDS_FILEPATH_ON_SERVICE)
+	dashboard_providers_data_json = json.encode(dashboard_providers_data)
+	dashboard_providers_template_and_data = new_template_and_data(dashboard_providers_config_template, dashboard_providers_data_json)
+
+	template_and_data_by_rel_dest_filepath = {}
+	template_and_data_by_rel_dest_filepath[DATASOURCE_CONFIG_REL_FILEPATH] = datasource_template_and_data
+	template_and_data_by_rel_dest_filepath[DASHBOARD_PROVIDERS_CONFIG_REL_FILEPATH] = dashboard_providers_template_and_data
+
+	grafana_config_artifacts_uuid = render_templates(template_and_data_by_rel_dest_filepath)
+
+	# TODO return actual UUID after upload_files is implemented
+	grafana_dashboards_artifacts_uuid = ""
+
+	return grafana_config_artifacts_uuid, grafana_dashboards_artifacts_uuid
+
+
+def new_datasource_config_template_data(prometheus_url):
+	return {
+		"PromtehusURL": prometheus_url
+	}
+
+
+def new_dashboard_providers_config_template_data(dashboards_dirpath):
+	return {
+		"DashboardsDirpath": dashboards_dirpath
+	}

src/participant_network/cl/cl_client_context.star

-# differs from eth2 in the sense it dosen't have the rest_client
+# differs from kurtosis-tech/eth2-merge-kurtosis-module in the sense it dosen't have the rest_client
 def new_cl_client_context(client_name, enr, ip_addr, http_port_num, cl_nodes_metrics_info):
 	return struct(
 		client_name = client_name,

src/participant_network/prelaunch_data_generator/cl_genesis/cl_genesis_data_generator.star

@@ -52,7 +52,7 @@ def generate_cl_genesis_data(
 
 	genesis_generation_config_artifact_uuid = render_templates(template_and_data_by_rel_dest_filepath)
 
-	# TODO Make this the actual data generator
+	# TODO Make this the actual data generator - comment copied from the original module
 	launcher_service_id = launch_prelaunch_data_generator(
 		{
 			genesis_generation_config_artifact_uuid:  CONFIG_DIRPATH_ON_GENERATOR,
@@ -60,9 +60,6 @@ def generate_cl_genesis_data(
 		},
 	)
 
-	# TODO add a remove_service call that removes the launcher_service_id, before the function returns
-	# do that when https://github.com/kurtosis-tech/kurtosis/issues/244 is closed
-
 	all_dirpaths_to_create_on_generator = [
 		CONFIG_DIRPATH_ON_GENERATOR,
 		OUTPUT_DIRPATH_ON_GENERATOR,
@@ -148,6 +145,8 @@ def generate_cl_genesis_data(
 		genesis_ssz_rel_filepath,
 	)
 
+	# we cleanup as the data generation is done
+	remove_service(launcher_service_id)
 	return result
 
 

src/participant_network/prelaunch_data_generator/cl_validator_keystores/cl_validator_keystore_generator.star

@@ -32,14 +32,11 @@ def generate_cl_validator_keystores(
 	mnemonic,
 	num_nodes,
 	num_validators_per_node):
-
+	
 	service_id = launch_prelaunch_data_generator(
 		{},
 	)
 
-	# TODO remove the service added above
-
-
 	all_output_dirpaths = []
 	all_sub_command_strs = []
 
@@ -107,4 +104,6 @@ def generate_cl_validator_keystores(
 		keystore_files,
 	)
 
+	# we cleanup as the data generation is done
+	remove_service(service_id)
 	return result

src/participant_network/prelaunch_data_generator/el_genesis/el_genesis_data_generator.star

@@ -45,15 +45,13 @@ def generate_el_genesis_data(
 	genesis_generation_config_artifact_uuid = render_templates(template_and_data_by_rel_dest_filepath)
 
 
-	# TODO Make this the actual data generator
+	# TODO Make this the actual data generator - comment copied from the original module
 	launcher_service_id = launch_prelaunch_data_generator(
 		{
 			genesis_generation_config_artifact_uuid: CONFIG_DIRPATH_ON_GENERATOR,
 		},
 	)
 
-	# TODO defer remove the above generated service
-
 
 	all_dirpaths_to_create_on_generator = [
 		CONFIG_DIRPATH_ON_GENERATOR,
@@ -119,7 +117,9 @@ def generate_el_genesis_data(
 		genesis_filename_to_relative_filepath_in_artifact[NETHERMIND_GENESIS_FILENAME],
 		genesis_filename_to_relative_filepath_in_artifact[BESU_GENESIS_FILENAME],
 	)
-	
+
+	# we cleanup as the data generation is done
+	remove_service(launcher_service_id)
 	return result
 
 

src/prometheus/prometheus_launcher.star

+load("github.com/kurtosis-tech/eth2-module/src/shared_utils/shared_utils.star", "new_port_spec", "new_template_and_data", "path_join", "path_base")
+
+SERVICE_ID = "prometheus"
+
+# TODO I'm not sure if we should use latest version or ping an specific version instead
+IMAGE_NAME = "prom/prometheus:latest"
+
+HTTP_PORT_ID = "http"
+HTTP_PORT_NUMBER = 9090
+HTTP_PORT_PROTOCOL = "TCP"
+
+CONFIG_FILENAME = "prometheus-config.yml"
+
+CONFIG_DIR_MOUNTPOINT_ON_PROMETHEUS = "/config"
+
+USED_PORTS = {
+	HTTP_PORT_ID: new_port_spec(HTTP_PORT_NUMBER, HTTP_PORT_PROTOCOL)
+}
+
+
+def launch_prometheus(config_template, cl_client_contexts):
+	all_cl_nodes_metrics_info = []
+	for client in cl_client_contexts:
+		all_cl_nodes_metrics_info.append(client.cl_nodes_metrics_info)
+
+	template_data = new_config_template_data(all_cl_nodes_metrics_info)
+	template_data_json = json.encode(template_data)
+
+	template_and_data = new_template_and_data(config_template, template_data_json)
+	template_and_data_by_rel_dest_filepath = {}
+	template_and_data_by_rel_dest_filepath[CONFIG_FILENAME] = template_and_data
+
+	config_files_artifact_uuid = render_templates(template_and_data_by_rel_dest_filepath)
+
+	service_config = get_service_config(config_files_artifact_uuid)
+	prometheus_service = add_service(SERVICE_ID, service_config)
+
+	private_ip_address = prometheus_service.ip_address
+	prometheus_service_http_port = prometheus_service.ports[HTTP_PORT_ID].number
+
+	return "http://{0}:{1}".format(private_ip_address, prometheus_service_http_port)
+
+
+def get_service_config(config_files_artifact_uuid):
+	config_file_path = path_join(CONFIG_DIR_MOUNTPOINT_ON_PROMETHEUS, path_base(CONFIG_FILENAME))
+	return struct(
+		container_image_name = IMAGE_NAME,
+		used_ports = USED_PORTS,
+		files_artifact_mount_dirpaths = {
+			config_files_artifact_uuid : CONFIG_DIR_MOUNTPOINT_ON_PROMETHEUS
+		},
+		cmd_args = [
+			# You can check all the cli flags starting the container and going to the flags section
+			# in Prometheus admin page "{{prometheusPublicURL}}/flags" section
+			"--config.file=" + config_file_path,
+			"--storage.tsdb.path=/prometheus",
+			"--storage.tsdb.retention.time=1d",
+			"--storage.tsdb.retention.size=512MB",
+			"--storage.tsdb.wal-compression",
+			"--web.console.libraries=/etc/prometheus/console_libraries",
+			"--web.console.templates=/etc/prometheus/consoles",
+			"--web.enable-lifecycle",
+		]
+	)
+
+
+def new_config_template_data(cl_nodes_metrics_info):
+	return {
+		"CLNodesMetricsInfo": cl_nodes_metrics_info
+	}

src/testnet_verifier/testnet_verifier.star

+IMAGE_NAME = "marioevz/merge-testnet-verifier:latest"
+SERVICE_ID = "testnet-verifier"
+
+# We use Docker exec commands to run the commands we need, so we override the default
+SYNCHRONOUS_ENTRYPOINT_ARGS = {
+	"sleep",
+	"999999",
+}
+
+
+# this is broken check - https://github.com/ethereum/merge-testnet-verifier/issues/4
+def launch_testnet_verifier(params, el_client_contexts, cl_client_contexts):
+	service_config = get_asynchronous_verification_service_config(params, el_client_contexts, cl_client_contexts)
+	add_service(SERVICE_ID, service_config)
+
+
+def run_synchronous_testnet_verification(params, el_client_contexts, cl_client_contexts):
+	service_config = get_synchronous_verification_service_config()
+	add_service(SERVICE_ID, service_config)
+
+	command = get_cmd()
+	exec(SERVICE_ID, command)
+
+
+def get_cmd(params, el_client_contexts, cl_client_contexts, add_binary_name):
+	command = []
+
+	if add_binary_name:
+		command.append("./merge_testnet_verifier")
+
+	command.append("--ttd 0")
+
+	for el_client_context in el_client_contexts:
+		command.append("--client")
+		command.append("{0},http://{1}:{2}".format(el_client_context.client_name, el_client_context.ip_addr, el_client_context.rpc_port_num))
+
+	for cl_client_context in cl_client_contexts:
+		command.append("--client")
+		command.append("{0},http://{1}:{2}".format(cl_client_context.client_name, cl_client_context.ip_addr, cl_client_context.http_port_num))
+
+	command.append("--ttd-epoch-limit 0")
+	command.append("--verif-epoch-limit")
+	# TODO make this an actual param
+	command.append("{0}".fomrat(param.verifications_epoch_limit))
+
+
+
+
+def get_asynchronous_verification_service_config(params, el_client_contexts, cl_client_contexts):
+	commands = get_cmd(params, el_client_contexts, cl_client_contexts)
+	return struct(
+		container_image_name = IMAGE_NAME,
+		cmd_args = commands,
+		# TODO remove this when used_ports is optional in add_service
+		used_ports = {},
+	)
+
+
+def get_synchronous_verification_service_config():
+	return struct(
+		container_image_name = IMAGE_NAME,
+		entry_point_args = ENTRYPOINT_ARGS,
+		# TODO remove this when used_ports is optional in add_service
+		used_ports = {},
+	)

src/transaction_spammer/transaction_spammer.star

+IMAGE_NAME = "kurtosistech/tx-fuzz:0.2.0"
+SERVICE_ID = "transaction-spammer"
+
+def launch_transaction_spammer(prefunded_addresses, el_client_context):
+	service_config = get_service_config(prefunded_addresses, el_client_context)
+	add_service(SERVICE_ID, service_config)
+
+
+def get_service_config(prefunded_addresses, el_client_context):
+	private_keys_strs = []
+	address_strs = []
+
+	for prefunded_address in prefunded_addresses:
+		private_keys_strs.append(prefunded_address.private_key)
+		address_strs.append(prefunded_address.address)
+
+	comma_separated_private_keys = ",".join(private_keys_strs)
+	comma_separated_addresses = ",".join(address_strs)
+	return struct(
+		container_image_name = IMAGE_NAME,
+		cmd_args = [
+			"http://{0}:{1}".fomrat(el_client_context.ip_addr, el_client_context.rpc_port_num),
+			"spam",
+			comma_separated_private_keys,
+			comma_separated_addresses
+		]
+	)
+