Merge pull request #8 from kurtosis-tech/gyani/module-io

do module io to set defaults

README.md

@@ -6,7 +6,7 @@ This is the Startosis version of the popular [eth2-merge-kurtosis-module](https:
 
 ### Parity Missing Tasks
 
-- [ ] Module IO (this is blocked on Startosis Args working)
+- [x] Module IO (this is blocked on Startosis Args working)
 - [x] forkmon (this is blocked on CL clients running)
 - [x] prometheus (this is blocked on CL clients running)
 - [x] grafana (this is blocked on prometheus running)

main.star

 load("github.com/kurtosis-tech/eth2-module/src/participant_network/participant_network.star", "launch_participant_network")
+load("github.com/kurtosis-tech/eth2-module/src/module_io/parse_input.star", "parse_input")
 
+module_io = import_types("github.com/kurtosis-tech/eth2-module/types.proto")
 
-def main():
-	network_params = new_network_params()
-	num_participants = 2
-	print("Launching participant network with {0} participants and the following network params {1}".format(num_participants, json.indent(json.encode(network_params))))
-	launch_participant_network(num_participants, network_params)
-
-def new_network_params():
-	# this is temporary till we get params working
-	return struct(
-		preregistered_validator_keys_mnemonic =  "giant issue aisle success illegal bike spike question tent bar rely arctic volcano long crawl hungry vocal artwork sniff fantasy very lucky have athlete",
-		num_validator_keys_per_node = 64,
-		network_id = "3151908",
-		deposit_contract_address = "0x4242424242424242424242424242424242424242",
-		seconds_per_slot = 12,
-		mev_boost_relay_endpoints = []
+def main(input_args):
+	input_args_with_right_defaults = module_io.ModuleInput(parse_input(input_args))
+	num_participants = len(input_args_with_right_defaults.participants)
+	print("Launching participant network with {0} participants and the following network params {1}".format(num_participants, input_args_with_right_defaults.network_params))
+	launch_participant_network(num_participants, input_args_with_right_defaults.network_params)
+	#TODO replace with actual values
+	grafana_info = module_io.GrafanaInfo(
+		dashboard_path = "dummy_path",
+		user = "user",
+		password = "password"
 	)
+	output = module_io.ModuleOutput({"grafana_info": grafana_info})
+	return output
+
+

src/module_io/parse_input.star

+DEFAULT_EL_IMAGES = {
+	"geth": "ethereum/client-go:latest",
+	"erigon": "thorax/erigon:devel",
+	"nethermind": "nethermind/nethermind:latest",
+	"besu": "hyperledger/besu:develop"
+}
+
+DEFAULT_CL_IMAGES = {
+	"lighthouse": "sigp/lighthouse:latest",
+	"teku":       "consensys/teku:latest",
+	"nimbus":     "parithoshj/nimbus:merge-a845450",
+	"prysm":    "gcr.io/prysmaticlabs/prysm/beacon-chain:latest,gcr.io/prysmaticlabs/prysm/validator:latest",
+	"lodestar": "chainsafe/lodestar:next",	
+}
+
+BESU_NODE_NAME = "besu"
+NETHERMIND_NODE_NAME = "nethermind"
+
+DESCRIPTOR_ATTR_NAME = "descriptor"
+ATTR_TO_BE_SKIPPED_AT_ROOT = ("network_params", "participants", DESCRIPTOR_ATTR_NAME)
+ENUM_TYPE = "proto.EnumValueDescriptor"
+
+def parse_input(input_args):
+	default_input = default_module_input()
+	result = {}
+	for attr in dir(input_args):
+		value = getattr(input_args, attr)
+		# if its insterted we use the value inserted
+		if attr not in ATTR_TO_BE_SKIPPED_AT_ROOT and proto.has(input_args, attr):
+			result[attr] = get_value_or_name(value)
+		# if not we grab the value from the default value dictionary
+		elif attr not in ATTR_TO_BE_SKIPPED_AT_ROOT:
+			result[attr] = default_input[attr]
+		elif attr == "network_params":
+			result["network_params"] = {}
+			for sub_attr in dir(input_args.network_params):
+				sub_value = getattr(input_args.network_params, sub_attr)
+				# if its insterted we use the value inserted				
+				if sub_attr not in (DESCRIPTOR_ATTR_NAME) and proto.has(input_args.network_params, sub_attr):
+					result["network_params"][sub_attr] = get_value_or_name(sub_value)
+				# if not we grab the value from the default value dictionary					
+				elif sub_attr not in (DESCRIPTOR_ATTR_NAME):
+					result["network_params"][sub_attr] = default_input["network_params"][sub_attr]	
+		# no participants are assigned at all
+		elif attr == "participants" and len(value) == 0:
+			result["participants"] = default_input["participants"]
+		elif attr == "participants":
+			participants = []
+			for participant in input_args.participants:
+				participant_value = {}
+				for sub_attr in dir(participant):
+					sub_value = getattr(participant, sub_attr)
+					# if its insterted we use the value inserted
+					if sub_attr not in (DESCRIPTOR_ATTR_NAME) and proto.has(participant, sub_attr):
+						participant_value[sub_attr] = get_value_or_name(sub_value)
+					# if not we grab the value from the default value dictionary
+					elif sub_attr not in (DESCRIPTOR_ATTR_NAME):
+						participant_value[attr] = default_input["participants"][0].get(sub_attr, "")
+				participants.append(participant_value)
+			result["participants"] = participants
+
+
+	# validation of the above defaults
+	for index, participant in enumerate(result["participants"]):
+		el_client_type = participant["el_client_type"]
+		cl_client_type = participant["cl_client_type"]
+
+		if index == 0 and el_client_type in (BESU_NODE_NAME, NETHERMIND_NODE_NAME):
+			fail("besu/nethermind cant be the first participant")
+		
+		el_image = participant["el_client_image"]
+		if el_image == "":
+			default_image = DEFAULT_EL_IMAGES.get(el_client_type, "")
+			if default_image == "":
+				fail("{0} received an empty image name and we don't have a default for it".format(el_client_type))
+			participant["el_client_image"] = default_image
+
+		cl_image = participant["cl_client_image"]
+		if cl_image == "":
+			default_image = DEFAULT_CL_IMAGES.get(cl_client_type, "")
+			if default_image == "":
+				fail("{0} received an empty image name and we don't have a default for it".format(cl_client_type))
+			participant["cl_client_image"] = default_image
+
+		beacon_extra_params = participant.get("beacon_extra_params", [])
+		participant["beacon_extra_params"] = beacon_extra_params
+
+		validator_extra_params = participant.get("validator_extra_params", [])
+		participant["validator_extra_params"] = validator_extra_params
+
+	if result["network_params"]["network_id"].strip() == "":
+		fail("network_id is empty or spaces it needs to be of non zero length")
+
+	if result["network_params"]["deposit_contract_address"].strip() == "":
+		fail("deposit_contract_address is empty or spaces it needs to be of non zero length")
+
+	if result["network_params"]["preregistered_validator_keys_mnemonic"].strip() == "":
+		fail("preregistered_validator_keys_mnemonic is empty or spaces it needs to be of non zero length")
+
+	if result["network_params"]["slots_per_epoch"] == 0:
+		fail("slots_per_epoch is 0 needs to be > 0 ")
+
+	if result["network_params"]["seconds_per_slot"] == 0:
+		fail("seconds_per_slot is 0 needs to be > 0 ")
+
+	required_num_validtors = 2 * result["network_params"]["slots_per_epoch"]
+	actual_num_validators = len(result["participants"]) * result["network_params"]["num_validators_per_keynode"]
+	if required_num_validtors < actual_num_validators:
+		fail("required_num_validtors - {0} is greater than actual_num_validators - {1}".format(required_num_validtors, actual_num_validators))
+
+	# Remove if nethermind doesn't break as second node we already test above if its the first node
+	if len(result["participants"]) >= 2 and result["participants"][1]["el_client_type"] == NETHERMIND_NODE_NAME:
+		fail("nethermind can't be the first or second node")
+
+	return result
+
+
+
+def get_value_or_name(value):
+	if type(value) == ENUM_TYPE:
+		return value.name
+	return value
+
+
+def default_module_input():
+	network_params = default_network_params()
+	participants = default_partitcipants()
+	return {
+		"participants": participants,
+		"network_params": network_params,
+		"launch_additional_services" : True,
+		"wait_for_finalization":      False,
+		"wait_for_verifications":     False,
+		"verifications_epoch_limit":  5,
+		"global_log_level": "info"
+	}
+
+
+
+def default_network_params():
+	# this is temporary till we get params working
+	return {
+		"preregistered_validator_keys_mnemonic" :  "giant issue aisle success illegal bike spike question tent bar rely arctic volcano long crawl hungry vocal artwork sniff fantasy very lucky have athlete",
+		"num_validators_per_keynode" : 64,
+		"network_id" : "3151908",
+		"deposit_contract_address" : "0x4242424242424242424242424242424242424242",
+		"seconds_per_slot" : 12,
+		"slots_per_epoch" : 32,
+	}
+
+def default_partitcipants():
+	participant = {
+			"el_client_type": "geth",
+			"el_client_image": "",
+			"el_client_log_level": "",
+			"cl_client_type": "lighthouse",
+			"cl_client_image": "",
+			"cl_client_log_level": ""
+	}
+	return [participant]

src/participant_network/participant_network.star

@@ -15,7 +15,7 @@ def launch_participant_network(num_participants, network_params):
 	keystore_result = generate_cl_validator_keystores(
 		network_params.preregistered_validator_keys_mnemonic,
 		num_participants,
-		network_params.num_validator_keys_per_node
+		network_params.num_validators_per_keynode
 	)
 
 	
@@ -38,7 +38,7 @@ def launch_participant_network(num_participants, network_params):
 	print("Generating CL data")
 	genesis_generation_config_yml_template = read_file("github.com/kurtosis-tech/eth2-module/static_files/genesis-generation-config/cl/config.yaml.tmpl")
 	genesis_generation_mnemonics_yml_template = read_file("github.com/kurtosis-tech/eth2-module/static_files/genesis-generation-config/cl/mnemonics.yaml.tmpl")
-	total_number_of_validator_keys = network_params.num_validator_keys_per_node * num_participants
+	total_number_of_validator_keys = network_params.num_validators_per_keynode * num_participants
 	cl_data = generate_cl_genesis_data(
 		genesis_generation_config_yml_template,
 		genesis_generation_mnemonics_yml_template,
@@ -57,7 +57,7 @@ def launch_participant_network(num_participants, network_params):
 	print("launching mev boost")
 	# TODO make this launch only for participants that have the participants[i].builderNetworkParams.relayEndpoints defined
 	# At the moment this lies here just to test, and the relay end points is an empty list
-	mev_boost_launcher = new_mev_boost_launcher(MEV_BOOST_SHOULD_RELAY, network_params.mev_boost_relay_endpoints)
+	mev_boost_launcher = new_mev_boost_launcher(MEV_BOOST_SHOULD_RELAY, [])
 	mev_boost_service_id = MEV_BOOST_SERVICE_ID_PREFIX.format(1)
 	mev_boost_context = launch_mevboost(mev_boost_launcher, mev_boost_service_id, network_params.network_id)
 	print(mev_boost_endpoint(mev_boost_context))

types.proto

+syntax = "proto3";
+
+// Module Input
+message ModuleInput {
+	// Parameters controlling the types of clients that compose the network
+	repeated Participant participants = 1;
+	
+	// Parameters controlling the settings of the network itself
+	optional NetworkParams network_params = 2;
+
+	// True by defaults such that in addition to the Ethereum network:
+	//  - A transaction spammer is launched to fake transactions sent to the network
+	//  - Forkmon will be launched after CL genesis has happened
+	//  - a prometheus will be started, coupled with grafana
+	// If set to false:
+	//  - only Ethereum network (EL and CL nodes) will be launched. Nothing else (no transaction spammer)
+	//  - params for the CL nodes will be ignored (e.g. CL node image, CL node extra params)
+	// This is a hack - it's not very elegant - but this is a commonly-requested feature
+	// The longterm solution is making the module trivial to decompose so we don't need flags like this; we're working
+	//  on this at the Kurtosis product level
+	optional bool launch_additional_services = 3;
+
+	// If set, the module will block until a finalized epoch has occurred.
+	// If `waitForVerifications` is set to true, this extra wait will be skipped.
+	optional bool wait_for_finalization = 4;
+
+	// If set to true, the module will block until all verifications have passed
+	optional bool wait_for_verifications = 5;
+
+	// If set, after the merge, this will be the maximum number of epochs wait for the verifications to succeed.
+	optional uint64 verifications_epoch_limit = 6;
+
+	// The log level that the started clients should log at
+	optional GlobalLogLevel global_log_level = 7;
+}
+
+enum GlobalLogLevel {
+	info = 0;
+	error = 1;
+	warn = 2;
+	debug = 3;
+	trace = 4;
+}
+
+enum ELClientType {
+	geth = 0;
+	erigon = 1;
+	nethermind = 2;
+	besu = 3;
+}
+
+enum CLClientType {
+	lighthouse = 0;
+	teku = 1;
+	nimbus = 2;
+	prysm = 3;
+	loadstar = 4;
+}
+
+message BuilderNetworkParams {
+	// A list of endpoints to reach block builder relays
+	repeated string relay_endpoints = 1;
+}
+
+message Participant {
+	// The type of EL client that should be started
+	optional ELClientType el_client_type = 1;
+	// The Docker image that should be used for the EL client; leave blank to use the default
+	optional string el_client_image  = 2;
+	// The log level string that this participant's EL client should log at
+	// If this is emptystring then the global `logLevel` parameter's value will be translated into a string appropriate for the client (e.g. if
+	//  global `logLevel` = `info` then Geth would receive `3`, Besu would receive `INFO`, etc.)
+	// If this is not emptystring, then this value will override the global `logLevel` setting to allow for fine-grained control
+	//  over a specific participant's logging	
+	optional string el_client_log_level = 3;
+	// Optional extra parameters that will be passed to the EL client
+	repeated string el_extra_params = 4;
+
+	// The type of CL client that should be started
+	optional CLClientType cl_client_type = 5;
+	// The Docker image that should be used for the EL client; leave blank to use the default
+	// NOTE: Prysm is different in that it requires two images - a Beacon and a validator
+	//  For Prysm and Prysm only, this field should contain a comma-separated string of "beacon_image,validator_image"	
+	optional string cl_client_image  = 6;
+	// The log level string that this participant's CL client should log at
+	// If this is emptystring then the global `logLevel` parameter's value will be translated into a string appropriate for the client (e.g. if
+	//  global `logLevel` = `info` then Nimbus would receive `INFO`, Prysm would receive `info`, etc.)
+	// If this is not emptystring, then this value will override the global `logLevel` setting to allow for fine-grained control
+	//  over a specific participant's logging	
+	optional string cl_client_log_level = 7;
+	
+	// Extra parameters that will be passed to the Beacon container (if a separate one exists), or to the combined node if
+	// the Beacon and validator are combined
+	repeated string beacon_extra_params = 8;
+
+	// Extra parameters that will be passed to the validator container (if a separate one exists), or to the combined node if
+	// the Beacon and validator are combined
+	repeated string validator_extra_params = 9;
+
+	optional BuilderNetworkParams builder_network_params = 10;
+}
+
+message NetworkParams {
+	// The network ID of the Eth1 network
+	optional string network_id = 1;
+	// The address of the staking contract address on the Eth1 chain
+	optional string deposit_contract_address = 2;
+	// Number of seconds per slot on the Beacon chain
+	optional uint32 seconds_per_slot = 3;
+	// Number of slots in an epoch on the Beacon chain
+	optional uint32 slots_per_epoch = 4;
+	// The number of validator keys that each CL validator node should get
+	optional uint32 num_validators_per_keynode = 5;
+	// This menmonic will a) be used to create keystores for all the types of validators that we have and b) be used to generate a CL genesis.ssz that has the children
+	//  validator keys already preregistered as validators	
+	optional string preregistered_validator_keys_mnemonic = 6;
+}
+
+
+// Module Output
+message ModuleOutput {
+	GrafanaInfo grafana_info = 1;
+}
+
+message GrafanaInfo {
+	string dashboard_path = 1;
+	string user = 2;
+	string password = 3;
+}
+