Skip to content
Projects
Groups
Snippets
Help
Loading...
Help
Support
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
E
ethereum-package
Project
Project
Details
Activity
Releases
Cycle Analytics
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Charts
Issues
0
Issues
0
List
Boards
Labels
Milestones
Merge Requests
0
Merge Requests
0
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Charts
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Charts
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
vicotor
ethereum-package
Commits
7e361913
Unverified
Commit
7e361913
authored
Mar 20, 2024
by
Barnabas Busa
Committed by
GitHub
Mar 20, 2024
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
feat: add beacon snooper (#520)
parent
177beeb9
Changes
27
Hide whitespace changes
Inline
Side-by-side
Showing
27 changed files
with
236 additions
and
56 deletions
+236
-56
mix-persistence.yaml
.github/tests/mix-persistence.yaml
+2
-2
mix-with-tools.yaml
.github/tests/mix-with-tools.yaml
+3
-1
main.star
main.star
+3
-2
assertoor_launcher.star
src/assertoor/assertoor_launcher.star
+11
-7
cl_launcher.star
src/cl/cl_launcher.star
+6
-4
grandine_launcher.star
src/cl/grandine/grandine_launcher.star
+4
-1
lighthouse_launcher.star
src/cl/lighthouse/lighthouse_launcher.star
+1
-0
lodestar_launcher.star
src/cl/lodestar/lodestar_launcher.star
+1
-0
nimbus_launcher.star
src/cl/nimbus/nimbus_launcher.star
+4
-1
prysm_launcher.star
src/cl/prysm/prysm_launcher.star
+1
-0
teku_launcher.star
src/cl/teku/teku_launcher.star
+4
-4
dora_launcher.star
src/dora/dora_launcher.star
+15
-5
constants.star
src/package_io/constants.star
+2
-0
participant_network.star
src/participant_network.star
+51
-3
validator_ranges_generator.star
...rator/validator_keystores/validator_ranges_generator.star
+8
-6
shared_utils.star
src/shared_utils/shared_utils.star
+17
-0
snooper_beacon_context.star
src/snooper/snooper_beacon_context.star
+5
-0
snooper_beacon_launcher.star
src/snooper/snooper_beacon_launcher.star
+61
-0
snooper_engine_launcher.star
src/snooper/snooper_engine_launcher.star
+3
-4
lighthouse.star
src/vc/lighthouse.star
+2
-1
lodestar.star
src/vc/lodestar.star
+2
-1
nimbus.star
src/vc/nimbus.star
+2
-1
prysm.star
src/vc/prysm.star
+2
-1
teku.star
src/vc/teku.star
+2
-4
vc_context.star
src/vc/vc_context.star
+2
-2
vc_launcher.star
src/vc/vc_launcher.star
+21
-5
config.yaml.tmpl
static_files/dora-config/config.yaml.tmpl
+1
-1
No files found.
.github/tests/mix-persistence.yaml
View file @
7e361913
...
@@ -4,11 +4,11 @@ participants:
...
@@ -4,11 +4,11 @@ participants:
use_separate_vc
:
true
use_separate_vc
:
true
-
el_type
:
nethermind
-
el_type
:
nethermind
cl_type
:
prysm
cl_type
:
prysm
-
el_type
:
besu
cl_type
:
lighthouse
-
el_type
:
erigon
-
el_type
:
erigon
cl_type
:
nimbus
cl_type
:
nimbus
use_separate_vc
:
true
use_separate_vc
:
true
-
el_type
:
besu
cl_type
:
lighthouse
-
el_type
:
reth
-
el_type
:
reth
cl_type
:
lodestar
cl_type
:
lodestar
-
el_type
:
ethereumjs
-
el_type
:
ethereumjs
...
...
.github/tests/mix-with-tools.yaml
View file @
7e361913
...
@@ -10,7 +10,9 @@ participants:
...
@@ -10,7 +10,9 @@ participants:
-
el_type
:
reth
-
el_type
:
reth
cl_type
:
lodestar
cl_type
:
lodestar
-
el_type
:
ethereumjs
-
el_type
:
ethereumjs
cl_type
:
teku
cl_type
:
grandine
-
el_type
:
geth
cl_type
:
grandine
additional_services
:
additional_services
:
-
tx_spammer
-
tx_spammer
-
blob_spammer
-
blob_spammer
...
...
main.star
View file @
7e361913
...
@@ -138,7 +138,7 @@ def run(plan, args={}):
...
@@ -138,7 +138,7 @@ def run(plan, args={}):
ranges = validator_ranges.generate_validator_ranges(
ranges = validator_ranges.generate_validator_ranges(
plan,
plan,
validator_ranges_config_template,
validator_ranges_config_template,
all_
cl_contex
ts,
all_
participan
ts,
args_with_right_defaults.participants,
args_with_right_defaults.participants,
)
)
...
@@ -370,7 +370,8 @@ def run(plan, args={}):
...
@@ -370,7 +370,8 @@ def run(plan, args={}):
dora.launch_dora(
dora.launch_dora(
plan,
plan,
dora_config_template,
dora_config_template,
all_cl_contexts,
all_participants,
args_with_right_defaults.participants,
el_cl_data_files_artifact_uuid,
el_cl_data_files_artifact_uuid,
network_params.electra_fork_epoch,
network_params.electra_fork_epoch,
network_params.network,
network_params.network,
...
...
src/assertoor/assertoor_launcher.star
View file @
7e361913
...
@@ -42,17 +42,21 @@ def launch_assertoor(
...
@@ -42,17 +42,21 @@ def launch_assertoor(
vc_info = []
vc_info = []
for index, participant in enumerate(participant_contexts):
for index, participant in enumerate(participant_contexts):
participant_config = participant_configs[index]
(
cl_client = participant.cl_context
full_name,
el_client = participant.el_context
cl_client,
el_client,
participant_config,
) = shared_utils.get_client_names(
participant, index, participant_contexts, participant_configs
)
all_client_info.append(
all_client_info.append(
new_client_info(
new_client_info(
cl_client.ip_addr,
cl_client.ip_addr,
cl_client.http_port_num,
cl_client.http_port_num,
el_client.ip_addr,
el_client.ip_addr,
el_client.rpc_port_num,
el_client.rpc_port_num,
cl_client.beacon_service
_name,
full
_name,
)
)
)
)
...
@@ -162,11 +166,11 @@ def new_config_template_data(listen_port_num, client_info, vc_info, assertoor_pa
...
@@ -162,11 +166,11 @@ def new_config_template_data(listen_port_num, client_info, vc_info, assertoor_pa
}
}
def new_client_info(cl_ip_addr, cl_port_num, el_ip_addr, el_port_num,
service
_name):
def new_client_info(cl_ip_addr, cl_port_num, el_ip_addr, el_port_num,
full
_name):
return {
return {
"CLIPAddr": cl_ip_addr,
"CLIPAddr": cl_ip_addr,
"CLPortNum": cl_port_num,
"CLPortNum": cl_port_num,
"ELIPAddr": el_ip_addr,
"ELIPAddr": el_ip_addr,
"ELPortNum": el_port_num,
"ELPortNum": el_port_num,
"Name":
service
_name,
"Name":
full
_name,
}
}
src/cl/cl_launcher.star
View file @
7e361913
...
@@ -9,7 +9,7 @@ constants = import_module("../package_io/constants.star")
...
@@ -9,7 +9,7 @@ constants = import_module("../package_io/constants.star")
input_parser = import_module("../package_io/input_parser.star")
input_parser = import_module("../package_io/input_parser.star")
shared_utils = import_module("../shared_utils/shared_utils.star")
shared_utils = import_module("../shared_utils/shared_utils.star")
snooper = import_module("../snooper/snooper_engine_launcher.star")
engine_
snooper = import_module("../snooper/snooper_engine_launcher.star")
cl_context_BOOTNODE = None
cl_context_BOOTNODE = None
...
@@ -130,10 +130,10 @@ def launch(
...
@@ -130,10 +130,10 @@ def launch(
cl_context = None
cl_context = None
snooper_engine_context = None
snooper_engine_context = None
if participant.snooper_enabled:
if participant.snooper_enabled:
snooper_service_name = "snooper-{0}-{1}-{2}".format(
snooper_service_name = "snooper-
engine-
{0}-{1}-{2}".format(
index_str, cl_type, el_type
index_str, cl_type, el_type
)
)
snooper_engine_context = snooper.launch(
snooper_engine_context =
engine_
snooper.launch(
plan,
plan,
snooper_service_name,
snooper_service_name,
el_context,
el_context,
...
@@ -145,7 +145,7 @@ def launch(
...
@@ -145,7 +145,7 @@ def launch(
)
)
)
)
all_snooper_engine_contexts.append(snooper_engine_context)
all_snooper_engine_contexts.append(snooper_engine_context)
full_name = "{0}-{1}-{2}".format(index_str, el_type, cl_type)
if index == 0:
if index == 0:
cl_context = launch_method(
cl_context = launch_method(
plan,
plan,
...
@@ -156,6 +156,7 @@ def launch(
...
@@ -156,6 +156,7 @@ def launch(
global_log_level,
global_log_level,
cl_context_BOOTNODE,
cl_context_BOOTNODE,
el_context,
el_context,
full_name,
new_cl_node_validator_keystores,
new_cl_node_validator_keystores,
participant.cl_min_cpu,
participant.cl_min_cpu,
participant.cl_max_cpu,
participant.cl_max_cpu,
...
@@ -187,6 +188,7 @@ def launch(
...
@@ -187,6 +188,7 @@ def launch(
global_log_level,
global_log_level,
boot_cl_client_ctx,
boot_cl_client_ctx,
el_context,
el_context,
full_name,
new_cl_node_validator_keystores,
new_cl_node_validator_keystores,
participant.cl_min_cpu,
participant.cl_min_cpu,
participant.cl_max_cpu,
participant.cl_max_cpu,
...
...
src/cl/grandine/grandine_launcher.star
View file @
7e361913
...
@@ -69,6 +69,7 @@ def launch(
...
@@ -69,6 +69,7 @@ def launch(
global_log_level,
global_log_level,
bootnode_context,
bootnode_context,
el_context,
el_context,
full_name,
node_keystore_files,
node_keystore_files,
cl_min_cpu,
cl_min_cpu,
cl_max_cpu,
cl_max_cpu,
...
@@ -130,6 +131,7 @@ def launch(
...
@@ -130,6 +131,7 @@ def launch(
beacon_service_name,
beacon_service_name,
bootnode_context,
bootnode_context,
el_context,
el_context,
full_name,
log_level,
log_level,
node_keystore_files,
node_keystore_files,
cl_min_cpu,
cl_min_cpu,
...
@@ -207,6 +209,7 @@ def get_beacon_config(
...
@@ -207,6 +209,7 @@ def get_beacon_config(
service_name,
service_name,
bootnode_contexts,
bootnode_contexts,
el_context,
el_context,
full_name,
log_level,
log_level,
node_keystore_files,
node_keystore_files,
cl_min_cpu,
cl_min_cpu,
...
@@ -274,7 +277,7 @@ def get_beacon_config(
...
@@ -274,7 +277,7 @@ def get_beacon_config(
"--keystore-dir=" + validator_keys_dirpath,
"--keystore-dir=" + validator_keys_dirpath,
"--keystore-password-file=" + validator_secrets_dirpath,
"--keystore-password-file=" + validator_secrets_dirpath,
"--suggested-fee-recipient=" + constants.VALIDATING_REWARDS_ACCOUNT,
"--suggested-fee-recipient=" + constants.VALIDATING_REWARDS_ACCOUNT,
"--graffiti=" +
constants.CL_TYPE.grandine + "-" + el_context.client
_name,
"--graffiti=" +
full
_name,
]
]
if network not in constants.PUBLIC_NETWORKS:
if network not in constants.PUBLIC_NETWORKS:
...
...
src/cl/lighthouse/lighthouse_launcher.star
View file @
7e361913
...
@@ -71,6 +71,7 @@ def launch(
...
@@ -71,6 +71,7 @@ def launch(
global_log_level,
global_log_level,
bootnode_contexts,
bootnode_contexts,
el_context,
el_context,
full_name,
node_keystore_files,
node_keystore_files,
cl_min_cpu,
cl_min_cpu,
cl_max_cpu,
cl_max_cpu,
...
...
src/cl/lodestar/lodestar_launcher.star
View file @
7e361913
...
@@ -60,6 +60,7 @@ def launch(
...
@@ -60,6 +60,7 @@ def launch(
global_log_level,
global_log_level,
bootnode_contexts,
bootnode_contexts,
el_context,
el_context,
full_name,
node_keystore_files,
node_keystore_files,
cl_min_cpu,
cl_min_cpu,
cl_max_cpu,
cl_max_cpu,
...
...
src/cl/nimbus/nimbus_launcher.star
View file @
7e361913
...
@@ -82,6 +82,7 @@ def launch(
...
@@ -82,6 +82,7 @@ def launch(
global_log_level,
global_log_level,
bootnode_contexts,
bootnode_contexts,
el_context,
el_context,
full_name,
node_keystore_files,
node_keystore_files,
cl_min_cpu,
cl_min_cpu,
cl_max_cpu,
cl_max_cpu,
...
@@ -143,6 +144,7 @@ def launch(
...
@@ -143,6 +144,7 @@ def launch(
beacon_service_name,
beacon_service_name,
bootnode_contexts,
bootnode_contexts,
el_context,
el_context,
full_name,
log_level,
log_level,
node_keystore_files,
node_keystore_files,
cl_min_cpu,
cl_min_cpu,
...
@@ -219,6 +221,7 @@ def get_beacon_config(
...
@@ -219,6 +221,7 @@ def get_beacon_config(
service_name,
service_name,
bootnode_contexts,
bootnode_contexts,
el_context,
el_context,
full_name,
log_level,
log_level,
node_keystore_files,
node_keystore_files,
cl_min_cpu,
cl_min_cpu,
...
@@ -298,7 +301,7 @@ def get_beacon_config(
...
@@ -298,7 +301,7 @@ def get_beacon_config(
"--validators-dir=" + validator_keys_dirpath,
"--validators-dir=" + validator_keys_dirpath,
"--secrets-dir=" + validator_secrets_dirpath,
"--secrets-dir=" + validator_secrets_dirpath,
"--suggested-fee-recipient=" + constants.VALIDATING_REWARDS_ACCOUNT,
"--suggested-fee-recipient=" + constants.VALIDATING_REWARDS_ACCOUNT,
"--graffiti=" +
constants.CL_TYPE.nimbus + "-" + el_context.client
_name,
"--graffiti=" +
full
_name,
"--keymanager",
"--keymanager",
"--keymanager-port={0}".format(vc_shared.VALIDATOR_HTTP_PORT_NUM),
"--keymanager-port={0}".format(vc_shared.VALIDATOR_HTTP_PORT_NUM),
"--keymanager-address=0.0.0.0",
"--keymanager-address=0.0.0.0",
...
...
src/cl/prysm/prysm_launcher.star
View file @
7e361913
...
@@ -67,6 +67,7 @@ def launch(
...
@@ -67,6 +67,7 @@ def launch(
global_log_level,
global_log_level,
bootnode_contexts,
bootnode_contexts,
el_context,
el_context,
full_name,
node_keystore_files,
node_keystore_files,
cl_min_cpu,
cl_min_cpu,
cl_max_cpu,
cl_max_cpu,
...
...
src/cl/teku/teku_launcher.star
View file @
7e361913
...
@@ -71,6 +71,7 @@ def launch(
...
@@ -71,6 +71,7 @@ def launch(
global_log_level,
global_log_level,
bootnode_context,
bootnode_context,
el_context,
el_context,
full_name,
node_keystore_files,
node_keystore_files,
cl_min_cpu,
cl_min_cpu,
cl_max_cpu,
cl_max_cpu,
...
@@ -134,6 +135,7 @@ def launch(
...
@@ -134,6 +135,7 @@ def launch(
beacon_service_name,
beacon_service_name,
bootnode_context,
bootnode_context,
el_context,
el_context,
full_name,
log_level,
log_level,
node_keystore_files,
node_keystore_files,
cl_min_cpu,
cl_min_cpu,
...
@@ -213,6 +215,7 @@ def get_beacon_config(
...
@@ -213,6 +215,7 @@ def get_beacon_config(
service_name,
service_name,
bootnode_contexts,
bootnode_contexts,
el_context,
el_context,
full_name,
log_level,
log_level,
node_keystore_files,
node_keystore_files,
cl_min_cpu,
cl_min_cpu,
...
@@ -295,10 +298,7 @@ def get_beacon_config(
...
@@ -295,10 +298,7 @@ def get_beacon_config(
),
),
"--validators-proposer-default-fee-recipient="
"--validators-proposer-default-fee-recipient="
+ constants.VALIDATING_REWARDS_ACCOUNT,
+ constants.VALIDATING_REWARDS_ACCOUNT,
"--validators-graffiti="
"--validators-graffiti=" + full_name,
+ constants.CL_TYPE.teku
+ "-"
+ el_context.client_name,
"--validator-api-enabled=true",
"--validator-api-enabled=true",
"--validator-api-host-allowlist=*",
"--validator-api-host-allowlist=*",
"--validator-api-port={0}".format(vc_shared.VALIDATOR_HTTP_PORT_NUM),
"--validator-api-port={0}".format(vc_shared.VALIDATOR_HTTP_PORT_NUM),
...
...
src/dora/dora_launcher.star
View file @
7e361913
...
@@ -30,17 +30,23 @@ USED_PORTS = {
...
@@ -30,17 +30,23 @@ USED_PORTS = {
def launch_dora(
def launch_dora(
plan,
plan,
config_template,
config_template,
cl_contexts,
participant_contexts,
participant_configs,
el_cl_data_files_artifact_uuid,
el_cl_data_files_artifact_uuid,
electra_fork_epoch,
electra_fork_epoch,
network,
network,
global_node_selectors,
global_node_selectors,
):
):
all_cl_client_info = []
all_cl_client_info = []
for index, client in enumerate(cl_contexts):
for index, participant in enumerate(participant_contexts):
full_name, cl_client, _, _ = shared_utils.get_client_names(
participant, index, participant_contexts, participant_configs
)
all_cl_client_info.append(
all_cl_client_info.append(
new_cl_client_info(
new_cl_client_info(
client.ip_addr, client.http_port_num, client.beacon_service_name
cl_client.ip_addr,
cl_client.http_port_num,
full_name,
)
)
)
)
...
@@ -113,5 +119,9 @@ def new_config_template_data(network, listen_port_num, cl_client_info):
...
@@ -113,5 +119,9 @@ def new_config_template_data(network, listen_port_num, cl_client_info):
}
}
def new_cl_client_info(ip_addr, port_num, service_name):
def new_cl_client_info(ip_addr, port_num, full_name):
return {"IPAddr": ip_addr, "PortNum": port_num, "Name": service_name}
return {
"IPAddr": ip_addr,
"PortNum": port_num,
"FullName": full_name,
}
src/package_io/constants.star
View file @
7e361913
...
@@ -65,6 +65,8 @@ KEYMANAGER_P12_MOUNT_PATH_ON_CONTAINER = (
...
@@ -65,6 +65,8 @@ KEYMANAGER_P12_MOUNT_PATH_ON_CONTAINER = (
KEYMANAGER_P12_MOUNT_PATH_ON_CLIENTS + "/validator_keystore.p12"
KEYMANAGER_P12_MOUNT_PATH_ON_CLIENTS + "/validator_keystore.p12"
)
)
DEFAULT_SNOOPER_IMAGE = "ethpandaops/json-rpc-snoop:1.1.0"
GENESIS_FORK_VERSION = "0x10000038"
GENESIS_FORK_VERSION = "0x10000038"
BELLATRIX_FORK_VERSION = "0x30000038"
BELLATRIX_FORK_VERSION = "0x30000038"
CAPELLA_FORK_VERSION = "0x40000038"
CAPELLA_FORK_VERSION = "0x40000038"
...
...
src/participant_network.star
View file @
7e361913
...
@@ -24,6 +24,8 @@ el_client_launcher = import_module("./el/el_launcher.star")
...
@@ -24,6 +24,8 @@ el_client_launcher = import_module("./el/el_launcher.star")
cl_client_launcher = import_module("./cl/cl_launcher.star")
cl_client_launcher = import_module("./cl/cl_launcher.star")
vc = import_module("./vc/vc_launcher.star")
vc = import_module("./vc/vc_launcher.star")
beacon_snooper = import_module("./snooper/snooper_beacon_launcher.star")
def launch_participant_network(
def launch_participant_network(
plan,
plan,
...
@@ -185,6 +187,7 @@ def launch_participant_network(
...
@@ -185,6 +187,7 @@ def launch_participant_network(
all_ethereum_metrics_exporter_contexts = []
all_ethereum_metrics_exporter_contexts = []
all_xatu_sentry_contexts = []
all_xatu_sentry_contexts = []
all_vc_contexts = []
all_vc_contexts = []
all_snooper_beacon_contexts = []
# Some CL clients cannot run validator clients in the same process and need
# Some CL clients cannot run validator clients in the same process and need
# a separate validator client
# a separate validator client
_cls_that_need_separate_vc = [
_cls_that_need_separate_vc = [
...
@@ -192,6 +195,7 @@ def launch_participant_network(
...
@@ -192,6 +195,7 @@ def launch_participant_network(
constants.CL_TYPE.lodestar,
constants.CL_TYPE.lodestar,
constants.CL_TYPE.lighthouse,
constants.CL_TYPE.lighthouse,
]
]
for index, participant in enumerate(participants):
for index, participant in enumerate(participants):
el_type = participant.el_type
el_type = participant.el_type
cl_type = participant.cl_type
cl_type = participant.cl_type
...
@@ -199,6 +203,11 @@ def launch_participant_network(
...
@@ -199,6 +203,11 @@ def launch_participant_network(
index_str = shared_utils.zfill_custom(index + 1, len(str(len(participants))))
index_str = shared_utils.zfill_custom(index + 1, len(str(len(participants))))
el_context = all_el_contexts[index]
el_context = all_el_contexts[index]
cl_context = all_cl_contexts[index]
cl_context = all_cl_contexts[index]
node_selectors = input_parser.get_client_node_selectors(
participant.node_selectors,
global_node_selectors,
)
if participant.ethereum_metrics_exporter_enabled:
if participant.ethereum_metrics_exporter_enabled:
pair_name = "{0}-{1}-{2}".format(index_str, cl_type, el_type)
pair_name = "{0}-{1}-{2}".format(index_str, cl_type, el_type)
...
@@ -212,7 +221,7 @@ def launch_participant_network(
...
@@ -212,7 +221,7 @@ def launch_participant_network(
ethereum_metrics_exporter_service_name,
ethereum_metrics_exporter_service_name,
el_context,
el_context,
cl_context,
cl_context,
participant.
node_selectors,
node_selectors,
)
)
plan.print(
plan.print(
"Successfully added {0} ethereum metrics exporter participants".format(
"Successfully added {0} ethereum metrics exporter participants".format(
...
@@ -236,7 +245,7 @@ def launch_participant_network(
...
@@ -236,7 +245,7 @@ def launch_participant_network(
xatu_sentry_params,
xatu_sentry_params,
network_params,
network_params,
pair_name,
pair_name,
participant.
node_selectors,
node_selectors,
)
)
plan.print(
plan.print(
"Successfully added {0} xatu sentry participants".format(
"Successfully added {0} xatu sentry participants".format(
...
@@ -270,6 +279,42 @@ def launch_participant_network(
...
@@ -270,6 +279,42 @@ def launch_participant_network(
if participant.validator_count != 0:
if participant.validator_count != 0:
vc_keystores = preregistered_validator_keys_for_nodes[index]
vc_keystores = preregistered_validator_keys_for_nodes[index]
vc_context = None
snooper_beacon_context = None
if participant.snooper_enabled:
if (
participant.cl_type == constants.CL_TYPE.teku
or participant.cl_type == constants.CL_TYPE.nimbus
or participant.cl_type == constants.CL_TYPE.grandine
) and participant.use_separate_vc != False:
plan.print(
"Beacon snooper not supported for non split operation for {0}".format(
participant.cl_type
)
)
continue
snooper_service_name = "snooper-beacon-{0}-{1}-{2}".format(
index_str, cl_type, vc_type
)
snooper_beacon_context = beacon_snooper.launch(
plan,
snooper_service_name,
cl_context,
node_selectors,
)
plan.print(
"Successfully added {0} snooper participants".format(
snooper_beacon_context
)
)
all_snooper_beacon_contexts.append(snooper_beacon_context)
full_name = (
"{0}-{1}-{2}".format(index_str, el_type, cl_type) + "-{0}".format(vc_type)
if participant.cl_type != participant.vc_type
else "{0}-{1}-{2}".format(index_str, el_type, cl_type)
)
vc_context = vc.launch(
vc_context = vc.launch(
plan=plan,
plan=plan,
launcher=vc.new_vc_launcher(el_cl_genesis_data=el_cl_data),
launcher=vc.new_vc_launcher(el_cl_genesis_data=el_cl_data),
...
@@ -282,6 +327,9 @@ def launch_participant_network(
...
@@ -282,6 +327,9 @@ def launch_participant_network(
global_log_level=global_log_level,
global_log_level=global_log_level,
cl_context=cl_context,
cl_context=cl_context,
el_context=el_context,
el_context=el_context,
full_name=full_name,
snooper_enabled=participant.snooper_enabled,
snooper_beacon_context=snooper_beacon_context,
node_keystore_files=vc_keystores,
node_keystore_files=vc_keystores,
vc_min_cpu=participant.vc_min_cpu,
vc_min_cpu=participant.vc_min_cpu,
vc_max_cpu=participant.vc_max_cpu,
vc_max_cpu=participant.vc_max_cpu,
...
@@ -295,7 +343,7 @@ def launch_participant_network(
...
@@ -295,7 +343,7 @@ def launch_participant_network(
vc_tolerations=participant.vc_tolerations,
vc_tolerations=participant.vc_tolerations,
participant_tolerations=participant.tolerations,
participant_tolerations=participant.tolerations,
global_tolerations=global_tolerations,
global_tolerations=global_tolerations,
node_selectors=
participant.
node_selectors,
node_selectors=node_selectors,
network=network_params.network,
network=network_params.network,
electra_fork_epoch=network_params.electra_fork_epoch,
electra_fork_epoch=network_params.electra_fork_epoch,
)
)
...
...
src/prelaunch_data_generator/validator_keystores/validator_ranges_generator.star
View file @
7e361913
...
@@ -5,22 +5,24 @@ shared_utils = import_module("../../shared_utils/shared_utils.star")
...
@@ -5,22 +5,24 @@ shared_utils = import_module("../../shared_utils/shared_utils.star")
def generate_validator_ranges(
def generate_validator_ranges(
plan,
plan,
config_template,
config_template,
cl
_contexts,
participant
_contexts,
participants,
participant
_config
s,
):
):
data = []
data = []
running_total_validator_count = 0
running_total_validator_count = 0
for index, client in enumerate(cl_contexts):
for index, participant in enumerate(participant_contexts):
participant = participants[index]
full_name, _, _, _ = shared_utils.get_client_names(
participant, index, participant_contexts, participant_configs
)
participant = participant_configs[index]
if participant.validator_count == 0:
if participant.validator_count == 0:
continue
continue
start_index = running_total_validator_count
start_index = running_total_validator_count
running_total_validator_count += participant.validator_count
running_total_validator_count += participant.validator_count
end_index = start_index + participant.validator_count - 1
end_index = start_index + participant.validator_count - 1
service_name = client.beacon_service_name
data.append(
data.append(
{
{
"ClientName":
service
_name,
"ClientName":
full
_name,
"Range": "{0}-{1}".format(start_index, end_index),
"Range": "{0}-{1}".format(start_index, end_index),
}
}
)
)
...
...
src/shared_utils/shared_utils.star
View file @
7e361913
...
@@ -196,3 +196,20 @@ def calculate_devnet_url(network):
...
@@ -196,3 +196,20 @@ def calculate_devnet_url(network):
return "github.com/ethpandaops/{0}-devnets/network-configs/{1}{2}-{3}".format(
return "github.com/ethpandaops/{0}-devnets/network-configs/{1}{2}-{3}".format(
devnet_category, devnet_subname, network_type, devnet_number
devnet_category, devnet_subname, network_type, devnet_number
)
)
def get_client_names(participant, index, participant_contexts, participant_configs):
index_str = zfill_custom(index + 1, len(str(len(participant_contexts))))
participant_config = participant_configs[index]
cl_client = participant.cl_context
el_client = participant.el_context
vc_client = participant.vc_context
full_name = (
"{0}-{1}-{2}".format(index_str, el_client.client_name, cl_client.client_name)
+ "-{0}".format(vc_client.client_name)
if vc_client != None and cl_client.client_name != vc_client.client_name
else "{0}-{1}-{2}".format(
index_str, el_client.client_name, cl_client.client_name
)
)
return full_name, cl_client, el_client, participant_config
src/snooper/snooper_beacon_context.star
0 → 100644
View file @
7e361913
def new_snooper_beacon_client_context(ip_addr, beacon_rpc_port_num):
return struct(
ip_addr=ip_addr,
beacon_rpc_port_num=beacon_rpc_port_num,
)
src/snooper/snooper_beacon_launcher.star
0 → 100644
View file @
7e361913
shared_utils = import_module("../shared_utils/shared_utils.star")
constants = import_module("../package_io/constants.star")
input_parser = import_module("../package_io/input_parser.star")
cl_context = import_module("../cl/cl_context.star")
snooper_beacon_context = import_module("../snooper/snooper_beacon_context.star")
SNOOPER_BEACON_RPC_PORT_NUM = 8562
SNOOPER_BEACON_RPC_PORT_ID = "http"
SNOOPER_BINARY_COMMAND = "./json_rpc_snoop"
PRIVATE_IP_ADDRESS_PLACEHOLDER = "KURTOSIS_IP_ADDR_PLACEHOLDER"
SNOOPER_USED_PORTS = {
SNOOPER_BEACON_RPC_PORT_ID: shared_utils.new_port_spec(
SNOOPER_BEACON_RPC_PORT_NUM, shared_utils.TCP_PROTOCOL, wait="5s"
),
}
# The min/max CPU/memory that snooper can use
MIN_CPU = 10
MAX_CPU = 100
MIN_MEMORY = 10
MAX_MEMORY = 300
def launch(plan, service_name, cl_context, node_selectors):
snooper_service_name = "{0}".format(service_name)
snooper_config = get_config(service_name, cl_context, node_selectors)
snooper_service = plan.add_service(snooper_service_name, snooper_config)
snooper_http_port = snooper_service.ports[SNOOPER_BEACON_RPC_PORT_ID]
return snooper_beacon_context.new_snooper_beacon_client_context(
snooper_service.ip_address, SNOOPER_BEACON_RPC_PORT_NUM
)
def get_config(service_name, cl_context, node_selectors):
beacon_rpc_port_num = "http://{0}:{1}".format(
cl_context.ip_addr,
cl_context.http_port_num,
)
cmd = [
SNOOPER_BINARY_COMMAND,
"-b=0.0.0.0",
"-p={0}".format(SNOOPER_BEACON_RPC_PORT_NUM),
"{0}".format(beacon_rpc_port_num),
]
return ServiceConfig(
image=constants.DEFAULT_SNOOPER_IMAGE,
ports=SNOOPER_USED_PORTS,
cmd=cmd,
private_ip_address_placeholder=PRIVATE_IP_ADDRESS_PLACEHOLDER,
min_cpu=MIN_CPU,
max_cpu=MAX_CPU,
min_memory=MIN_MEMORY,
max_memory=MAX_MEMORY,
node_selectors=node_selectors,
)
src/snooper/snooper_engine_launcher.star
View file @
7e361913
shared_utils = import_module("../shared_utils/shared_utils.star")
shared_utils = import_module("../shared_utils/shared_utils.star")
constants = import_module("../package_io/constants.star")
input_parser = import_module("../package_io/input_parser.star")
input_parser = import_module("../package_io/input_parser.star")
el_context = import_module("../el/el_context.star")
el_context = import_module("../el/el_context.star")
el_admin_node_info = import_module("../el/el_admin_node_info.star")
snooper_engine_context = import_module("../snooper/snooper_engine_context.star")
snooper_engine_context = import_module("../snooper/snooper_engine_context.star")
SNOOPER_ENGINE_RPC_PORT_NUM = 8561
SNOOPER_ENGINE_RPC_PORT_NUM = 8561
...
@@ -10,14 +10,13 @@ SNOOPER_BINARY_COMMAND = "./json_rpc_snoop"
...
@@ -10,14 +10,13 @@ SNOOPER_BINARY_COMMAND = "./json_rpc_snoop"
PRIVATE_IP_ADDRESS_PLACEHOLDER = "KURTOSIS_IP_ADDR_PLACEHOLDER"
PRIVATE_IP_ADDRESS_PLACEHOLDER = "KURTOSIS_IP_ADDR_PLACEHOLDER"
DEFAULT_SNOOPER_IMAGE = "ethpandaops/json-rpc-snoop:1.1.0"
SNOOPER_USED_PORTS = {
SNOOPER_USED_PORTS = {
SNOOPER_ENGINE_RPC_PORT_ID: shared_utils.new_port_spec(
SNOOPER_ENGINE_RPC_PORT_ID: shared_utils.new_port_spec(
SNOOPER_ENGINE_RPC_PORT_NUM, shared_utils.TCP_PROTOCOL, wait="5s"
SNOOPER_ENGINE_RPC_PORT_NUM, shared_utils.TCP_PROTOCOL, wait="5s"
),
),
}
}
# The min/max CPU/memory that snooper can use
# The min/max CPU/memory that snooper can use
MIN_CPU = 10
MIN_CPU = 10
MAX_CPU = 100
MAX_CPU = 100
...
@@ -50,7 +49,7 @@ def get_config(service_name, el_context, node_selectors):
...
@@ -50,7 +49,7 @@ def get_config(service_name, el_context, node_selectors):
]
]
return ServiceConfig(
return ServiceConfig(
image=DEFAULT_SNOOPER_IMAGE,
image=
constants.
DEFAULT_SNOOPER_IMAGE,
ports=SNOOPER_USED_PORTS,
ports=SNOOPER_USED_PORTS,
cmd=cmd,
cmd=cmd,
private_ip_address_placeholder=PRIVATE_IP_ADDRESS_PLACEHOLDER,
private_ip_address_placeholder=PRIVATE_IP_ADDRESS_PLACEHOLDER,
...
...
src/vc/lighthouse.star
View file @
7e361913
...
@@ -23,6 +23,7 @@ def get_config(
...
@@ -23,6 +23,7 @@ def get_config(
beacon_http_url,
beacon_http_url,
cl_context,
cl_context,
el_context,
el_context,
full_name,
node_keystore_files,
node_keystore_files,
vc_min_cpu,
vc_min_cpu,
vc_max_cpu,
vc_max_cpu,
...
@@ -74,7 +75,7 @@ def get_config(
...
@@ -74,7 +75,7 @@ def get_config(
"--metrics-allow-origin=*",
"--metrics-allow-origin=*",
"--metrics-port={0}".format(vc_shared.VALIDATOR_CLIENT_METRICS_PORT_NUM),
"--metrics-port={0}".format(vc_shared.VALIDATOR_CLIENT_METRICS_PORT_NUM),
# ^^^^^^^^^^^^^^^^^^^ PROMETHEUS CONFIG ^^^^^^^^^^^^^^^^^^^^^
# ^^^^^^^^^^^^^^^^^^^ PROMETHEUS CONFIG ^^^^^^^^^^^^^^^^^^^^^
"--graffiti=" +
cl_context.client_name + "-" + el_context.client
_name,
"--graffiti=" +
full
_name,
]
]
if not (constants.NETWORK_NAME.verkle in network or electra_fork_epoch != None):
if not (constants.NETWORK_NAME.verkle in network or electra_fork_epoch != None):
...
...
src/vc/lodestar.star
View file @
7e361913
...
@@ -20,6 +20,7 @@ def get_config(
...
@@ -20,6 +20,7 @@ def get_config(
beacon_http_url,
beacon_http_url,
cl_context,
cl_context,
el_context,
el_context,
full_name,
node_keystore_files,
node_keystore_files,
vc_min_cpu,
vc_min_cpu,
vc_max_cpu,
vc_max_cpu,
...
@@ -65,7 +66,7 @@ def get_config(
...
@@ -65,7 +66,7 @@ def get_config(
"--metrics.address=0.0.0.0",
"--metrics.address=0.0.0.0",
"--metrics.port={0}".format(vc_shared.VALIDATOR_CLIENT_METRICS_PORT_NUM),
"--metrics.port={0}".format(vc_shared.VALIDATOR_CLIENT_METRICS_PORT_NUM),
# ^^^^^^^^^^^^^^^^^^^ PROMETHEUS CONFIG ^^^^^^^^^^^^^^^^^^^^^
# ^^^^^^^^^^^^^^^^^^^ PROMETHEUS CONFIG ^^^^^^^^^^^^^^^^^^^^^
"--graffiti=" +
cl_context.client_name + "-" + el_context.client
_name,
"--graffiti=" +
full
_name,
"--useProduceBlockV3",
"--useProduceBlockV3",
]
]
...
...
src/vc/nimbus.star
View file @
7e361913
...
@@ -10,6 +10,7 @@ def get_config(
...
@@ -10,6 +10,7 @@ def get_config(
beacon_http_url,
beacon_http_url,
cl_context,
cl_context,
el_context,
el_context,
full_name,
node_keystore_files,
node_keystore_files,
vc_min_cpu,
vc_min_cpu,
vc_max_cpu,
vc_max_cpu,
...
@@ -47,7 +48,7 @@ def get_config(
...
@@ -47,7 +48,7 @@ def get_config(
"--metrics",
"--metrics",
"--metrics-address=0.0.0.0",
"--metrics-address=0.0.0.0",
"--metrics-port={0}".format(vc_shared.VALIDATOR_CLIENT_METRICS_PORT_NUM),
"--metrics-port={0}".format(vc_shared.VALIDATOR_CLIENT_METRICS_PORT_NUM),
"--graffiti=" +
cl_context.client_name + "-" + el_context.client
_name,
"--graffiti=" +
full
_name,
]
]
if len(extra_params) > 0:
if len(extra_params) > 0:
...
...
src/vc/prysm.star
View file @
7e361913
...
@@ -12,6 +12,7 @@ def get_config(
...
@@ -12,6 +12,7 @@ def get_config(
beacon_http_url,
beacon_http_url,
cl_context,
cl_context,
el_context,
el_context,
full_name,
node_keystore_files,
node_keystore_files,
vc_min_cpu,
vc_min_cpu,
vc_max_cpu,
vc_max_cpu,
...
@@ -56,7 +57,7 @@ def get_config(
...
@@ -56,7 +57,7 @@ def get_config(
"--monitoring-host=0.0.0.0",
"--monitoring-host=0.0.0.0",
"--monitoring-port={0}".format(vc_shared.VALIDATOR_CLIENT_METRICS_PORT_NUM),
"--monitoring-port={0}".format(vc_shared.VALIDATOR_CLIENT_METRICS_PORT_NUM),
# ^^^^^^^^^^^^^^^^^^^ METRICS CONFIG ^^^^^^^^^^^^^^^^^^^^^
# ^^^^^^^^^^^^^^^^^^^ METRICS CONFIG ^^^^^^^^^^^^^^^^^^^^^
"--graffiti=" +
cl_context.client_name + "-" + el_context.client
_name,
"--graffiti=" +
full
_name,
]
]
if len(extra_params) > 0:
if len(extra_params) > 0:
...
...
src/vc/teku.star
View file @
7e361913
...
@@ -11,6 +11,7 @@ def get_config(
...
@@ -11,6 +11,7 @@ def get_config(
beacon_http_url,
beacon_http_url,
cl_context,
cl_context,
el_context,
el_context,
full_name,
node_keystore_files,
node_keystore_files,
vc_min_cpu,
vc_min_cpu,
vc_max_cpu,
vc_max_cpu,
...
@@ -46,10 +47,7 @@ def get_config(
...
@@ -46,10 +47,7 @@ def get_config(
),
),
"--validators-proposer-default-fee-recipient="
"--validators-proposer-default-fee-recipient="
+ constants.VALIDATING_REWARDS_ACCOUNT,
+ constants.VALIDATING_REWARDS_ACCOUNT,
"--validators-graffiti="
"--validators-graffiti=" + full_name,
+ cl_context.client_name
+ "-"
+ el_context.client_name,
"--validator-api-enabled=true",
"--validator-api-enabled=true",
"--validator-api-host-allowlist=*",
"--validator-api-host-allowlist=*",
"--validator-api-port={0}".format(vc_shared.VALIDATOR_HTTP_PORT_NUM),
"--validator-api-port={0}".format(vc_shared.VALIDATOR_HTTP_PORT_NUM),
...
...
src/vc/vc_context.star
View file @
7e361913
def new_vc_context(
def new_vc_context(
service_name,
client_name,
client_name,
service_name,
metrics_info,
metrics_info,
):
):
return struct(
return struct(
service_name=service_name,
client_name=client_name,
client_name=client_name,
service_name=service_name,
metrics_info=metrics_info,
metrics_info=metrics_info,
)
)
src/vc/vc_launcher.star
View file @
7e361913
...
@@ -29,6 +29,9 @@ def launch(
...
@@ -29,6 +29,9 @@ def launch(
global_log_level,
global_log_level,
cl_context,
cl_context,
el_context,
el_context,
full_name,
snooper_enabled,
snooper_beacon_context,
node_keystore_files,
node_keystore_files,
vc_min_cpu,
vc_min_cpu,
vc_max_cpu,
vc_max_cpu,
...
@@ -53,10 +56,16 @@ def launch(
...
@@ -53,10 +56,16 @@ def launch(
vc_tolerations, participant_tolerations, global_tolerations
vc_tolerations, participant_tolerations, global_tolerations
)
)
beacon_http_url = "http://{}:{}".format(
if snooper_enabled:
cl_context.ip_addr,
beacon_http_url = "http://{}:{}".format(
cl_context.http_port_num,
snooper_beacon_context.ip_addr,
)
snooper_beacon_context.beacon_rpc_port_num,
)
else:
beacon_http_url = "http://{}:{}".format(
cl_context.ip_addr,
cl_context.http_port_num,
)
vc_min_cpu = int(vc_min_cpu) if int(vc_min_cpu) > 0 else MIN_CPU
vc_min_cpu = int(vc_min_cpu) if int(vc_min_cpu) > 0 else MIN_CPU
vc_max_cpu = int(vc_max_cpu) if int(vc_max_cpu) > 0 else MAX_CPU
vc_max_cpu = int(vc_max_cpu) if int(vc_max_cpu) > 0 else MAX_CPU
...
@@ -72,6 +81,7 @@ def launch(
...
@@ -72,6 +81,7 @@ def launch(
beacon_http_url=beacon_http_url,
beacon_http_url=beacon_http_url,
cl_context=cl_context,
cl_context=cl_context,
el_context=el_context,
el_context=el_context,
full_name=full_name,
node_keystore_files=node_keystore_files,
node_keystore_files=node_keystore_files,
vc_min_cpu=vc_min_cpu,
vc_min_cpu=vc_min_cpu,
vc_max_cpu=vc_max_cpu,
vc_max_cpu=vc_max_cpu,
...
@@ -94,6 +104,7 @@ def launch(
...
@@ -94,6 +104,7 @@ def launch(
beacon_http_url=beacon_http_url,
beacon_http_url=beacon_http_url,
cl_context=cl_context,
cl_context=cl_context,
el_context=el_context,
el_context=el_context,
full_name=full_name,
node_keystore_files=node_keystore_files,
node_keystore_files=node_keystore_files,
vc_min_cpu=vc_min_cpu,
vc_min_cpu=vc_min_cpu,
vc_max_cpu=vc_max_cpu,
vc_max_cpu=vc_max_cpu,
...
@@ -114,6 +125,7 @@ def launch(
...
@@ -114,6 +125,7 @@ def launch(
beacon_http_url=beacon_http_url,
beacon_http_url=beacon_http_url,
cl_context=cl_context,
cl_context=cl_context,
el_context=el_context,
el_context=el_context,
full_name=full_name,
node_keystore_files=node_keystore_files,
node_keystore_files=node_keystore_files,
vc_min_cpu=vc_min_cpu,
vc_min_cpu=vc_min_cpu,
vc_max_cpu=vc_max_cpu,
vc_max_cpu=vc_max_cpu,
...
@@ -133,6 +145,7 @@ def launch(
...
@@ -133,6 +145,7 @@ def launch(
beacon_http_url=beacon_http_url,
beacon_http_url=beacon_http_url,
cl_context=cl_context,
cl_context=cl_context,
el_context=el_context,
el_context=el_context,
full_name=full_name,
node_keystore_files=node_keystore_files,
node_keystore_files=node_keystore_files,
vc_min_cpu=vc_min_cpu,
vc_min_cpu=vc_min_cpu,
vc_max_cpu=vc_max_cpu,
vc_max_cpu=vc_max_cpu,
...
@@ -158,6 +171,7 @@ def launch(
...
@@ -158,6 +171,7 @@ def launch(
beacon_http_url=beacon_http_url,
beacon_http_url=beacon_http_url,
cl_context=cl_context,
cl_context=cl_context,
el_context=el_context,
el_context=el_context,
full_name=full_name,
node_keystore_files=node_keystore_files,
node_keystore_files=node_keystore_files,
vc_min_cpu=vc_min_cpu,
vc_min_cpu=vc_min_cpu,
vc_max_cpu=vc_max_cpu,
vc_max_cpu=vc_max_cpu,
...
@@ -171,6 +185,8 @@ def launch(
...
@@ -171,6 +185,8 @@ def launch(
tolerations=tolerations,
tolerations=tolerations,
node_selectors=node_selectors,
node_selectors=node_selectors,
)
)
elif vc_type == constants.VC_TYPE.grandine:
fail("Grandine VC is not yet supported")
else:
else:
fail("Unsupported vc_type: {0}".format(vc_type))
fail("Unsupported vc_type: {0}".format(vc_type))
...
@@ -189,8 +205,8 @@ def launch(
...
@@ -189,8 +205,8 @@ def launch(
validator_http_port = validator_service.ports[vc_shared.VALIDATOR_HTTP_PORT_ID]
validator_http_port = validator_service.ports[vc_shared.VALIDATOR_HTTP_PORT_ID]
return vc_context.new_vc_context(
return vc_context.new_vc_context(
service_name=service_name,
client_name=vc_type,
client_name=vc_type,
service_name=service_name,
metrics_info=validator_node_metrics_info,
metrics_info=validator_node_metrics_info,
)
)
...
...
static_files/dora-config/config.yaml.tmpl
View file @
7e361913
...
@@ -35,7 +35,7 @@ beaconapi:
...
@@ -35,7 +35,7 @@ beaconapi:
endpoints:
endpoints:
{{ range $clClient := .CLClientInfo }}
{{ range $clClient := .CLClientInfo }}
- url: "http://{{ $clClient.IPAddr }}:{{ $clClient.PortNum }}"
- url: "http://{{ $clClient.IPAddr }}:{{ $clClient.PortNum }}"
name: "{{ $clClient.Name }}"
name: "{{ $clClient.
Full
Name }}"
archive: true
archive: true
{{- end }}
{{- end }}
# local cache for page models
# local cache for page models
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment