Commit 836eda4e authored by Barnabas Busa's avatar Barnabas Busa Committed by GitHub

feat: add keymanager to all validator processes (#502)

parent f9343a29
......@@ -6,7 +6,7 @@ orbs:
executors:
ubuntu_vm:
machine:
image: ubuntu-2204:2023.07.2
image: ubuntu-2204:current
parameters:
should-enable-check-latest-version-workflow:
......
......@@ -2,7 +2,6 @@ participants:
- el_client_type: geth
el_client_image: ethpandaops/geth:master
cl_client_type: lighthouse
cl_client_image: ethpandaops/lighthouse:sidecar-inclusion-proof-c6be31c
blobber_enabled: true
blobber_extra_params:
- --proposal-action-frequency=1
......@@ -11,7 +10,6 @@ participants:
- el_client_type: geth
el_client_image: ethpandaops/geth:master
cl_client_type: lodestar
cl_client_image: ethpandaops/lodestar:blobs-inclproof-d5a5a47
count: 1
network_params:
deneb_fork_epoch: 1
......
participants:
- el_client_type: geth
cl_client_type: teku
use_separate_validator_client: true
validator_count: 0
use_separate_validator_client: true
- el_client_type: nethermind
cl_client_type: teku
use_separate_validator_client: true
......
......@@ -75,6 +75,14 @@ def run(plan, args={}):
src=static_files.JWT_PATH_FILEPATH,
name="jwt_file",
)
keymanager_file = plan.upload_files(
src=static_files.KEYMANAGER_PATH_FILEPATH,
name="keymanager_file",
)
keymanager_p12_file = plan.upload_files(
src=static_files.KEYMANAGER_P12_PATH_FILEPATH,
name="keymanager_p12_file",
)
plan.print("Read the prometheus, grafana templates")
plan.print(
......@@ -93,6 +101,8 @@ def run(plan, args={}):
network_params,
args_with_right_defaults.global_client_log_level,
jwt_file,
keymanager_file,
keymanager_p12_file,
persistent,
xatu_sentry_params,
global_tolerations,
......
......@@ -5,7 +5,7 @@ cl_client_context = import_module("../../cl/cl_client_context.star")
cl_node_ready_conditions = import_module("../../cl/cl_node_ready_conditions.star")
node_metrics = import_module("../../node_metrics_info.star")
constants = import_module("../../package_io/constants.star")
validator_client_shared = import_module("../../validator_client/shared.star")
# ---------------------------------- Beacon client -------------------------------------
# Nimbus requires that its data directory already exists (because it expects you to bind-mount it), so we
# have to to create it
......@@ -15,6 +15,7 @@ BEACON_TCP_DISCOVERY_PORT_ID = "tcp-discovery"
BEACON_UDP_DISCOVERY_PORT_ID = "udp-discovery"
BEACON_HTTP_PORT_ID = "http"
BEACON_METRICS_PORT_ID = "metrics"
VALIDATOR_HTTP_PORT_ID = "http-validator"
# Port nums
BEACON_DISCOVERY_PORT_NUM = 9000
......@@ -135,6 +136,7 @@ def launch(
plan,
launcher.el_cl_genesis_data,
launcher.jwt_file,
launcher.keymanager_file,
launcher.network,
image,
beacon_service_name,
......@@ -209,6 +211,7 @@ def get_beacon_config(
plan,
el_cl_genesis_data,
jwt_file,
keymanager_file,
network,
image,
service_name,
......@@ -296,11 +299,13 @@ def get_beacon_config(
+ constants.CL_CLIENT_TYPE.nimbus
+ "-"
+ el_client_context.client_name,
"--keymanager",
"--keymanager-port={0}".format(validator_client_shared.VALIDATOR_HTTP_PORT_NUM),
"--keymanager-address=0.0.0.0",
"--keymanager-allow-origin=*",
"--keymanager-token-file=" + constants.KEYMANAGER_MOUNT_PATH_ON_CONTAINER,
]
if node_keystore_files != None and not use_separate_validator_client:
cmd.extend(validator_flags)
if network not in constants.PUBLIC_NETWORKS:
cmd.append(
"--bootstrap-file="
......@@ -325,10 +330,22 @@ def get_beacon_config(
constants.GENESIS_DATA_MOUNTPOINT_ON_CLIENTS: el_cl_genesis_data.files_artifact_uuid,
constants.JWT_MOUNTPOINT_ON_CLIENTS: jwt_file,
}
beacon_validator_used_ports = {}
beacon_validator_used_ports.update(BEACON_USED_PORTS)
if node_keystore_files != None and not use_separate_validator_client:
validator_http_port_id_spec = shared_utils.new_port_spec(
validator_client_shared.VALIDATOR_HTTP_PORT_NUM,
shared_utils.TCP_PROTOCOL,
shared_utils.HTTP_APPLICATION_PROTOCOL,
)
beacon_validator_used_ports.update(
{VALIDATOR_HTTP_PORT_ID: validator_http_port_id_spec}
)
cmd.extend(validator_flags)
files[
VALIDATOR_KEYS_MOUNTPOINT_ON_CLIENTS
] = node_keystore_files.files_artifact_uuid
files[constants.KEYMANAGER_MOUNT_PATH_ON_CLIENTS] = keymanager_file
if persistent:
files[BEACON_DATA_DIRPATH_ON_SERVICE_CONTAINER] = Directory(
......@@ -338,7 +355,7 @@ def get_beacon_config(
return ServiceConfig(
image=image,
ports=BEACON_USED_PORTS,
ports=beacon_validator_used_ports,
cmd=cmd,
files=files,
private_ip_address_placeholder=PRIVATE_IP_ADDRESS_PLACEHOLDER,
......@@ -362,9 +379,10 @@ def get_beacon_config(
)
def new_nimbus_launcher(el_cl_genesis_data, jwt_file, network):
def new_nimbus_launcher(el_cl_genesis_data, jwt_file, network, keymanager_file):
return struct(
el_cl_genesis_data=el_cl_genesis_data,
jwt_file=jwt_file,
network=network,
keymanager_file=keymanager_file,
)
......@@ -4,9 +4,10 @@ cl_client_context = import_module("../../cl/cl_client_context.star")
node_metrics = import_module("../../node_metrics_info.star")
cl_node_ready_conditions = import_module("../../cl/cl_node_ready_conditions.star")
constants = import_module("../../package_io/constants.star")
validator_client_shared = import_module("../../validator_client/shared.star")
# ---------------------------------- Beacon client -------------------------------------
TEKU_BINARY_FILEPATH_IN_IMAGE = "/opt/teku/bin/teku"
# ---------------------------------- Beacon client -------------------------------------
# The Docker container runs as the "teku" user so we can't write to root
BEACON_DATA_DIRPATH_ON_SERVICE_CONTAINER = "/data/teku/teku-beacon-data"
......@@ -15,6 +16,7 @@ BEACON_TCP_DISCOVERY_PORT_ID = "tcp-discovery"
BEACON_UDP_DISCOVERY_PORT_ID = "udp-discovery"
BEACON_HTTP_PORT_ID = "http"
BEACON_METRICS_PORT_ID = "metrics"
VALIDATOR_HTTP_PORT_ID = "http-validator"
# Port nums
BEACON_DISCOVERY_PORT_NUM = 9000
......@@ -124,6 +126,8 @@ def launch(
plan,
launcher.el_cl_genesis_data,
launcher.jwt_file,
launcher.keymanager_file,
launcher.keymanager_p12_file,
launcher.network,
image,
beacon_service_name,
......@@ -200,6 +204,8 @@ def get_beacon_config(
plan,
el_cl_genesis_data,
jwt_file,
keymanager_file,
keymanager_p12_file,
network,
image,
service_name,
......@@ -290,11 +296,19 @@ def get_beacon_config(
+ constants.CL_CLIENT_TYPE.teku
+ "-"
+ el_client_context.client_name,
"--validator-api-enabled=true",
"--validator-api-host-allowlist=*",
"--validator-api-port={0}".format(
validator_client_shared.VALIDATOR_HTTP_PORT_NUM
),
"--validator-api-interface=0.0.0.0",
"--validator-api-keystore-file="
+ constants.KEYMANAGER_P12_MOUNT_PATH_ON_CONTAINER,
"--validator-api-keystore-password-file="
+ constants.KEYMANAGER_MOUNT_PATH_ON_CONTAINER,
"--validator-api-docs-enabled=true",
]
if node_keystore_files != None and not use_separate_validator_client:
cmd.extend(validator_flags)
if network not in constants.PUBLIC_NETWORKS:
cmd.append(
"--initial-state="
......@@ -366,10 +380,23 @@ def get_beacon_config(
constants.GENESIS_DATA_MOUNTPOINT_ON_CLIENTS: el_cl_genesis_data.files_artifact_uuid,
constants.JWT_MOUNTPOINT_ON_CLIENTS: jwt_file,
}
beacon_validator_used_ports = {}
beacon_validator_used_ports.update(BEACON_USED_PORTS)
if node_keystore_files != None and not use_separate_validator_client:
validator_http_port_id_spec = shared_utils.new_port_spec(
validator_client_shared.VALIDATOR_HTTP_PORT_NUM,
shared_utils.TCP_PROTOCOL,
shared_utils.HTTP_APPLICATION_PROTOCOL,
)
beacon_validator_used_ports.update(
{VALIDATOR_HTTP_PORT_ID: validator_http_port_id_spec}
)
cmd.extend(validator_flags)
files[
VALIDATOR_KEYS_DIRPATH_ON_SERVICE_CONTAINER
] = node_keystore_files.files_artifact_uuid
files[constants.KEYMANAGER_MOUNT_PATH_ON_CLIENTS] = keymanager_file
files[constants.KEYMANAGER_P12_MOUNT_PATH_ON_CLIENTS] = keymanager_p12_file
if persistent:
files[BEACON_DATA_DIRPATH_ON_SERVICE_CONTAINER] = Directory(
......@@ -378,7 +405,7 @@ def get_beacon_config(
)
return ServiceConfig(
image=image,
ports=BEACON_USED_PORTS,
ports=beacon_validator_used_ports,
cmd=cmd,
# entrypoint=ENTRYPOINT_ARGS,
files=files,
......@@ -403,7 +430,13 @@ def get_beacon_config(
)
def new_teku_launcher(el_cl_genesis_data, jwt_file, network):
def new_teku_launcher(
el_cl_genesis_data, jwt_file, network, keymanager_file, keymanager_p12_file
):
return struct(
el_cl_genesis_data=el_cl_genesis_data, jwt_file=jwt_file, network=network
el_cl_genesis_data=el_cl_genesis_data,
jwt_file=jwt_file,
network=network,
keymanager_file=keymanager_file,
keymanager_p12_file=keymanager_p12_file,
)
......@@ -55,6 +55,14 @@ GENESIS_CONFIG_MOUNT_PATH_ON_CONTAINER = (
JWT_MOUNTPOINT_ON_CLIENTS = "/jwt"
JWT_MOUNT_PATH_ON_CONTAINER = JWT_MOUNTPOINT_ON_CLIENTS + "/jwtsecret"
KEYMANAGER_MOUNT_PATH_ON_CLIENTS = "/keymanager"
KEYMANAGER_MOUNT_PATH_ON_CONTAINER = (
KEYMANAGER_MOUNT_PATH_ON_CLIENTS + "/keymanager.txt"
)
KEYMANAGER_P12_MOUNT_PATH_ON_CLIENTS = "/keymanager-p12"
KEYMANAGER_P12_MOUNT_PATH_ON_CONTAINER = (
KEYMANAGER_P12_MOUNT_PATH_ON_CLIENTS + "/validator_keystore.p12"
)
GENESIS_FORK_VERSION = "0x10000038"
BELLATRIX_FORK_VERSION = "0x30000038"
......
......@@ -68,6 +68,8 @@ def launch_participant_network(
network_params,
global_log_level,
jwt_file,
keymanager_file,
keymanager_p12_file,
persistent,
xatu_sentry_params,
global_tolerations,
......@@ -523,7 +525,7 @@ def launch_participant_network(
},
constants.CL_CLIENT_TYPE.nimbus: {
"launcher": nimbus.new_nimbus_launcher(
el_cl_data, jwt_file, network_params.network
el_cl_data, jwt_file, network_params.network, keymanager_file
),
"launch_method": nimbus.launch,
},
......@@ -542,6 +544,8 @@ def launch_participant_network(
el_cl_data,
jwt_file,
network_params.network,
keymanager_file,
keymanager_p12_file,
),
"launch_method": teku.launch,
},
......@@ -775,6 +779,8 @@ def launch_participant_network(
launcher=validator_client.new_validator_client_launcher(
el_cl_genesis_data=el_cl_data
),
keymanager_file=keymanager_file,
keymanager_p12_file=keymanager_p12_file,
service_name="vc-{0}-{1}-{2}".format(
index_str, validator_client_type, el_client_type
),
......@@ -797,6 +803,8 @@ def launch_participant_network(
participant_tolerations=participant.tolerations,
global_tolerations=global_tolerations,
node_selectors=node_selectors,
network=network_params.network, # TODO: remove when deneb rebase is done
electra_fork_epoch=network_params.electra_fork_epoch, # TODO: remove when deneb rebase is done
)
all_validator_client_contexts.append(validator_client_context)
......
......@@ -68,5 +68,9 @@ CL_GENESIS_GENERATION_MNEMONICS_TEMPLATE_FILEPATH = (
)
JWT_PATH_FILEPATH = STATIC_FILES_DIRPATH + "/jwt/jwtsecret"
KEYMANAGER_PATH_FILEPATH = STATIC_FILES_DIRPATH + "/keymanager/keymanager.txt"
KEYMANAGER_P12_PATH_FILEPATH = (
STATIC_FILES_DIRPATH + "/keymanager/validator_keystore.p12"
)
SHADOWFORK_FILEPATH = "/network-configs/latest_block.json"
......@@ -32,6 +32,8 @@ def get_config(
extra_labels,
tolerations,
node_selectors,
network,
electra_fork_epoch,
):
log_level = input_parser.get_client_log_level_or_default(
participant_log_level, global_log_level, VERBOSITY_LEVELS
......@@ -60,6 +62,11 @@ def get_config(
# "--enable-doppelganger-protection", // Disabled to not have to wait 2 epochs before validator can start
# burn address - If unset, the validator will scream in its logs
"--suggested-fee-recipient=" + constants.VALIDATING_REWARDS_ACCOUNT,
"--http",
"--http-port={0}".format(validator_client_shared.VALIDATOR_HTTP_PORT_NUM),
"--http-address=0.0.0.0",
"--http-allow-origin=*",
"--unencrypted-http-transport",
# vvvvvvvvvvvvvvvvvvv PROMETHEUS CONFIG vvvvvvvvvvvvvvvvvvvvv
"--metrics",
"--metrics-address=0.0.0.0",
......@@ -74,6 +81,9 @@ def get_config(
+ el_client_context.client_name,
]
if not (constants.NETWORK_NAME.verkle in network and electra_fork_epoch == None):
cmd.append("--produce-block-v3")
if len(extra_params):
cmd.extend([param for param in extra_params])
......
......@@ -54,6 +54,11 @@ def get_config(
"--keystoresDir=" + validator_keys_dirpath,
"--secretsDir=" + validator_secrets_dirpath,
"--suggestedFeeRecipient=" + constants.VALIDATING_REWARDS_ACCOUNT,
"--keymanager",
"--keymanager.authEnabled=true",
"--keymanager.port={0}".format(validator_client_shared.VALIDATOR_HTTP_PORT_NUM),
"--keymanager.address=0.0.0.0",
"--keymanager.cors=*",
# vvvvvvvvvvvvvvvvvvv PROMETHEUS CONFIG vvvvvvvvvvvvvvvvvvvvv
"--metrics",
"--metrics.address=0.0.0.0",
......@@ -65,6 +70,7 @@ def get_config(
+ cl_client_context.client_name
+ "-"
+ el_client_context.client_name,
"--useProduceBlockV3",
]
if len(extra_params) > 0:
......
......@@ -6,6 +6,7 @@ validator_client_shared = import_module("./shared.star")
def get_config(
el_cl_genesis_data,
image,
keymanager_file,
beacon_http_url,
cl_client_context,
el_client_context,
......@@ -36,6 +37,11 @@ def get_config(
"--validators-dir=" + validator_keys_dirpath,
"--secrets-dir=" + validator_secrets_dirpath,
"--suggested-fee-recipient=" + constants.VALIDATING_REWARDS_ACCOUNT,
"--keymanager",
"--keymanager-port={0}".format(validator_client_shared.VALIDATOR_HTTP_PORT_NUM),
"--keymanager-address=0.0.0.0",
"--keymanager-allow-origin=*",
"--keymanager-token-file=" + constants.KEYMANAGER_MOUNT_PATH_ON_CONTAINER,
# vvvvvvvvvvvvvvvvvvv METRICS CONFIG vvvvvvvvvvvvvvvvvvvvv
"--metrics",
"--metrics-address=0.0.0.0",
......@@ -54,6 +60,7 @@ def get_config(
files = {
validator_client_shared.VALIDATOR_CLIENT_KEYS_MOUNTPOINT: node_keystore_files.files_artifact_uuid,
constants.KEYMANAGER_MOUNT_PATH_ON_CLIENTS: keymanager_file,
}
return ServiceConfig(
......
......@@ -47,6 +47,9 @@ def get_config(
"--wallet-dir=" + validator_keys_dirpath,
"--wallet-password-file=" + validator_secrets_dirpath,
"--suggested-fee-recipient=" + constants.VALIDATING_REWARDS_ACCOUNT,
"--rpc",
"--rpc-port={0}".format(validator_client_shared.VALIDATOR_HTTP_PORT_NUM),
"--rpc-host=0.0.0.0",
# vvvvvvvvvvvvvvvvvvv METRICS CONFIG vvvvvvvvvvvvvvvvvvvvv
"--disable-monitoring=false",
"--monitoring-host=0.0.0.0",
......
......@@ -3,11 +3,19 @@ shared_utils = import_module("../shared_utils/shared_utils.star")
PRIVATE_IP_ADDRESS_PLACEHOLDER = "KURTOSIS_IP_ADDR_PLACEHOLDER"
VALIDATOR_CLIENT_KEYS_MOUNTPOINT = "/keystores"
VALIDATOR_HTTP_PORT_NUM = 5056
VALIDATOR_HTTP_PORT_ID = "http"
VALIDATOR_CLIENT_METRICS_PORT_NUM = 8080
VALIDATOR_CLIENT_METRICS_PORT_ID = "metrics"
METRICS_PATH = "/metrics"
VALIDATOR_CLIENT_USED_PORTS = {
VALIDATOR_HTTP_PORT_ID: shared_utils.new_port_spec(
VALIDATOR_HTTP_PORT_NUM,
shared_utils.TCP_PROTOCOL,
shared_utils.HTTP_APPLICATION_PROTOCOL,
),
VALIDATOR_CLIENT_METRICS_PORT_ID: shared_utils.new_port_spec(
VALIDATOR_CLIENT_METRICS_PORT_NUM,
shared_utils.TCP_PROTOCOL,
......
......@@ -5,6 +5,8 @@ validator_client_shared = import_module("./shared.star")
def get_config(
el_cl_genesis_data,
keymanager_file,
keymanager_p12_file,
image,
beacon_http_url,
cl_client_context,
......@@ -47,6 +49,16 @@ def get_config(
+ cl_client_context.client_name
+ "-"
+ el_client_context.client_name,
"--validator-api-enabled=true",
"--validator-api-host-allowlist=*",
"--validator-api-port={0}".format(
validator_client_shared.VALIDATOR_HTTP_PORT_NUM
),
"--validator-api-interface=0.0.0.0",
"--validator-api-keystore-file="
+ constants.KEYMANAGER_P12_MOUNT_PATH_ON_CONTAINER,
"--validator-api-keystore-password-file="
+ constants.KEYMANAGER_MOUNT_PATH_ON_CONTAINER,
# vvvvvvvvvvvvvvvvvvv METRICS CONFIG vvvvvvvvvvvvvvvvvvvvv
"--metrics-enabled=true",
"--metrics-host-allowlist=*",
......@@ -63,6 +75,8 @@ def get_config(
files = {
constants.GENESIS_DATA_MOUNTPOINT_ON_CLIENTS: el_cl_genesis_data.files_artifact_uuid,
validator_client_shared.VALIDATOR_CLIENT_KEYS_MOUNTPOINT: node_keystore_files.files_artifact_uuid,
constants.KEYMANAGER_MOUNT_PATH_ON_CLIENTS: keymanager_file,
constants.KEYMANAGER_P12_MOUNT_PATH_ON_CLIENTS: keymanager_p12_file,
}
return ServiceConfig(
......
......@@ -20,6 +20,8 @@ MAX_MEMORY = 512
def launch(
plan,
launcher,
keymanager_file,
keymanager_p12_file,
service_name,
validator_client_type,
image,
......@@ -40,6 +42,8 @@ def launch(
participant_tolerations,
global_tolerations,
node_selectors,
network, # TODO: remove when deneb rebase is done
electra_fork_epoch, # TODO: remove when deneb rebase is done
):
if node_keystore_files == None:
return None
......@@ -76,6 +80,8 @@ def launch(
extra_labels=extra_labels,
tolerations=tolerations,
node_selectors=node_selectors,
network=network, # TODO: remove when deneb rebase is done
electra_fork_epoch=electra_fork_epoch, # TODO: remove when deneb rebase is done
)
elif validator_client_type == constants.VC_CLIENT_TYPE.lodestar:
config = lodestar.get_config(
......@@ -99,6 +105,8 @@ def launch(
elif validator_client_type == constants.VC_CLIENT_TYPE.teku:
config = teku.get_config(
el_cl_genesis_data=launcher.el_cl_genesis_data,
keymanager_file=keymanager_file,
keymanager_p12_file=keymanager_p12_file,
image=image,
beacon_http_url=beacon_http_url,
cl_client_context=cl_client_context,
......@@ -116,6 +124,7 @@ def launch(
elif validator_client_type == constants.VC_CLIENT_TYPE.nimbus:
config = nimbus.get_config(
el_cl_genesis_data=launcher.el_cl_genesis_data,
keymanager_file=keymanager_file,
image=image,
beacon_http_url=beacon_http_url,
cl_client_context=cl_client_context,
......@@ -168,6 +177,10 @@ def launch(
service_name, validator_client_shared.METRICS_PATH, validator_metrics_url
)
validator_http_port = validator_service.ports[
validator_client_shared.VALIDATOR_HTTP_PORT_ID
]
return validator_client_context.new_validator_client_context(
service_name=service_name,
client_name=validator_client_type,
......
# To run this script, you need to have openssl installed on your machine
# This script generates a self-signed certificate and a private key, and then exports them to a PKCS12 keystore
# The keystore is encrypted with a password that is stored in a file called keymanager.txt
# The keystore is then saved to a file called validator_keystore.p12
# https://docs.teku.consensys.io/23.12.0/how-to/use-external-signer/manage-keys#support-multiple-domains-and-ips
openssl req -x509 -nodes -days 3650 -newkey rsa:2048 -config openssl.cnf | openssl pkcs12 -export -out validator_keystore.p12 -passout file:keymanager.txt
api-token-0x7443c65f8cb0eb4ef6ab78c173d085f28b349f40dda27c74604439e07848a6d4
\ No newline at end of file
[req]
distinguished_name = Kurtosis
x509_extensions = v3_req
prompt = no
[Kurtosis]
countryName = EU
stateOrProvinceName = CA
localityName = San Francisco
organizationName = Kurtosis
organizationalUnitName = ethereum-package
[v3_req]
subjectKeyIdentifier = hash
authorityKeyIdentifier = keyid,issuer
basicConstraints = CA:TRUE
subjectAltName = @alt_names
[alt_names]
DNS.1 = mydomain.com
DNS.2 = localhost
IP.1 = 127.0.0.1
IP.2 = 10.0.0.6
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment