Commit aabc942c authored by Gyanendra Mishra's avatar Gyanendra Mishra Committed by GitHub

feat: allow setting exit ip address (#584)

Co-authored-by: default avatarBarnabas Busa <busa.barnabas@gmail.com>
parent e0622a77
participants:
- el_type: geth
cl_type: teku
- el_type: nethermind
cl_type: prysm
- el_type: erigon
cl_type: nimbus
- el_type: besu
cl_type: lighthouse
- el_type: reth
cl_type: lodestar
- el_type: nimbus
cl_type: teku
- el_type: ethereumjs
cl_type: grandine
additional_services: []
port_publisher:
public_port_start: 50000
......@@ -33,6 +33,7 @@ jobs:
"./.github/tests/mev-mock.yaml",
"./.github/tests/mix-with-tools.yaml",
"./.github/tests/mix-persistence.yaml",
"./.github/tests/mix-public.yaml",
"./network_params.yaml"
]
runs-on: ubuntu-latest
......
......@@ -752,6 +752,17 @@ global_node_selectors: {}
# This will open up http ports to your validator services!
# Defaults to false
keymanager_enabled: false
# Global paarameter to set the exit ip address of services and public ports
port_publisher:
# if you have a service that you want to expose on a specific interfact; set that IP here
# if you set it to auto it gets the public ip from ident.me and sets it
# Defaults to constants.PRIVATE_IP_ADDRESS_PLACEHOLDER
# The default value just means its the IP address of the container in which the service is running
nat_exit_ip: KURTOSIS_IP_ADDR_PLACEHOLDER
# The start value gets used as a seed for TCP and UDP discovery ports for el/cl client
# Defaults to None - no public ports
public_port_start: None
```
#### Example configurations
......
......@@ -115,6 +115,7 @@ def run(plan, args={}):
global_node_selectors,
keymanager_enabled,
parallel_keystore_generation,
args_with_right_defaults.port_publisher,
)
plan.print(
......
......@@ -138,3 +138,6 @@ xatu_sentry_params:
global_tolerations: []
global_node_selectors: {}
keymanager_enabled: false
port_publisher:
public_port_start: null
nat_exit_ip: KURTOSIS_IP_ADDR_PLACEHOLDER
shared_utils = import_module("../shared_utils/shared_utils.star")
input_parser = import_module("../package_io/input_parser.star")
constants = import_module("../package_io/constants.star")
cl_context = import_module("../cl/cl_context.star")
blobber_context = import_module("../blobber/blobber_context.star")
......@@ -10,8 +11,6 @@ BLOBBER_BEACON_PORT_UDP_ID = "discovery-udp"
BLOBBER_VALIDATOR_PROXY_PORT_NUM = 5000
BLOBBER_VALIDATOR_PROXY_PORT_ID = "http"
PRIVATE_IP_ADDRESS_PLACEHOLDER = "KURTOSIS_IP_ADDR_PLACEHOLDER"
DEFAULT_BLOBBER_IMAGE = "ethpandaops/blobber:1.1.0"
VALIDATOR_KEYS_MOUNTPOINT_ON_CLIENTS = "/validator-keys"
......@@ -76,7 +75,8 @@ def get_config(
"--cl={0}".format(beacon_http_url),
"--validator-key-folder={0}".format(validator_root_dirpath),
"--enable-unsafe-mode",
"--external-ip={0}".format(PRIVATE_IP_ADDRESS_PLACEHOLDER),
# Does this get affected by public ip address changes?
"--external-ip={0}".format(constants.PRIVATE_IP_ADDRESS_PLACEHOLDER),
"--validator-proxy-port-start={0}".format(BLOBBER_VALIDATOR_PROXY_PORT_NUM),
]
......@@ -90,7 +90,7 @@ def get_config(
VALIDATOR_KEYS_MOUNTPOINT_ON_CLIENTS: node_keystore_files.files_artifact_uuid
},
cmd=cmd,
private_ip_address_placeholder=PRIVATE_IP_ADDRESS_PLACEHOLDER,
private_ip_address_placeholder=constants.PRIVATE_IP_ADDRESS_PLACEHOLDER,
min_cpu=MIN_CPU,
max_cpu=MAX_CPU,
min_memory=MIN_MEMORY,
......
......@@ -31,6 +31,7 @@ def launch(
validator_data,
prysm_password_relative_filepath,
prysm_password_artifact_uuid,
port_publisher,
):
plan.print("Launching CL network")
......@@ -175,6 +176,7 @@ def launch(
node_selectors,
participant.use_separate_vc,
participant.keymanager_enabled,
port_publisher,
)
else:
boot_cl_client_ctx = all_cl_contexts
......@@ -208,6 +210,7 @@ def launch(
node_selectors,
participant.use_separate_vc,
participant.keymanager_enabled,
port_publisher,
)
# Add participant cl additional prometheus labels
......
......@@ -29,14 +29,14 @@ BEACON_METRICS_PATH = "/metrics"
MIN_PEERS = 1
PRIVATE_IP_ADDRESS_PLACEHOLDER = "KURTOSIS_IP_ADDR_PLACEHOLDER"
BEACON_USED_PORTS = {
def get_used_ports(discovery_port):
beacon_used_ports = {
BEACON_TCP_DISCOVERY_PORT_ID: shared_utils.new_port_spec(
BEACON_DISCOVERY_PORT_NUM, shared_utils.TCP_PROTOCOL
discovery_port, shared_utils.TCP_PROTOCOL
),
BEACON_UDP_DISCOVERY_PORT_ID: shared_utils.new_port_spec(
BEACON_DISCOVERY_PORT_NUM, shared_utils.UDP_PROTOCOL
discovery_port, shared_utils.UDP_PROTOCOL
),
BEACON_HTTP_PORT_ID: shared_utils.new_port_spec(
BEACON_HTTP_PORT_NUM, shared_utils.TCP_PROTOCOL
......@@ -44,7 +44,8 @@ BEACON_USED_PORTS = {
BEACON_METRICS_PORT_ID: shared_utils.new_port_spec(
BEACON_METRICS_PORT_NUM, shared_utils.TCP_PROTOCOL
),
}
}
return beacon_used_ports
ENTRYPOINT_ARGS = ["sh", "-c"]
......@@ -88,6 +89,7 @@ def launch(
node_selectors,
use_separate_vc,
keymanager_enabled,
port_publisher,
):
beacon_service_name = "{0}".format(service_name)
log_level = input_parser.get_client_log_level_or_default(
......@@ -148,6 +150,7 @@ def launch(
cl_volume_size,
tolerations,
node_selectors,
port_publisher,
)
beacon_service = plan.add_service(service_name, config)
......@@ -227,6 +230,7 @@ def get_beacon_config(
cl_volume_size,
tolerations,
node_selectors,
port_publisher,
):
validator_keys_dirpath = ""
validator_secrets_dirpath = ""
......@@ -250,6 +254,23 @@ def get_beacon_config(
el_context.ip_addr,
el_context.engine_rpc_port_num,
)
public_ports = {}
discovery_port = BEACON_DISCOVERY_PORT_NUM
if port_publisher.public_port_start:
discovery_port = port_publisher.cl_start
if bootnode_contexts and len(bootnode_contexts) > 0:
discovery_port = discovery_port + len(bootnode_contexts)
public_ports = {
BEACON_TCP_DISCOVERY_PORT_ID: shared_utils.new_port_spec(
discovery_port, shared_utils.TCP_PROTOCOL
),
BEACON_UDP_DISCOVERY_PORT_ID: shared_utils.new_port_spec(
discovery_port, shared_utils.UDP_PROTOCOL
),
}
used_ports = get_used_ports(discovery_port)
cmd = [
"--network={0}".format(
network if network in constants.PUBLIC_NETWORKS else "custom"
......@@ -257,15 +278,15 @@ def get_beacon_config(
"--data-dir=" + BEACON_DATA_DIRPATH_ON_SERVICE_CONTAINER,
"--http-address=0.0.0.0",
"--http-port={0}".format(BEACON_HTTP_PORT_NUM),
"--libp2p-port={0}".format(BEACON_DISCOVERY_PORT_NUM),
"--discovery-port={0}".format(BEACON_DISCOVERY_PORT_NUM),
"--libp2p-port={0}".format(discovery_port),
"--discovery-port={0}".format(discovery_port),
"--jwt-secret=" + constants.JWT_MOUNT_PATH_ON_CONTAINER,
"--eth1-rpc-urls=" + EXECUTION_ENGINE_ENDPOINT,
# vvvvvvvvvvvvvvvvvvv REMOVE THESE WHEN CONNECTING TO EXTERNAL NET vvvvvvvvvvvvvvvvvvvvv
"--disable-enr-auto-update",
"--enr-address=" + PRIVATE_IP_ADDRESS_PLACEHOLDER,
"--enr-udp-port={0}".format(BEACON_DISCOVERY_PORT_NUM),
"--enr-tcp-port={0}".format(BEACON_DISCOVERY_PORT_NUM),
"--enr-address=" + port_publisher.nat_exit_ip,
"--enr-udp-port={0}".format(discovery_port),
"--enr-tcp-port={0}".format(discovery_port),
# ^^^^^^^^^^^^^^^^^^^ REMOVE THESE WHEN CONNECTING TO EXTERNAL NET ^^^^^^^^^^^^^^^^^^^^^
# vvvvvvvvvvvvvvvvvvv METRICS CONFIG vvvvvvvvvvvvvvvvvvvvv
"--metrics",
......@@ -353,7 +374,7 @@ def get_beacon_config(
}
ports = {}
ports.update(BEACON_USED_PORTS)
ports.update(used_ports)
if node_keystore_files != None and not use_separate_vc:
cmd.extend(validator_default_cmd)
files[
......@@ -373,10 +394,11 @@ def get_beacon_config(
return ServiceConfig(
image=image,
ports=ports,
public_ports=public_ports,
cmd=cmd,
env_vars=extra_env_vars,
files=files,
private_ip_address_placeholder=PRIVATE_IP_ADDRESS_PLACEHOLDER,
private_ip_address_placeholder=constants.PRIVATE_IP_ADDRESS_PLACEHOLDER,
ready_conditions=cl_node_ready_conditions.get_ready_conditions(
BEACON_HTTP_PORT_ID
),
......
......@@ -32,14 +32,14 @@ BEACON_MIN_MEMORY = 256
METRICS_PATH = "/metrics"
PRIVATE_IP_ADDRESS_PLACEHOLDER = "KURTOSIS_IP_ADDR_PLACEHOLDER"
BEACON_USED_PORTS = {
def get_used_ports(discovery_port):
beacon_used_ports = {
BEACON_TCP_DISCOVERY_PORT_ID: shared_utils.new_port_spec(
BEACON_DISCOVERY_PORT_NUM, shared_utils.TCP_PROTOCOL
discovery_port, shared_utils.TCP_PROTOCOL
),
BEACON_UDP_DISCOVERY_PORT_ID: shared_utils.new_port_spec(
BEACON_DISCOVERY_PORT_NUM, shared_utils.UDP_PROTOCOL
discovery_port, shared_utils.UDP_PROTOCOL
),
BEACON_HTTP_PORT_ID: shared_utils.new_port_spec(
BEACON_HTTP_PORT_NUM,
......@@ -51,7 +51,9 @@ BEACON_USED_PORTS = {
shared_utils.TCP_PROTOCOL,
shared_utils.HTTP_APPLICATION_PROTOCOL,
),
}
}
return beacon_used_ports
VERBOSITY_LEVELS = {
constants.GLOBAL_LOG_LEVEL.error: "error",
......@@ -90,8 +92,9 @@ def launch(
participant_tolerations,
global_tolerations,
node_selectors,
use_separate_vc=True,
keymanager_enabled=False,
use_separate_vc,
keymanager_enabled,
port_publisher,
):
beacon_service_name = "{0}".format(service_name)
......@@ -148,6 +151,7 @@ def launch(
cl_volume_size,
tolerations,
node_selectors,
port_publisher,
)
beacon_service = plan.add_service(beacon_service_name, beacon_config)
......@@ -242,6 +246,7 @@ def get_beacon_config(
cl_volume_size,
tolerations,
node_selectors,
port_publisher,
):
# If snooper is enabled use the snooper engine context, otherwise use the execution client context
if snooper_enabled:
......@@ -255,6 +260,22 @@ def get_beacon_config(
el_context.engine_rpc_port_num,
)
public_ports = {}
discovery_port = BEACON_DISCOVERY_PORT_NUM
if port_publisher.public_port_start:
discovery_port = port_publisher.cl_start
if boot_cl_client_ctxs and len(boot_cl_client_ctxs) > 0:
discovery_port = discovery_port + len(boot_cl_client_ctxs)
public_ports = {
BEACON_TCP_DISCOVERY_PORT_ID: shared_utils.new_port_spec(
discovery_port, shared_utils.TCP_PROTOCOL
),
BEACON_UDP_DISCOVERY_PORT_ID: shared_utils.new_port_spec(
discovery_port, shared_utils.UDP_PROTOCOL
),
}
used_ports = get_used_ports(discovery_port)
# NOTE: If connecting to the merge devnet remotely we DON'T want the following flags; when they're not set, the node's external IP address is auto-detected
# from the peers it communicates with but when they're set they basically say "override the autodetection and
# use what I specify instead." This requires having a know external IP address and port, which we definitely won't
......@@ -270,13 +291,13 @@ def get_beacon_config(
"--datadir=" + BEACON_DATA_DIRPATH_ON_BEACON_SERVICE_CONTAINER,
# vvvvvvvvvvvvvvvvvvv REMOVE THESE WHEN CONNECTING TO EXTERNAL NET vvvvvvvvvvvvvvvvvvvvv
"--disable-enr-auto-update",
"--enr-address=" + PRIVATE_IP_ADDRESS_PLACEHOLDER,
"--enr-udp-port={0}".format(BEACON_DISCOVERY_PORT_NUM),
"--enr-tcp-port={0}".format(BEACON_DISCOVERY_PORT_NUM),
"--enr-address=" + port_publisher.nat_exit_ip,
"--enr-udp-port={0}".format(discovery_port),
"--enr-tcp-port={0}".format(discovery_port),
# ^^^^^^^^^^^^^^^^^^^ REMOVE THESE WHEN CONNECTING TO EXTERNAL NET ^^^^^^^^^^^^^^^^^^^^^
"--listen-address=0.0.0.0",
"--port={0}".format(
BEACON_DISCOVERY_PORT_NUM
discovery_port
), # NOTE: Remove for connecting to external net!
"--http",
"--http-address=0.0.0.0",
......@@ -368,11 +389,12 @@ def get_beacon_config(
env.update(extra_env_vars)
return ServiceConfig(
image=image,
ports=BEACON_USED_PORTS,
ports=used_ports,
public_ports=public_ports,
cmd=cmd,
files=files,
env_vars=env,
private_ip_address_placeholder=PRIVATE_IP_ADDRESS_PLACEHOLDER,
private_ip_address_placeholder=constants.PRIVATE_IP_ADDRESS_PLACEHOLDER,
ready_conditions=cl_node_ready_conditions.get_ready_conditions(
BEACON_HTTP_PORT_ID
),
......
......@@ -25,14 +25,14 @@ BEACON_MIN_MEMORY = 256
METRICS_PATH = "/metrics"
PRIVATE_IP_ADDRESS_PLACEHOLDER = "KURTOSIS_IP_ADDR_PLACEHOLDER"
BEACON_USED_PORTS = {
def get_used_ports(discovery_port):
beacon_used_ports = {
TCP_DISCOVERY_PORT_ID: shared_utils.new_port_spec(
DISCOVERY_PORT_NUM, shared_utils.TCP_PROTOCOL
discovery_port, shared_utils.TCP_PROTOCOL
),
UDP_DISCOVERY_PORT_ID: shared_utils.new_port_spec(
DISCOVERY_PORT_NUM, shared_utils.UDP_PROTOCOL
discovery_port, shared_utils.UDP_PROTOCOL
),
BEACON_HTTP_PORT_ID: shared_utils.new_port_spec(
HTTP_PORT_NUM, shared_utils.TCP_PROTOCOL
......@@ -40,7 +40,9 @@ BEACON_USED_PORTS = {
METRICS_PORT_ID: shared_utils.new_port_spec(
METRICS_PORT_NUM, shared_utils.TCP_PROTOCOL
),
}
}
return beacon_used_ports
VERBOSITY_LEVELS = {
constants.GLOBAL_LOG_LEVEL.error: "error",
......@@ -79,8 +81,9 @@ def launch(
participant_tolerations,
global_tolerations,
node_selectors,
use_separate_vc=True,
keymanager_enabled=False,
use_separate_vc,
keymanager_enabled,
port_publisher,
):
beacon_service_name = "{0}".format(service_name)
log_level = input_parser.get_client_log_level_or_default(
......@@ -136,6 +139,7 @@ def launch(
cl_volume_size,
tolerations,
node_selectors,
port_publisher,
)
beacon_service = plan.add_service(beacon_service_name, beacon_config)
......@@ -234,6 +238,7 @@ def get_beacon_config(
cl_volume_size,
tolerations,
node_selectors,
port_publisher,
):
el_client_rpc_url_str = "http://{0}:{1}".format(
el_context.ip_addr,
......@@ -252,11 +257,27 @@ def get_beacon_config(
el_context.engine_rpc_port_num,
)
public_ports = {}
discovery_port = DISCOVERY_PORT_NUM
if port_publisher.public_port_start:
discovery_port = port_publisher.cl_start
if bootnode_contexts and len(bootnode_contexts) > 0:
discovery_port = discovery_port + len(bootnode_contexts)
public_ports = {
TCP_DISCOVERY_PORT_ID: shared_utils.new_port_spec(
discovery_port, shared_utils.TCP_PROTOCOL
),
UDP_DISCOVERY_PORT_ID: shared_utils.new_port_spec(
discovery_port, shared_utils.UDP_PROTOCOL
),
}
used_ports = get_used_ports(discovery_port)
cmd = [
"beacon",
"--logLevel=" + log_level,
"--port={0}".format(DISCOVERY_PORT_NUM),
"--discoveryPort={0}".format(DISCOVERY_PORT_NUM),
"--port={0}".format(discovery_port),
"--discoveryPort={0}".format(discovery_port),
"--dataDir=" + BEACON_DATA_DIRPATH_ON_SERVICE_CONTAINER,
"--eth1.depositContractDeployBlock=0",
"--network.connectToDiscv5Bootnodes=true",
......@@ -269,9 +290,9 @@ def get_beacon_config(
"--rest.namespace=*",
"--rest.port={0}".format(HTTP_PORT_NUM),
"--nat=true",
"--enr.ip=" + PRIVATE_IP_ADDRESS_PLACEHOLDER,
"--enr.tcp={0}".format(DISCOVERY_PORT_NUM),
"--enr.udp={0}".format(DISCOVERY_PORT_NUM),
"--enr.ip=" + port_publisher.nat_exit_ip,
"--enr.tcp={0}".format(discovery_port),
"--enr.udp={0}".format(discovery_port),
# Set per Pari's recommendation to reduce noise in the logs
"--subscribeAllSubnets=true",
"--jwt-secret=" + constants.JWT_MOUNT_PATH_ON_CONTAINER,
......@@ -348,11 +369,12 @@ def get_beacon_config(
)
return ServiceConfig(
image=image,
ports=BEACON_USED_PORTS,
ports=used_ports,
public_ports=public_ports,
cmd=cmd,
env_vars=extra_env_vars,
files=files,
private_ip_address_placeholder=PRIVATE_IP_ADDRESS_PLACEHOLDER,
private_ip_address_placeholder=constants.PRIVATE_IP_ADDRESS_PLACEHOLDER,
ready_conditions=cl_node_ready_conditions.get_ready_conditions(
BEACON_HTTP_PORT_ID
),
......
......@@ -42,13 +42,13 @@ VALIDATOR_KEYS_MOUNTPOINT_ON_CLIENTS = "/data/nimbus/validator-keys"
# ---------------------------------- Used Ports ----------------------------------
PRIVATE_IP_ADDRESS_PLACEHOLDER = "KURTOSIS_IP_ADDR_PLACEHOLDER"
BEACON_USED_PORTS = {
def get_used_ports(discovery_port):
used_ports = {
BEACON_TCP_DISCOVERY_PORT_ID: shared_utils.new_port_spec(
BEACON_DISCOVERY_PORT_NUM, shared_utils.TCP_PROTOCOL
discovery_port, shared_utils.TCP_PROTOCOL
),
BEACON_UDP_DISCOVERY_PORT_ID: shared_utils.new_port_spec(
BEACON_DISCOVERY_PORT_NUM, shared_utils.UDP_PROTOCOL
discovery_port, shared_utils.UDP_PROTOCOL
),
BEACON_HTTP_PORT_ID: shared_utils.new_port_spec(
BEACON_HTTP_PORT_NUM,
......@@ -60,7 +60,9 @@ BEACON_USED_PORTS = {
shared_utils.TCP_PROTOCOL,
shared_utils.HTTP_APPLICATION_PROTOCOL,
),
}
}
return used_ports
VERBOSITY_LEVELS = {
constants.GLOBAL_LOG_LEVEL.error: "ERROR",
......@@ -103,6 +105,7 @@ def launch(
node_selectors,
use_separate_vc,
keymanager_enabled,
port_publisher,
):
beacon_service_name = "{0}".format(service_name)
......@@ -163,6 +166,7 @@ def launch(
cl_volume_size,
tolerations,
node_selectors,
port_publisher,
)
beacon_service = plan.add_service(beacon_service_name, beacon_config)
......@@ -242,6 +246,7 @@ def get_beacon_config(
cl_volume_size,
tolerations,
node_selectors,
port_publisher,
):
validator_keys_dirpath = ""
validator_secrets_dirpath = ""
......@@ -266,11 +271,27 @@ def get_beacon_config(
el_context.engine_rpc_port_num,
)
public_ports = {}
discovery_port = BEACON_DISCOVERY_PORT_NUM
if port_publisher.public_port_start:
discovery_port = port_publisher.cl_start
if bootnode_contexts and len(bootnode_contexts) > 0:
discovery_port = discovery_port + len(bootnode_contexts)
public_ports = {
BEACON_TCP_DISCOVERY_PORT_ID: shared_utils.new_port_spec(
discovery_port, shared_utils.TCP_PROTOCOL
),
BEACON_UDP_DISCOVERY_PORT_ID: shared_utils.new_port_spec(
discovery_port, shared_utils.UDP_PROTOCOL
),
}
used_ports = get_used_ports(discovery_port)
cmd = [
"--non-interactive=true",
"--log-level=" + log_level,
"--udp-port={0}".format(BEACON_DISCOVERY_PORT_NUM),
"--tcp-port={0}".format(BEACON_DISCOVERY_PORT_NUM),
"--udp-port={0}".format(discovery_port),
"--tcp-port={0}".format(discovery_port),
"--network={0}".format(
network
if network in constants.PUBLIC_NETWORKS
......@@ -278,7 +299,7 @@ def get_beacon_config(
),
"--data-dir=" + BEACON_DATA_DIRPATH_ON_SERVICE_CONTAINER,
"--web3-url=" + EXECUTION_ENGINE_ENDPOINT,
"--nat=extip:" + PRIVATE_IP_ADDRESS_PLACEHOLDER,
"--nat=extip:" + port_publisher.nat_exit_ip,
"--enr-auto-update=false",
"--history={0}".format("archive" if constants.ARCHIVE_MODE else "prune"),
"--rest",
......@@ -340,7 +361,7 @@ def get_beacon_config(
constants.JWT_MOUNTPOINT_ON_CLIENTS: jwt_file,
}
ports = {}
ports.update(BEACON_USED_PORTS)
ports.update(used_ports)
if node_keystore_files != None and not use_separate_vc:
cmd.extend(validator_default_cmd)
files[
......@@ -360,11 +381,12 @@ def get_beacon_config(
return ServiceConfig(
image=image,
ports=ports,
ports=used_ports,
public_ports=public_ports,
cmd=cmd,
env_vars=extra_env_vars,
files=files,
private_ip_address_placeholder=PRIVATE_IP_ADDRESS_PLACEHOLDER,
private_ip_address_placeholder=constants.PRIVATE_IP_ADDRESS_PLACEHOLDER,
ready_conditions=cl_node_ready_conditions.get_ready_conditions(
BEACON_HTTP_PORT_ID
),
......
......@@ -31,23 +31,27 @@ METRICS_PATH = "/metrics"
MIN_PEERS = 1
PRIVATE_IP_ADDRESS_PLACEHOLDER = "KURTOSIS_IP_ADDR_PLACEHOLDER"
BEACON_NODE_USED_PORTS = {
def get_used_ports(discovery_port):
used_ports = {
TCP_DISCOVERY_PORT_ID: shared_utils.new_port_spec(
DISCOVERY_TCP_PORT_NUM, shared_utils.TCP_PROTOCOL
discovery_port, shared_utils.TCP_PROTOCOL
),
UDP_DISCOVERY_PORT_ID: shared_utils.new_port_spec(
DISCOVERY_UDP_PORT_NUM, shared_utils.UDP_PROTOCOL
discovery_port, shared_utils.UDP_PROTOCOL
),
RPC_PORT_ID: shared_utils.new_port_spec(
RPC_PORT_NUM, shared_utils.TCP_PROTOCOL
),
RPC_PORT_ID: shared_utils.new_port_spec(RPC_PORT_NUM, shared_utils.TCP_PROTOCOL),
BEACON_HTTP_PORT_ID: shared_utils.new_port_spec(
HTTP_PORT_NUM, shared_utils.TCP_PROTOCOL
),
BEACON_MONITORING_PORT_ID: shared_utils.new_port_spec(
BEACON_MONITORING_PORT_NUM, shared_utils.TCP_PROTOCOL
),
}
}
return used_ports
VERBOSITY_LEVELS = {
constants.GLOBAL_LOG_LEVEL.error: "error",
......@@ -86,8 +90,9 @@ def launch(
participant_tolerations,
global_tolerations,
node_selectors,
use_separate_vc=True,
keymanager_enabled=False,
use_separate_vc,
keymanager_enabled,
port_publisher,
):
beacon_service_name = "{0}".format(service_name)
log_level = input_parser.get_client_log_level_or_default(
......@@ -142,6 +147,7 @@ def launch(
cl_volume_size,
tolerations,
node_selectors,
port_publisher,
)
beacon_service = plan.add_service(beacon_service_name, beacon_config)
......@@ -219,6 +225,7 @@ def get_beacon_config(
cl_volume_size,
tolerations,
node_selectors,
port_publisher,
):
# If snooper is enabled use the snooper engine context, otherwise use the execution client context
if snooper_enabled:
......@@ -232,6 +239,22 @@ def get_beacon_config(
el_context.engine_rpc_port_num,
)
public_ports = {}
discovery_port = DISCOVERY_TCP_PORT_NUM
if port_publisher.public_port_start:
discovery_port = port_publisher.cl_start
if bootnode_contexts and len(bootnode_contexts) > 0:
discovery_port = discovery_port + len(bootnode_contexts)
public_ports = {
TCP_DISCOVERY_PORT_ID: shared_utils.new_port_spec(
discovery_port, shared_utils.TCP_PROTOCOL
),
UDP_DISCOVERY_PORT_ID: shared_utils.new_port_spec(
discovery_port, shared_utils.UDP_PROTOCOL
),
}
used_ports = get_used_ports(discovery_port)
cmd = [
"--accept-terms-of-use=true", # it's mandatory in order to run the node
"--datadir=" + BEACON_DATA_DIRPATH_ON_SERVICE_CONTAINER,
......@@ -241,9 +264,9 @@ def get_beacon_config(
"--grpc-gateway-host=0.0.0.0",
"--grpc-gateway-corsdomain=*",
"--grpc-gateway-port={0}".format(HTTP_PORT_NUM),
"--p2p-host-ip=" + PRIVATE_IP_ADDRESS_PLACEHOLDER,
"--p2p-tcp-port={0}".format(DISCOVERY_TCP_PORT_NUM),
"--p2p-udp-port={0}".format(DISCOVERY_UDP_PORT_NUM),
"--p2p-host-ip=" + port_publisher.nat_exit_ip,
"--p2p-tcp-port={0}".format(discovery_port),
"--p2p-udp-port={0}".format(discovery_port),
"--min-sync-peers={0}".format(MIN_PEERS),
"--verbosity=" + log_level,
"--slots-per-archive-point={0}".format(32 if constants.ARCHIVE_MODE else 8192),
......@@ -331,11 +354,12 @@ def get_beacon_config(
return ServiceConfig(
image=beacon_image,
ports=BEACON_NODE_USED_PORTS,
ports=used_ports,
public_ports=public_ports,
cmd=cmd,
env_vars=extra_env_vars,
files=files,
private_ip_address_placeholder=PRIVATE_IP_ADDRESS_PLACEHOLDER,
private_ip_address_placeholder=constants.PRIVATE_IP_ADDRESS_PLACEHOLDER,
ready_conditions=cl_node_ready_conditions.get_ready_conditions(
BEACON_HTTP_PORT_ID
),
......
......@@ -31,14 +31,14 @@ BEACON_METRICS_PATH = "/metrics"
MIN_PEERS = 1
PRIVATE_IP_ADDRESS_PLACEHOLDER = "KURTOSIS_IP_ADDR_PLACEHOLDER"
BEACON_USED_PORTS = {
def get_used_ports(discovery_port):
used_ports = {
BEACON_TCP_DISCOVERY_PORT_ID: shared_utils.new_port_spec(
BEACON_DISCOVERY_PORT_NUM, shared_utils.TCP_PROTOCOL
discovery_port, shared_utils.TCP_PROTOCOL
),
BEACON_UDP_DISCOVERY_PORT_ID: shared_utils.new_port_spec(
BEACON_DISCOVERY_PORT_NUM, shared_utils.UDP_PROTOCOL
discovery_port, shared_utils.UDP_PROTOCOL
),
BEACON_HTTP_PORT_ID: shared_utils.new_port_spec(
BEACON_HTTP_PORT_NUM, shared_utils.TCP_PROTOCOL
......@@ -46,7 +46,8 @@ BEACON_USED_PORTS = {
BEACON_METRICS_PORT_ID: shared_utils.new_port_spec(
BEACON_METRICS_PORT_NUM, shared_utils.TCP_PROTOCOL
),
}
}
return used_ports
ENTRYPOINT_ARGS = ["sh", "-c"]
......@@ -90,6 +91,7 @@ def launch(
node_selectors,
use_separate_vc,
keymanager_enabled,
port_publisher,
):
beacon_service_name = "{0}".format(service_name)
log_level = input_parser.get_client_log_level_or_default(
......@@ -151,6 +153,7 @@ def launch(
cl_volume_size,
tolerations,
node_selectors,
port_publisher,
)
beacon_service = plan.add_service(service_name, config)
......@@ -232,6 +235,7 @@ def get_beacon_config(
cl_volume_size,
tolerations,
node_selectors,
port_publisher,
):
validator_keys_dirpath = ""
validator_secrets_dirpath = ""
......@@ -255,6 +259,23 @@ def get_beacon_config(
el_context.ip_addr,
el_context.engine_rpc_port_num,
)
public_ports = {}
discovery_port = BEACON_DISCOVERY_PORT_NUM
if port_publisher.public_port_start:
discovery_port = port_publisher.cl_start
if bootnode_contexts and len(bootnode_contexts) > 0:
discovery_port = discovery_port + len(bootnode_contexts)
public_ports = {
BEACON_TCP_DISCOVERY_PORT_ID: shared_utils.new_port_spec(
discovery_port, shared_utils.TCP_PROTOCOL
),
BEACON_UDP_DISCOVERY_PORT_ID: shared_utils.new_port_spec(
discovery_port, shared_utils.UDP_PROTOCOL
),
}
used_ports = get_used_ports(discovery_port)
cmd = [
"--logging=" + log_level,
"--log-destination=CONSOLE",
......@@ -271,8 +292,9 @@ def get_beacon_config(
# Set per Pari's recommendation, to reduce noise in the logs
"--p2p-subscribe-all-subnets-enabled=true",
"--p2p-peer-lower-bound={0}".format(MIN_PEERS),
"--p2p-advertised-ip=" + PRIVATE_IP_ADDRESS_PLACEHOLDER,
"--p2p-advertised-ip=" + port_publisher.nat_exit_ip,
"--p2p-discovery-site-local-addresses-enabled=true",
"--p2p-port={0}".format(discovery_port),
"--rest-api-enabled=true",
"--rest-api-docs-enabled=true",
"--rest-api-interface=0.0.0.0",
......@@ -374,7 +396,7 @@ def get_beacon_config(
constants.JWT_MOUNTPOINT_ON_CLIENTS: jwt_file,
}
ports = {}
ports.update(BEACON_USED_PORTS)
ports.update(used_ports)
if node_keystore_files != None and not use_separate_vc:
cmd.extend(validator_default_cmd)
files[
......@@ -395,10 +417,11 @@ def get_beacon_config(
return ServiceConfig(
image=image,
ports=ports,
public_ports=public_ports,
cmd=cmd,
env_vars=extra_env_vars,
files=files,
private_ip_address_placeholder=PRIVATE_IP_ADDRESS_PLACEHOLDER,
private_ip_address_placeholder=constants.PRIVATE_IP_ADDRESS_PLACEHOLDER,
ready_conditions=cl_node_ready_conditions.get_ready_conditions(
BEACON_HTTP_PORT_ID
),
......
......@@ -29,18 +29,21 @@ UDP_DISCOVERY_PORT_ID = "udp-discovery"
ENGINE_HTTP_RPC_PORT_ID = "engine-rpc"
METRICS_PORT_ID = "metrics"
JAVA_OPTS = {"JAVA_OPTS": "-agentlib:jdwp=transport=dt_socket,server=y,suspend=n"}
PRIVATE_IP_ADDRESS_PLACEHOLDER = "KURTOSIS_IP_ADDR_PLACEHOLDER"
USED_PORTS = {
def get_used_ports(discovery_port=DISCOVERY_PORT_NUM):
used_ports = {
RPC_PORT_ID: shared_utils.new_port_spec(
RPC_PORT_NUM, shared_utils.TCP_PROTOCOL, shared_utils.HTTP_APPLICATION_PROTOCOL
RPC_PORT_NUM,
shared_utils.TCP_PROTOCOL,
shared_utils.HTTP_APPLICATION_PROTOCOL,
),
WS_PORT_ID: shared_utils.new_port_spec(WS_PORT_NUM, shared_utils.TCP_PROTOCOL),
TCP_DISCOVERY_PORT_ID: shared_utils.new_port_spec(
DISCOVERY_PORT_NUM, shared_utils.TCP_PROTOCOL
discovery_port, shared_utils.TCP_PROTOCOL
),
UDP_DISCOVERY_PORT_ID: shared_utils.new_port_spec(
DISCOVERY_PORT_NUM, shared_utils.UDP_PROTOCOL
discovery_port, shared_utils.UDP_PROTOCOL
),
ENGINE_HTTP_RPC_PORT_ID: shared_utils.new_port_spec(
ENGINE_HTTP_RPC_PORT_NUM, shared_utils.TCP_PROTOCOL
......@@ -48,7 +51,9 @@ USED_PORTS = {
METRICS_PORT_ID: shared_utils.new_port_spec(
METRICS_PORT_NUM, shared_utils.TCP_PROTOCOL
),
}
}
return used_ports
ENTRYPOINT_ARGS = ["sh", "-c"]
......@@ -80,6 +85,7 @@ def launch(
el_volume_size,
tolerations,
node_selectors,
port_publisher,
):
log_level = input_parser.get_client_log_level_or_default(
participant_log_level, global_log_level, VERBOSITY_LEVELS
......@@ -129,6 +135,7 @@ def launch(
el_volume_size,
tolerations,
node_selectors,
port_publisher,
)
service = plan.add_service(service_name, config)
......@@ -174,7 +181,22 @@ def get_config(
el_volume_size,
tolerations,
node_selectors,
port_publisher,
):
public_ports = {}
discovery_port = DISCOVERY_PORT_NUM
if port_publisher.public_port_start:
discovery_port = port_publisher.el_start + len(existing_el_clients)
public_ports = {
TCP_DISCOVERY_PORT_ID: shared_utils.new_port_spec(
discovery_port, shared_utils.TCP_PROTOCOL
),
UDP_DISCOVERY_PORT_ID: shared_utils.new_port_spec(
discovery_port, shared_utils.UDP_PROTOCOL
),
}
used_ports = get_used_ports(discovery_port)
cmd = [
"besu",
"--logging=" + log_level,
......@@ -190,8 +212,8 @@ def get_config(
"--rpc-ws-port={0}".format(WS_PORT_NUM),
"--rpc-ws-api=ADMIN,CLIQUE,ETH,NET,DEBUG,TXPOOL,ENGINE,TRACE,WEB3",
"--p2p-enabled=true",
"--p2p-host=" + PRIVATE_IP_ADDRESS_PLACEHOLDER,
"--p2p-port={0}".format(DISCOVERY_PORT_NUM),
"--p2p-host=" + port_publisher.nat_exit_ip,
"--p2p-port={0}".format(discovery_port),
"--engine-rpc-enabled=true",
"--engine-jwt-secret=" + constants.JWT_MOUNT_PATH_ON_CONTAINER,
"--engine-host-allowlist=*",
......@@ -255,12 +277,13 @@ def get_config(
)
return ServiceConfig(
image=image,
ports=USED_PORTS,
ports=used_ports,
public_ports=public_ports,
cmd=[cmd_str],
files=files,
env_vars=extra_env_vars,
entrypoint=ENTRYPOINT_ARGS,
private_ip_address_placeholder=PRIVATE_IP_ADDRESS_PLACEHOLDER,
private_ip_address_placeholder=constants.PRIVATE_IP_ADDRESS_PLACEHOLDER,
min_cpu=el_min_cpu,
max_cpu=el_max_cpu,
min_memory=el_min_mem,
......
......@@ -23,6 +23,7 @@ def launch(
persistent,
network_id,
num_participants,
port_publisher,
):
el_launchers = {
constants.EL_TYPE.geth: {
......@@ -147,6 +148,7 @@ def launch(
participant.el_volume_size,
tolerations,
node_selectors,
port_publisher,
)
# Add participant el additional prometheus metrics
for metrics_info in el_context.el_metrics_info:
......
......@@ -28,19 +28,18 @@ ENGINE_RPC_PORT_ID = "engine-rpc"
METRICS_PORT_ID = "metrics"
PRIVATE_IP_ADDRESS_PLACEHOLDER = "KURTOSIS_IP_ADDR_PLACEHOLDER"
USED_PORTS = {
def get_used_ports(discovery_port=DISCOVERY_PORT_NUM):
used_ports = {
WS_RPC_PORT_ID: shared_utils.new_port_spec(
WS_RPC_PORT_NUM,
shared_utils.TCP_PROTOCOL,
shared_utils.HTTP_APPLICATION_PROTOCOL,
),
TCP_DISCOVERY_PORT_ID: shared_utils.new_port_spec(
DISCOVERY_PORT_NUM, shared_utils.TCP_PROTOCOL
discovery_port, shared_utils.TCP_PROTOCOL
),
UDP_DISCOVERY_PORT_ID: shared_utils.new_port_spec(
DISCOVERY_PORT_NUM, shared_utils.UDP_PROTOCOL
discovery_port, shared_utils.UDP_PROTOCOL
),
ENGINE_RPC_PORT_ID: shared_utils.new_port_spec(
ENGINE_RPC_PORT_NUM, shared_utils.TCP_PROTOCOL
......@@ -48,7 +47,9 @@ USED_PORTS = {
METRICS_PORT_ID: shared_utils.new_port_spec(
METRICS_PORT_NUM, shared_utils.TCP_PROTOCOL
),
}
}
return used_ports
ENTRYPOINT_ARGS = ["sh", "-c"]
......@@ -80,6 +81,7 @@ def launch(
el_volume_size,
tolerations,
node_selectors,
port_publisher,
):
log_level = input_parser.get_client_log_level_or_default(
participant_log_level, global_log_level, VERBOSITY_LEVELS
......@@ -131,6 +133,7 @@ def launch(
el_volume_size,
tolerations,
node_selectors,
port_publisher,
)
service = plan.add_service(service_name, config)
......@@ -180,12 +183,27 @@ def get_config(
el_volume_size,
tolerations,
node_selectors,
port_publisher,
):
init_datadir_cmd_str = "erigon init --datadir={0} {1}".format(
EXECUTION_DATA_DIRPATH_ON_CLIENT_CONTAINER,
constants.GENESIS_CONFIG_MOUNT_PATH_ON_CONTAINER + "/genesis.json",
)
public_ports = {}
discovery_port = DISCOVERY_PORT_NUM
if port_publisher.public_port_start:
discovery_port = port_publisher.el_start + len(existing_el_clients)
public_ports = {
TCP_DISCOVERY_PORT_ID: shared_utils.new_port_spec(
discovery_port, shared_utils.TCP_PROTOCOL
),
UDP_DISCOVERY_PORT_ID: shared_utils.new_port_spec(
discovery_port, shared_utils.UDP_PROTOCOL
),
}
used_ports = get_used_ports(discovery_port)
cmd = [
"erigon",
"{0}".format(
......@@ -196,12 +214,12 @@ def get_config(
"--networkid={0}".format(networkid),
"--log.console.verbosity=" + verbosity_level,
"--datadir=" + EXECUTION_DATA_DIRPATH_ON_CLIENT_CONTAINER,
"--port={0}".format(DISCOVERY_PORT_NUM),
"--port={0}".format(discovery_port),
"--http.api=eth,erigon,engine,web3,net,debug,trace,txpool,admin",
"--http.vhosts=*",
"--ws",
"--allow-insecure-unlock",
"--nat=extip:" + PRIVATE_IP_ADDRESS_PLACEHOLDER,
"--nat=extip:" + port_publisher.nat_exit_ip,
"--http",
"--http.addr=0.0.0.0",
"--http.corsdomain=*",
......@@ -257,11 +275,12 @@ def get_config(
return ServiceConfig(
image=image,
ports=USED_PORTS,
ports=used_ports,
public_ports=public_ports,
cmd=[command_arg_str],
files=files,
entrypoint=ENTRYPOINT_ARGS,
private_ip_address_placeholder=PRIVATE_IP_ADDRESS_PLACEHOLDER,
private_ip_address_placeholder=constants.PRIVATE_IP_ADDRESS_PLACEHOLDER,
min_cpu=el_min_cpu,
max_cpu=el_max_cpu,
min_memory=el_min_mem,
......
......@@ -30,27 +30,31 @@ METRICS_PATH = "/metrics"
# The dirpath of the execution data directory on the client container
EXECUTION_DATA_DIRPATH_ON_CLIENT_CONTAINER = "/data/ethereumjs/execution-data"
PRIVATE_IP_ADDRESS_PLACEHOLDER = "KURTOSIS_IP_ADDR_PLACEHOLDER"
USED_PORTS = {
def get_used_ports(discovery_port=DISCOVERY_PORT_NUM):
used_ports = {
RPC_PORT_ID: shared_utils.new_port_spec(
RPC_PORT_NUM, shared_utils.TCP_PROTOCOL, shared_utils.HTTP_APPLICATION_PROTOCOL
RPC_PORT_NUM,
shared_utils.TCP_PROTOCOL,
shared_utils.HTTP_APPLICATION_PROTOCOL,
),
WS_PORT_ID: shared_utils.new_port_spec(WS_PORT_NUM, shared_utils.TCP_PROTOCOL),
WS_PORT_ENGINE_ID: shared_utils.new_port_spec(
WS_PORT_ENGINE_NUM, shared_utils.TCP_PROTOCOL
),
TCP_DISCOVERY_PORT_ID: shared_utils.new_port_spec(
DISCOVERY_PORT_NUM, shared_utils.TCP_PROTOCOL
discovery_port, shared_utils.TCP_PROTOCOL
),
UDP_DISCOVERY_PORT_ID: shared_utils.new_port_spec(
DISCOVERY_PORT_NUM, shared_utils.UDP_PROTOCOL
discovery_port, shared_utils.UDP_PROTOCOL
),
ENGINE_RPC_PORT_ID: shared_utils.new_port_spec(
ENGINE_RPC_PORT_NUM, shared_utils.TCP_PROTOCOL
),
# METRICS_PORT_ID: shared_utils.new_port_spec(METRICS_PORT_NUM, shared_utils.TCP_PROTOCOL)
}
}
return used_ports
ENTRYPOINT_ARGS = []
......@@ -82,6 +86,7 @@ def launch(
el_volume_size,
tolerations,
node_selectors,
port_publisher,
):
log_level = input_parser.get_client_log_level_or_default(
participant_log_level, global_log_level, VERBOSITY_LEVELS
......@@ -131,6 +136,7 @@ def launch(
el_volume_size,
tolerations,
node_selectors,
port_publisher,
)
service = plan.add_service(service_name, config)
......@@ -175,10 +181,25 @@ def get_config(
el_volume_size,
tolerations,
node_selectors,
port_publisher,
):
public_ports = {}
discovery_port = DISCOVERY_PORT_NUM
if port_publisher.public_port_start:
discovery_port = port_publisher.el_start + len(existing_el_clients)
public_ports = {
TCP_DISCOVERY_PORT_ID: shared_utils.new_port_spec(
discovery_port, shared_utils.TCP_PROTOCOL
),
UDP_DISCOVERY_PORT_ID: shared_utils.new_port_spec(
discovery_port, shared_utils.UDP_PROTOCOL
),
}
used_ports = get_used_ports(discovery_port)
cmd = [
"--dataDir=" + EXECUTION_DATA_DIRPATH_ON_CLIENT_CONTAINER,
"--port={0}".format(DISCOVERY_PORT_NUM),
"--port={0}".format(discovery_port),
"--rpc",
"--rpcAddr=0.0.0.0",
"--rpcPort={0}".format(RPC_PORT_NUM),
......@@ -192,7 +213,7 @@ def get_config(
"--wsEnginePort={0}".format(WS_PORT_ENGINE_NUM),
"--wsEngineAddr=0.0.0.0",
"--jwt-secret=" + constants.JWT_MOUNT_PATH_ON_CONTAINER,
"--extIP={0}".format(PRIVATE_IP_ADDRESS_PLACEHOLDER),
"--extIP={0}".format(port_publisher.nat_exit_ip),
"--sync=full",
"--isSingleNode=true",
"--logLevel={0}".format(verbosity_level),
......@@ -242,11 +263,12 @@ def get_config(
)
return ServiceConfig(
image=image,
ports=USED_PORTS,
ports=used_ports,
public_ports=public_ports,
cmd=cmd,
files=files,
entrypoint=ENTRYPOINT_ARGS,
private_ip_address_placeholder=PRIVATE_IP_ADDRESS_PLACEHOLDER,
private_ip_address_placeholder=constants.PRIVATE_IP_ADDRESS_PLACEHOLDER,
min_cpu=el_min_cpu,
max_cpu=el_max_cpu,
min_memory=el_min_mem,
......
......@@ -36,9 +36,9 @@ METRICS_PATH = "/debug/metrics/prometheus"
# The dirpath of the execution data directory on the client container
EXECUTION_DATA_DIRPATH_ON_CLIENT_CONTAINER = "/data/geth/execution-data"
PRIVATE_IP_ADDRESS_PLACEHOLDER = "KURTOSIS_IP_ADDR_PLACEHOLDER"
USED_PORTS = {
def get_used_ports(discovery_port=DISCOVERY_PORT_NUM):
used_ports = {
RPC_PORT_ID: shared_utils.new_port_spec(
RPC_PORT_NUM,
shared_utils.TCP_PROTOCOL,
......@@ -46,10 +46,10 @@ USED_PORTS = {
),
WS_PORT_ID: shared_utils.new_port_spec(WS_PORT_NUM, shared_utils.TCP_PROTOCOL),
TCP_DISCOVERY_PORT_ID: shared_utils.new_port_spec(
DISCOVERY_PORT_NUM, shared_utils.TCP_PROTOCOL
discovery_port, shared_utils.TCP_PROTOCOL
),
UDP_DISCOVERY_PORT_ID: shared_utils.new_port_spec(
DISCOVERY_PORT_NUM, shared_utils.UDP_PROTOCOL
discovery_port, shared_utils.UDP_PROTOCOL
),
ENGINE_RPC_PORT_ID: shared_utils.new_port_spec(
ENGINE_RPC_PORT_NUM,
......@@ -58,7 +58,9 @@ USED_PORTS = {
METRICS_PORT_ID: shared_utils.new_port_spec(
METRICS_PORT_NUM, shared_utils.TCP_PROTOCOL
),
}
}
return used_ports
ENTRYPOINT_ARGS = ["sh", "-c"]
......@@ -94,6 +96,7 @@ def launch(
el_volume_size,
tolerations,
node_selectors,
port_publisher,
):
log_level = input_parser.get_client_log_level_or_default(
participant_log_level, global_log_level, VERBOSITY_LEVELS
......@@ -146,6 +149,7 @@ def launch(
el_volume_size,
tolerations,
node_selectors,
port_publisher,
)
service = plan.add_service(service_name, config)
......@@ -196,6 +200,7 @@ def get_config(
el_volume_size,
tolerations,
node_selectors,
port_publisher,
):
if "--gcmode=archive" in extra_params or "--gcmode archive" in extra_params:
gcmode_archive = True
......@@ -232,6 +237,20 @@ def get_config(
constants.GENESIS_CONFIG_MOUNT_PATH_ON_CONTAINER + "/genesis.json",
)
public_ports = {}
discovery_port = DISCOVERY_PORT_NUM
if port_publisher.public_port_start:
discovery_port = port_publisher.el_start + len(existing_el_clients)
public_ports = {
TCP_DISCOVERY_PORT_ID: shared_utils.new_port_spec(
discovery_port, shared_utils.TCP_PROTOCOL
),
UDP_DISCOVERY_PORT_ID: shared_utils.new_port_spec(
discovery_port, shared_utils.UDP_PROTOCOL
),
}
used_ports = get_used_ports(discovery_port)
cmd = [
"geth",
# Disable path based storage scheme for electra fork and verkle
......@@ -271,7 +290,7 @@ def get_config(
"--ws.api=admin,engine,net,eth,web3,debug",
"--ws.origins=*",
"--allow-insecure-unlock",
"--nat=extip:" + PRIVATE_IP_ADDRESS_PLACEHOLDER,
"--nat=extip:" + port_publisher.nat_exit_ip,
"--verbosity=" + verbosity_level,
"--authrpc.port={0}".format(ENGINE_RPC_PORT_NUM),
"--authrpc.addr=0.0.0.0",
......@@ -282,6 +301,8 @@ def get_config(
"--metrics",
"--metrics.addr=0.0.0.0",
"--metrics.port={0}".format(METRICS_PORT_NUM),
"--discovery.port={0}".format(discovery_port),
"--port={0}".format(discovery_port),
]
if BUILDER_IMAGE_STR in image:
......@@ -352,11 +373,12 @@ def get_config(
)
return ServiceConfig(
image=image,
ports=USED_PORTS,
ports=used_ports,
public_ports=public_ports,
cmd=[command_str],
files=files,
entrypoint=ENTRYPOINT_ARGS,
private_ip_address_placeholder=PRIVATE_IP_ADDRESS_PLACEHOLDER,
private_ip_address_placeholder=constants.PRIVATE_IP_ADDRESS_PLACEHOLDER,
min_cpu=el_min_cpu,
max_cpu=el_max_cpu,
min_memory=el_min_mem,
......
......@@ -29,18 +29,20 @@ UDP_DISCOVERY_PORT_ID = "udp-discovery"
ENGINE_RPC_PORT_ID = "engine-rpc"
METRICS_PORT_ID = "metrics"
PRIVATE_IP_ADDRESS_PLACEHOLDER = "KURTOSIS_IP_ADDR_PLACEHOLDER"
USED_PORTS = {
def get_used_ports(discovery_port=DISCOVERY_PORT_NUM):
used_ports = {
RPC_PORT_ID: shared_utils.new_port_spec(
RPC_PORT_NUM, shared_utils.TCP_PROTOCOL, shared_utils.HTTP_APPLICATION_PROTOCOL
RPC_PORT_NUM,
shared_utils.TCP_PROTOCOL,
shared_utils.HTTP_APPLICATION_PROTOCOL,
),
WS_PORT_ID: shared_utils.new_port_spec(WS_PORT_NUM, shared_utils.TCP_PROTOCOL),
TCP_DISCOVERY_PORT_ID: shared_utils.new_port_spec(
DISCOVERY_PORT_NUM, shared_utils.TCP_PROTOCOL
discovery_port, shared_utils.TCP_PROTOCOL
),
UDP_DISCOVERY_PORT_ID: shared_utils.new_port_spec(
DISCOVERY_PORT_NUM, shared_utils.UDP_PROTOCOL
discovery_port, shared_utils.UDP_PROTOCOL
),
ENGINE_RPC_PORT_ID: shared_utils.new_port_spec(
ENGINE_RPC_PORT_NUM, shared_utils.TCP_PROTOCOL
......@@ -48,7 +50,9 @@ USED_PORTS = {
METRICS_PORT_ID: shared_utils.new_port_spec(
METRICS_PORT_NUM, shared_utils.TCP_PROTOCOL
),
}
}
return used_ports
VERBOSITY_LEVELS = {
constants.GLOBAL_LOG_LEVEL.error: "ERROR",
......@@ -78,6 +82,7 @@ def launch(
el_volume_size,
tolerations,
node_selectors,
port_publisher,
):
log_level = input_parser.get_client_log_level_or_default(
participant_log_level, global_log_level, VERBOSITY_LEVELS
......@@ -127,6 +132,7 @@ def launch(
el_volume_size,
tolerations,
node_selectors,
port_publisher,
)
service = plan.add_service(service_name, config)
......@@ -173,7 +179,22 @@ def get_config(
el_volume_size,
tolerations,
node_selectors,
port_publisher,
):
public_ports = {}
discovery_port = DISCOVERY_PORT_NUM
if port_publisher.public_port_start:
discovery_port = port_publisher.el_start + len(existing_el_clients)
public_ports = {
TCP_DISCOVERY_PORT_ID: shared_utils.new_port_spec(
discovery_port, shared_utils.TCP_PROTOCOL
),
UDP_DISCOVERY_PORT_ID: shared_utils.new_port_spec(
discovery_port, shared_utils.UDP_PROTOCOL
),
}
used_ports = get_used_ports(discovery_port)
cmd = [
"--log=" + log_level,
"--datadir=" + EXECUTION_DATA_DIRPATH_ON_CLIENT_CONTAINER,
......@@ -185,9 +206,9 @@ def get_config(
"--JsonRpc.WebSocketsPort={0}".format(WS_PORT_NUM),
"--JsonRpc.EngineHost=0.0.0.0",
"--JsonRpc.EnginePort={0}".format(ENGINE_RPC_PORT_NUM),
"--Network.ExternalIp={0}".format(PRIVATE_IP_ADDRESS_PLACEHOLDER),
"--Network.DiscoveryPort={0}".format(DISCOVERY_PORT_NUM),
"--Network.P2PPort={0}".format(DISCOVERY_PORT_NUM),
"--Network.ExternalIp={0}".format(port_publisher.nat_exit_ip),
"--Network.DiscoveryPort={0}".format(discovery_port),
"--Network.P2PPort={0}".format(discovery_port),
"--JsonRpc.JwtSecretFile=" + constants.JWT_MOUNT_PATH_ON_CONTAINER,
"--Metrics.Enabled=true",
"--Metrics.ExposePort={0}".format(METRICS_PORT_NUM),
......@@ -250,10 +271,11 @@ def get_config(
return ServiceConfig(
image=image,
ports=USED_PORTS,
ports=used_ports,
public_ports=public_ports,
cmd=cmd,
files=files,
private_ip_address_placeholder=PRIVATE_IP_ADDRESS_PLACEHOLDER,
private_ip_address_placeholder=constants.PRIVATE_IP_ADDRESS_PLACEHOLDER,
min_cpu=el_min_cpu,
max_cpu=el_max_cpu,
min_memory=el_min_mem,
......
......@@ -27,20 +27,20 @@ METRICS_PATH = "/metrics"
# The dirpath of the execution data directory on the client container
EXECUTION_DATA_DIRPATH_ON_CLIENT_CONTAINER = "/data/nimbus/execution-data"
PRIVATE_IP_ADDRESS_PLACEHOLDER = "KURTOSIS_IP_ADDR_PLACEHOLDER"
USED_PORTS = {
def get_used_ports(discovery_port=DISCOVERY_PORT_NUM):
used_ports = {
WS_RPC_PORT_ID: shared_utils.new_port_spec(
WS_RPC_PORT_NUM,
shared_utils.TCP_PROTOCOL,
shared_utils.HTTP_APPLICATION_PROTOCOL,
),
TCP_DISCOVERY_PORT_ID: shared_utils.new_port_spec(
DISCOVERY_PORT_NUM,
discovery_port,
shared_utils.TCP_PROTOCOL,
),
UDP_DISCOVERY_PORT_ID: shared_utils.new_port_spec(
DISCOVERY_PORT_NUM,
discovery_port,
shared_utils.UDP_PROTOCOL,
),
ENGINE_RPC_PORT_ID: shared_utils.new_port_spec(
......@@ -51,7 +51,9 @@ USED_PORTS = {
METRICS_PORT_NUM,
shared_utils.TCP_PROTOCOL,
),
}
}
return used_ports
VERBOSITY_LEVELS = {
constants.GLOBAL_LOG_LEVEL.error: "ERROR",
......@@ -82,6 +84,7 @@ def launch(
el_volume_size,
tolerations,
node_selectors,
port_publisher,
):
log_level = input_parser.get_client_log_level_or_default(
participant_log_level, global_log_level, VERBOSITY_LEVELS
......@@ -131,6 +134,7 @@ def launch(
el_volume_size,
tolerations,
node_selectors,
port_publisher,
)
service = plan.add_service(service_name, config)
......@@ -176,7 +180,22 @@ def get_config(
el_volume_size,
tolerations,
node_selectors,
port_publisher,
):
public_ports = {}
discovery_port = DISCOVERY_PORT_NUM
if port_publisher.public_port_start:
discovery_port = port_publisher.el_start + len(existing_el_clients)
public_ports = {
TCP_DISCOVERY_PORT_ID: shared_utils.new_port_spec(
discovery_port, shared_utils.TCP_PROTOCOL
),
UDP_DISCOVERY_PORT_ID: shared_utils.new_port_spec(
discovery_port, shared_utils.UDP_PROTOCOL
),
}
used_ports = get_used_ports(discovery_port)
cmd = [
"--log-level={0}".format(verbosity_level),
"--data-dir=" + EXECUTION_DATA_DIRPATH_ON_CLIENT_CONTAINER,
......@@ -193,7 +212,8 @@ def get_config(
"--metrics",
"--metrics-address=0.0.0.0",
"--metrics-port={0}".format(METRICS_PORT_NUM),
"--nat=extip:{0}".format(PRIVATE_IP_ADDRESS_PLACEHOLDER),
"--nat=extip:{0}".format(port_publisher.nat_exit_ip),
"--tcp-port={0}".format(discovery_port),
]
if (
network not in constants.PUBLIC_NETWORKS
......@@ -243,10 +263,11 @@ def get_config(
return ServiceConfig(
image=image,
ports=USED_PORTS,
ports=used_ports,
public_ports=public_ports,
cmd=cmd,
files=files,
private_ip_address_placeholder=PRIVATE_IP_ADDRESS_PLACEHOLDER,
private_ip_address_placeholder=constants.PRIVATE_IP_ADDRESS_PLACEHOLDER,
min_cpu=el_min_cpu,
max_cpu=el_max_cpu,
min_memory=el_min_mem,
......
......@@ -29,18 +29,20 @@ METRICS_PATH = "/metrics"
# The dirpath of the execution data directory on the client container
EXECUTION_DATA_DIRPATH_ON_CLIENT_CONTAINER = "/data/reth/execution-data"
PRIVATE_IP_ADDRESS_PLACEHOLDER = "KURTOSIS_IP_ADDR_PLACEHOLDER"
USED_PORTS = {
def get_used_ports(discovery_port=DISCOVERY_PORT_NUM):
used_ports = {
RPC_PORT_ID: shared_utils.new_port_spec(
RPC_PORT_NUM, shared_utils.TCP_PROTOCOL, shared_utils.HTTP_APPLICATION_PROTOCOL
RPC_PORT_NUM,
shared_utils.TCP_PROTOCOL,
shared_utils.HTTP_APPLICATION_PROTOCOL,
),
WS_PORT_ID: shared_utils.new_port_spec(WS_PORT_NUM, shared_utils.TCP_PROTOCOL),
TCP_DISCOVERY_PORT_ID: shared_utils.new_port_spec(
DISCOVERY_PORT_NUM, shared_utils.TCP_PROTOCOL
discovery_port, shared_utils.TCP_PROTOCOL
),
UDP_DISCOVERY_PORT_ID: shared_utils.new_port_spec(
DISCOVERY_PORT_NUM, shared_utils.UDP_PROTOCOL
discovery_port, shared_utils.UDP_PROTOCOL
),
ENGINE_RPC_PORT_ID: shared_utils.new_port_spec(
ENGINE_RPC_PORT_NUM, shared_utils.TCP_PROTOCOL
......@@ -48,7 +50,9 @@ USED_PORTS = {
METRICS_PORT_ID: shared_utils.new_port_spec(
METRICS_PORT_NUM, shared_utils.TCP_PROTOCOL
),
}
}
return used_ports
ENTRYPOINT_ARGS = ["sh", "-c"]
......@@ -81,6 +85,7 @@ def launch(
el_volume_size,
tolerations,
node_selectors,
port_publisher,
):
log_level = input_parser.get_client_log_level_or_default(
participant_log_level, global_log_level, VERBOSITY_LEVELS
......@@ -130,6 +135,7 @@ def launch(
el_volume_size,
tolerations,
node_selectors,
port_publisher,
)
service = plan.add_service(service_name, config)
......@@ -175,12 +181,27 @@ def get_config(
el_volume_size,
tolerations,
node_selectors,
port_publisher,
):
init_datadir_cmd_str = "reth init --datadir={0} --chain={1}".format(
EXECUTION_DATA_DIRPATH_ON_CLIENT_CONTAINER,
constants.GENESIS_CONFIG_MOUNT_PATH_ON_CONTAINER + "/genesis.json",
)
public_ports = {}
discovery_port = DISCOVERY_PORT_NUM
if port_publisher.public_port_start:
discovery_port = port_publisher.el_start + len(existing_el_clients)
public_ports = {
TCP_DISCOVERY_PORT_ID: shared_utils.new_port_spec(
discovery_port, shared_utils.TCP_PROTOCOL
),
UDP_DISCOVERY_PORT_ID: shared_utils.new_port_spec(
discovery_port, shared_utils.UDP_PROTOCOL
),
}
used_ports = get_used_ports(discovery_port)
cmd = [
"reth",
"node",
......@@ -203,11 +224,13 @@ def get_config(
"--ws.port={0}".format(WS_PORT_NUM),
"--ws.api=net,eth",
"--ws.origins=*",
"--nat=extip:" + PRIVATE_IP_ADDRESS_PLACEHOLDER,
"--nat=extip:" + port_publisher.nat_exit_ip,
"--authrpc.port={0}".format(ENGINE_RPC_PORT_NUM),
"--authrpc.jwtsecret=" + constants.JWT_MOUNT_PATH_ON_CONTAINER,
"--authrpc.addr=0.0.0.0",
"--metrics=0.0.0.0:{0}".format(METRICS_PORT_NUM),
"--discovery.port={0}".format(discovery_port),
"--port={0}".format(discovery_port),
]
if network == constants.NETWORK_NAME.kurtosis:
if len(existing_el_clients) > 0:
......@@ -256,11 +279,12 @@ def get_config(
return ServiceConfig(
image=image,
ports=USED_PORTS,
ports=used_ports,
public_ports=public_ports,
cmd=[command_str],
files=files,
entrypoint=ENTRYPOINT_ARGS,
private_ip_address_placeholder=PRIVATE_IP_ADDRESS_PLACEHOLDER,
private_ip_address_placeholder=constants.PRIVATE_IP_ADDRESS_PLACEHOLDER,
min_cpu=el_min_cpu,
max_cpu=el_max_cpu,
min_memory=el_min_mem,
......
......@@ -67,6 +67,8 @@ KEYMANAGER_P12_MOUNT_PATH_ON_CONTAINER = (
DEFAULT_SNOOPER_IMAGE = "ethpandaops/rpc-snooper:latest"
PRIVATE_IP_ADDRESS_PLACEHOLDER = "KURTOSIS_IP_ADDR_PLACEHOLDER"
GENESIS_FORK_VERSION = "0x10000038"
BELLATRIX_FORK_VERSION = "0x30000038"
CAPELLA_FORK_VERSION = "0x40000038"
......
......@@ -65,6 +65,7 @@ ATTR_TO_BE_SKIPPED_AT_ROOT = (
"tx_spammer_params",
"custom_flood_params",
"xatu_sentry_params",
"port_publisher",
)
......@@ -135,6 +136,10 @@ def input_parser(plan, input_args):
for sub_attr in input_args["xatu_sentry_params"]:
sub_value = input_args["xatu_sentry_params"][sub_attr]
result["xatu_sentry_params"][sub_attr] = sub_value
elif attr == "port_publisher":
for sub_attr in input_args["port_publisher"]:
sub_value = input_args["port_publisher"][sub_attr]
result["port_publisher"][sub_attr] = sub_value
if result.get("disable_peer_scoring"):
result = enrich_disable_peer_scoring(result)
......@@ -147,6 +152,14 @@ def input_parser(plan, input_args):
result.get("mev_type"),
)
if result["port_publisher"]["nat_exit_ip"] == "auto":
result["port_publisher"]["nat_exit_ip"] = get_public_ip(plan)
if result["port_publisher"]["public_port_start"] != None:
start = result["port_publisher"]["public_port_start"]
result["port_publisher"]["el_start"] = start
result["port_publisher"]["cl_start"] = start + len(result["participants"])
return struct(
participants=[
struct(
......@@ -303,6 +316,12 @@ def input_parser(plan, input_args):
global_tolerations=result["global_tolerations"],
global_node_selectors=result["global_node_selectors"],
keymanager_enabled=result["keymanager_enabled"],
port_publisher=struct(
public_port_start=result["port_publisher"]["public_port_start"],
nat_exit_ip=result["port_publisher"]["nat_exit_ip"],
el_start=result["port_publisher"].get("el_start"),
cl_start=result["port_publisher"].get("cl_start"),
),
)
......@@ -611,6 +630,10 @@ def default_input_args():
"global_tolerations": [],
"global_node_selectors": {},
"keymanager_enabled": False,
"port_publisher": {
"nat_exit_ip": constants.PRIVATE_IP_ADDRESS_PLACEHOLDER,
"public_port_start": None,
},
}
......@@ -878,3 +901,10 @@ def deep_copy_participant(participant):
else:
part[k] = v
return part
def get_public_ip(plan):
response = plan.run_sh(
run="curl -s https://ident.me",
)
return response.output
......@@ -39,7 +39,8 @@ def launch_participant_network(
global_tolerations,
global_node_selectors,
keymanager_enabled,
parallel_keystore_generation=False,
parallel_keystore_generation,
port_publisher,
):
network_id = network_params.network_id
latest_block = ""
......@@ -145,6 +146,7 @@ def launch_participant_network(
persistent,
network_id,
num_participants,
port_publisher,
)
# Launch all consensus layer clients
......@@ -180,6 +182,7 @@ def launch_participant_network(
validator_data,
prysm_password_relative_filepath,
prysm_password_artifact_uuid,
port_publisher,
)
ethereum_metrics_exporter_context = None
......
......@@ -8,9 +8,6 @@ SNOOPER_BEACON_RPC_PORT_NUM = 8562
SNOOPER_BEACON_RPC_PORT_ID = "http"
SNOOPER_BINARY_COMMAND = "./json_rpc_snoop"
PRIVATE_IP_ADDRESS_PLACEHOLDER = "KURTOSIS_IP_ADDR_PLACEHOLDER"
SNOOPER_USED_PORTS = {
SNOOPER_BEACON_RPC_PORT_ID: shared_utils.new_port_spec(
SNOOPER_BEACON_RPC_PORT_NUM, shared_utils.TCP_PROTOCOL, wait="5s"
......@@ -51,7 +48,6 @@ def get_config(service_name, cl_context, node_selectors):
image=constants.DEFAULT_SNOOPER_IMAGE,
ports=SNOOPER_USED_PORTS,
cmd=cmd,
private_ip_address_placeholder=PRIVATE_IP_ADDRESS_PLACEHOLDER,
min_cpu=MIN_CPU,
max_cpu=MAX_CPU,
min_memory=MIN_MEMORY,
......
......@@ -8,8 +8,6 @@ SNOOPER_ENGINE_RPC_PORT_NUM = 8561
SNOOPER_ENGINE_RPC_PORT_ID = "http"
SNOOPER_BINARY_COMMAND = "./json_rpc_snoop"
PRIVATE_IP_ADDRESS_PLACEHOLDER = "KURTOSIS_IP_ADDR_PLACEHOLDER"
SNOOPER_USED_PORTS = {
SNOOPER_ENGINE_RPC_PORT_ID: shared_utils.new_port_spec(
SNOOPER_ENGINE_RPC_PORT_NUM, shared_utils.TCP_PROTOCOL, wait="5s"
......@@ -52,7 +50,6 @@ def get_config(service_name, el_context, node_selectors):
image=constants.DEFAULT_SNOOPER_IMAGE,
ports=SNOOPER_USED_PORTS,
cmd=cmd,
private_ip_address_placeholder=PRIVATE_IP_ADDRESS_PLACEHOLDER,
min_cpu=MIN_CPU,
max_cpu=MAX_CPU,
min_memory=MIN_MEMORY,
......
......@@ -99,7 +99,6 @@ def get_config(
cmd=cmd,
env_vars=extra_env_vars,
files=files,
private_ip_address_placeholder=vc_shared.PRIVATE_IP_ADDRESS_PLACEHOLDER,
min_cpu=vc_min_cpu,
max_cpu=vc_max_cpu,
min_memory=vc_min_mem,
......
......@@ -77,7 +77,6 @@ def get_config(
cmd=cmd,
env_vars=extra_env_vars,
files=files,
private_ip_address_placeholder=vc_shared.PRIVATE_IP_ADDRESS_PLACEHOLDER,
min_cpu=vc_min_cpu,
max_cpu=vc_max_cpu,
min_memory=vc_min_mem,
......
......@@ -92,7 +92,6 @@ def get_config(
cmd=cmd,
env_vars=extra_env_vars,
files=files,
private_ip_address_placeholder=vc_shared.PRIVATE_IP_ADDRESS_PLACEHOLDER,
min_cpu=vc_min_cpu,
max_cpu=vc_max_cpu,
min_memory=vc_min_mem,
......
shared_utils = import_module("../shared_utils/shared_utils.star")
PRIVATE_IP_ADDRESS_PLACEHOLDER = "KURTOSIS_IP_ADDR_PLACEHOLDER"
VALIDATOR_HTTP_PORT_NUM = 5056
VALIDATOR_HTTP_PORT_ID = "vc-http"
......
......@@ -88,7 +88,6 @@ def get_config(
cmd=cmd,
env_vars=extra_env_vars,
files=files,
private_ip_address_placeholder=vc_shared.PRIVATE_IP_ADDRESS_PLACEHOLDER,
min_cpu=vc_min_cpu,
max_cpu=vc_max_cpu,
min_memory=vc_min_mem,
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment