feat: add full beacon chain explorer (#253)

I reworked #178 to work with the newly merged repository and other
changes we have made since. Big thanks to Peter (@peterbitfly ) the
original author of the PR

---------
Co-authored-by: peter <1674920+peterbitfly@users.noreply.github.com>

README.md

@@ -226,10 +226,16 @@ To configure the package behaviour, you can modify your `network_params.json` fi
         "cl_forkmon",
         "el_forkmon",
         "beacon_metrics_gazer",
-        "dora",
+        "explorer",
         "prometheus_grafana"
     ],
 
+    // Which blockchain explorer should be used
+    // "dora" will use the dora explorer developped by pk910
+    // "full" will use the explorer developped by the beaconcha.in team
+    // defaults to "light"
+    "explorer_version": "dora",
+
     //  If set, the package will block until a finalized epoch has occurred.
     "wait_for_finalization": false,
 

main.star

@@ -31,6 +31,9 @@ beacon_metrics_gazer = import_module(
 dora = import_module(
     "github.com/kurtosis-tech/ethereum-package/src/dora/dora_launcher.star"
 )
+full_beaconchain_explorer = import_module(
+    "github.com/kurtosis-tech/ethereum-package/src/full_beaconchain/full_beaconchain_launcher.star"
+)
 prometheus = import_module(
     "github.com/kurtosis-tech/ethereum-package/src/prometheus/prometheus_launcher.star"
 )
@@ -299,11 +302,32 @@ def run(plan, args={}):
                 beacon_metrics_gazer_prometheus_metrics_job
             )
             plan.print("Succesfully launched beacon metrics gazer")
-        elif additional_service == "dora":
-            plan.print("Launching dora")
-            dora_config_template = read_file(static_files.DORA_CONFIG_TEMPLATE_FILEPATH)
-            dora.launch_dora(plan, dora_config_template, all_cl_client_contexts)
-            plan.print("Succesfully launched dora")
+        elif additional_service == "explorer":
+            if args_with_right_defaults.explorer_version == "dora":
+                plan.print("Launching dora")
+                dora_config_template = read_file(
+                    static_files.DORA_CONFIG_TEMPLATE_FILEPATH
+                )
+                dora.launch_dora(plan, dora_config_template, all_cl_client_contexts)
+                plan.print("Succesfully launched dora")
+            elif args_with_right_defaults.explorer_version == "full":
+                plan.print("Launching full-beaconchain-explorer")
+                full_beaconchain_explorer_config_template = read_file(
+                    static_files.FULL_BEACONCHAIN_CONFIG_TEMPLATE_FILEPATH
+                )
+                full_beaconchain_explorer.launch_full_beacon(
+                    plan,
+                    full_beaconchain_explorer_config_template,
+                    all_cl_client_contexts,
+                    all_el_client_contexts,
+                )
+                plan.print("Succesfully launched full-beaconchain-explorer")
+            else:
+                fail(
+                    "expected explorer_version to be one of (dora, full) but got {0} which is invalid".format(
+                        args_with_right_defaults.explorer_version
+                    )
+                )
         elif additional_service == "prometheus_grafana":
             # Allow prometheus to be launched last so is able to collect metrics from other services
             launch_prometheus_grafana = True

network_params.json

@@ -27,6 +27,16 @@
 		"electra_fork_epoch": null
 	},
 	"launch_additional_services": true,
+	"additional_services": [
+		"tx_spammer",
+		"blob_spammer",
+		"cl_forkmon",
+		"el_forkmon",
+		"beacon_metrics_gazer",
+		"explorer",
+		"prometheus_grafana"
+	],
+	"explorer_version": "dora",
 	"wait_for_finalization": false,
 	"global_client_log_level": "info",
 	"snooper_enabled": false,

src/full_beaconchain/full_beaconchain_launcher.star

+shared_utils = import_module(
+    "github.com/kurtosis-tech/ethereum-package/src/shared_utils/shared_utils.star"
+)
+IMAGE_NAME = "gobitfly/eth2-beaconchain-explorer:kurtosis"
+
+POSTGRES_PORT_ID = "postgres"
+POSTGRES_PORT_NUMBER = 5432
+POSTGRES_DB = "db"
+POSTGRES_USER = "postgres"
+POSTGRES_PASSWORD = "pass"
+
+REDIS_PORT_ID = "redis"
+REDIS_PORT_NUMBER = 6379
+
+FRONTEND_PORT_ID = "http"
+FRONTEND_PORT_NUMBER = 8080
+
+LITTLE_BIGTABLE_PORT_ID = "littlebigtable"
+LITTLE_BIGTABLE_PORT_NUMBER = 9000
+
+FULL_BEACONCHAIN_CONFIG_FILENAME = "config.yml"
+
+
+USED_PORTS = {
+    FRONTEND_PORT_ID: shared_utils.new_port_spec(
+        FRONTEND_PORT_NUMBER,
+        shared_utils.TCP_PROTOCOL,
+        shared_utils.HTTP_APPLICATION_PROTOCOL,
+    )
+}
+
+
+def launch_full_beacon(
+    plan,
+    config_template,
+    cl_client_contexts,
+    el_client_contexts,
+):
+    # TODO perhaps use the official redis & postgres packages
+    db_services = plan.add_services(
+        configs={
+            # Add a Postgres server
+            "explorer-postgres": ServiceConfig(
+                image="postgres:15.2-alpine",
+                ports={
+                    POSTGRES_PORT_ID: PortSpec(
+                        POSTGRES_PORT_NUMBER, application_protocol="postgresql"
+                    ),
+                },
+                env_vars={
+                    "POSTGRES_DB": POSTGRES_DB,
+                    "POSTGRES_USER": POSTGRES_USER,
+                    "POSTGRES_PASSWORD": POSTGRES_PASSWORD,
+                },
+            ),
+            # Add a Redis server
+            "explorer-redis": ServiceConfig(
+                image="redis:7",
+                ports={
+                    REDIS_PORT_ID: PortSpec(
+                        REDIS_PORT_NUMBER, application_protocol="tcp"
+                    ),
+                },
+            ),
+            # Add a Bigtable Emulator server
+            "explorer-littlebigtable": ServiceConfig(
+                image="gobitfly/little_bigtable:latest",
+                ports={
+                    LITTLE_BIGTABLE_PORT_ID: PortSpec(
+                        LITTLE_BIGTABLE_PORT_NUMBER, application_protocol="tcp"
+                    ),
+                },
+            ),
+        }
+    )
+
+    el_uri = "http://{0}:{1}".format(
+        el_client_contexts[0].ip_addr, el_client_contexts[0].rpc_port_num
+    )
+    redis_uri = "{0}:{1}".format(
+        db_services["explorer-redis"].ip_address, REDIS_PORT_NUMBER
+    )
+
+    template_data = new_config_template_data(
+        cl_client_contexts[0],
+        el_uri,
+        db_services["explorer-littlebigtable"].ip_address,
+        LITTLE_BIGTABLE_PORT_NUMBER,
+        db_services["explorer-postgres"].ip_address,
+        POSTGRES_PORT_NUMBER,
+        redis_uri,
+        FRONTEND_PORT_NUMBER,
+    )
+
+    template_and_data = shared_utils.new_template_and_data(
+        config_template, template_data
+    )
+    template_and_data_by_rel_dest_filepath = {}
+    template_and_data_by_rel_dest_filepath[
+        FULL_BEACONCHAIN_CONFIG_FILENAME
+    ] = template_and_data
+
+    config_files_artifact_name = plan.render_templates(
+        template_and_data_by_rel_dest_filepath, "config.yml"
+    )
+
+    # Initialize the db schema
+    initdbschema = plan.add_service(
+        name="explorer-initdbschema",
+        config=ServiceConfig(
+            image=IMAGE_NAME,
+            files={
+                "/app/config/": config_files_artifact_name,
+            },
+            entrypoint=["./misc"],
+            cmd=["-config", "/app/config/config.yml", "-command", "applyDbSchema"],
+        ),
+    )
+    # Initialize the bigtable schema
+    initbigtableschema = plan.add_service(
+        name="explorer-initbigtableschema",
+        config=ServiceConfig(
+            image=IMAGE_NAME,
+            files={
+                "/app/config/": config_files_artifact_name,
+            },
+            entrypoint=["./misc"],
+            cmd=["-config", "/app/config/config.yml", "-command", "initBigtableSchema"],
+        ),
+    )
+    # Start the indexer
+    indexer = plan.add_service(
+        name="explorer-indexer",
+        config=ServiceConfig(
+            image=IMAGE_NAME,
+            files={
+                "/app/config/": config_files_artifact_name,
+            },
+            entrypoint=["./explorer"],
+            cmd=[
+                "-config",
+                "/app/config/config.yml",
+            ],
+            env_vars={
+                "INDEXER_ENABLED": "TRUE",
+            },
+        ),
+    )
+    # Start the eth1indexer
+    eth1indexer = plan.add_service(
+        name="explorer-eth1indexer",
+        config=ServiceConfig(
+            image=IMAGE_NAME,
+            files={
+                "/app/config/": config_files_artifact_name,
+            },
+            entrypoint=["./eth1indexer"],
+            cmd=[
+                "-config",
+                "/app/config/config.yml",
+                "-blocks.concurrency",
+                "1",
+                "-blocks.tracemode",
+                "geth",
+                "-data.concurrency",
+                "1",
+                "-balances.enabled",
+            ],
+        ),
+    )
+
+    rewardsexporter = plan.add_service(
+        name="explorer-rewardsexporter",
+        config=ServiceConfig(
+            image=IMAGE_NAME,
+            files={
+                "/app/config/": config_files_artifact_name,
+            },
+            entrypoint=["./rewards-exporter"],
+            cmd=[
+                "-config",
+                "/app/config/config.yml",
+            ],
+        ),
+    )
+
+    statistics = plan.add_service(
+        name="explorer-statistics",
+        config=ServiceConfig(
+            image=IMAGE_NAME,
+            files={
+                "/app/config/": config_files_artifact_name,
+            },
+            entrypoint=["./statistics"],
+            cmd=[
+                "-config",
+                "/app/config/config.yml",
+                "-charts.enabled",
+                "-graffiti.enabled",
+                "-validators.enabled",
+            ],
+        ),
+    )
+
+    fdu = plan.add_service(
+        name="explorer-fdu",
+        config=ServiceConfig(
+            image=IMAGE_NAME,
+            files={
+                "/app/config/": config_files_artifact_name,
+            },
+            entrypoint=["./frontend-data-updater"],
+            cmd=[
+                "-config",
+                "/app/config/config.yml",
+            ],
+        ),
+    )
+
+    frontend = plan.add_service(
+        name="explorer-frontend",
+        config=ServiceConfig(
+            image=IMAGE_NAME,
+            files={
+                "/app/config/": config_files_artifact_name,
+            },
+            entrypoint=["./explorer"],
+            cmd=[
+                "-config",
+                "/app/config/config.yml",
+            ],
+            env_vars={
+                "FRONTEND_ENABLED": "TRUE",
+            },
+            ports={
+                FRONTEND_PORT_ID: PortSpec(
+                    FRONTEND_PORT_NUMBER, application_protocol="http"
+                ),
+            },
+        ),
+    )
+
+
+def new_config_template_data(
+    cl_node_info, el_uri, lbt_host, lbt_port, db_host, db_port, redis_uri, frontend_port
+):
+    return {
+        "CLNodeHost": cl_node_info.ip_addr,
+        "CLNodePort": cl_node_info.http_port_num,
+        "ELNodeEndpoint": el_uri,
+        "LBTHost": lbt_host,
+        "LBTPort": lbt_port,
+        "DBHost": db_host,
+        "DBPort": db_port,
+        "RedisEndpoint": redis_uri,
+        "FrontendPort": frontend_port,
+    }
+
+
+def new_cl_client_info(ip_addr, port_num, service_name):
+    return {"IPAddr": ip_addr, "PortNum": port_num, "Name": service_name}

src/package_io/parse_input.star

@@ -33,7 +33,7 @@ DEFAULT_ADDITIONAL_SERVICES = [
     "cl_forkmon",
     "el_forkmon",
     "beacon_metrics_gazer",
-    "dora",
+    "explorer",
     "prometheus_grafana",
 ]
 
@@ -44,6 +44,8 @@ ATTR_TO_BE_SKIPPED_AT_ROOT = (
     "tx_spammer_params",
 )
 
+DEFAULT_EXPLORER_VERSION = "dora"
+
 package_io_constants = import_module(
     "github.com/kurtosis-tech/ethereum-package/src/package_io/constants.star"
 )
@@ -61,6 +63,7 @@ def parse_input(plan, input_args):
     result["mev_params"] = get_default_mev_params()
     result["launch_additional_services"] = True
     result["additional_services"] = DEFAULT_ADDITIONAL_SERVICES
+    result["explorer_version"] = DEFAULT_EXPLORER_VERSION
 
     for attr in input_args:
         value = input_args[attr]
@@ -162,6 +165,7 @@ def parse_input(plan, input_args):
         mev_type=result["mev_type"],
         snooper_enabled=result["snooper_enabled"],
         parallel_keystore_generation=result["parallel_keystore_generation"],
+        explorer_version=result["explorer_version"],
     )
 
 

src/static_files/static_files.star

@@ -23,6 +23,10 @@ BEACON_METRICS_GAZER_CONFIG_TEMPLATE_FILEPATH = (
 
 DORA_CONFIG_TEMPLATE_FILEPATH = STATIC_FILES_DIRPATH + "/dora-config/config.yaml.tmpl"
 
+FULL_BEACONCHAIN_CONFIG_TEMPLATE_FILEPATH = (
+    STATIC_FILES_DIRPATH + "/full-beaconchain-config/config.yaml.tmpl"
+)
+
 # Grafana config
 GRAFANA_CONFIG_DIRPATH = "/grafana-config"
 GRAFANA_DATASOURCE_CONFIG_TEMPLATE_FILEPATH = (

static_files/full-beaconchain-config/config.yaml.tmpl

+chain:
+  configPath: 'node'
+readerDatabase:
+  name: db
+  host: {{.DBHost}}
+  port: {{.DBPort}}
+  user: postgres
+  password: "pass"
+writerDatabase:
+  name: db
+  host: {{.DBHost}}
+  port: {{.DBPort}}
+  user: postgres
+  password: "pass"
+bigtable:
+  project: explorer
+  instance: explorer
+  emulator: true
+  emulatorHost: {{.LBTHost}}
+  emulatorPort: {{.LBTPort}}
+eth1ErigonEndpoint: '{{.ELNodeEndpoint}}'
+eth1GethEndpoint: '{{.ELNodeEndpoint}}'
+redisCacheEndpoint: '{{.RedisEndpoint}}'
+tieredCacheProvider: 'redis'
+frontend:
+  siteDomain: "localhost:8080"
+  siteName: 'Open Source Ethereum (ETH) Testnet Explorer' # Name of the site, displayed in the title tag
+  siteSubtitle: "Showing a local testnet."
+  server:
+    host: '0.0.0.0' # Address to listen on
+    port: '{{.FrontendPort}}' # Port to listen on
+  readerDatabase:
+    name: db
+    host: {{.DBHost}}
+    port: {{.DBPort}}
+    user: postgres
+    password: "pass"
+  writerDatabase:
+    name: db
+    host: {{.DBHost}}
+    port: {{.DBPort}}
+    user: postgres
+    password: "pass"
+  sessionSecret: "11111111111111111111111111111111"
+  jwtSigningSecret: "1111111111111111111111111111111111111111111111111111111111111111"
+  jwtIssuer: "localhost"
+  jwtValidityInMinutes: 30
+  maxMailsPerEmailPerDay: 10
+  mail:
+    mailgun:
+      sender: no-reply@localhost
+      domain: mg.localhost
+      privateKey: "key-11111111111111111111111111111111"
+  csrfAuthKey: '1111111111111111111111111111111111111111111111111111111111111111'
+indexer:
+  # fullIndexOnStartup: false # Perform a one time full db index on startup
+  # indexMissingEpochsOnStartup: true # Check for missing epochs and export them after startup
+  node:
+    host: '{{.CLNodeHost}}'
+    port: '{{.CLNodePort}}'
+    type: lighthouse
+  eth1DepositContractFirstBlock: 0