feat(kurtosis-devnet): user devnets (#13728)

This change is an attempt at bridging the gap between kurtosis devnets
and future alphanet/betanet specification.
(see ethereum-optimism/devnets#4)

It does so by defining specific override points in a more generic
template.
Generic building blocks are defined in templates/, and rely on a new
"include" capability at the template level.

This approach might also provide a somewhat lighter entry point for
user devnet definitions, following the same principle.

Note that this is completely optional at this point, and the full
scope of the kurtosis definition is still available to whomever
needs/wants it.

kurtosis-devnet/.gitignore

-*.json
+*-user.json

kurtosis-devnet/cmd/main.go

@@ -226,6 +226,7 @@ func (m *Main) renderTemplate(dir string) (*bytes.Buffer, error) {
 		m.localDockerImageOption(),
 		m.localContractArtifactsOption(dir),
 		m.localPrestateOption(dir),
+		tmpl.WithBaseDir(m.cfg.baseDir),
 	}
 
 	// Read and parse the data file if provided

kurtosis-devnet/foo-user.example.json

+{
+    "interop": true,
+    "l2s": {
+        "2151908": {
+            "nodes": ["op-geth", "op-geth"]
+        },
+        "2151909": {
+            "nodes": ["op-reth"]
+        }
+    },
+    "overrides": {
+        "flags": {
+            "log_level": "--log.level=debug"
+        }
+    }
+}
\ No newline at end of file

kurtosis-devnet/interop.yaml

@@ -7,7 +7,7 @@
   "op_supervisor" (localDockerImage "op-supervisor")
 -}}
 {{- $urls := dict
-  "prestate" "http://fileserver/proofs/op-program/cannon"
+  "prestate" (localPrestate.URL)
   "l1_artifacts" (localContractArtifacts "l1")
   "l2_artifacts" (localContractArtifacts "l2")
 -}}

kurtosis-devnet/justfile

@@ -44,13 +44,23 @@ op-wheel-image TAG='op-wheel:devnet': (_docker_build_stack TAG "op-wheel-target"
 KURTOSIS_PACKAGE := "github.com/ethpandaops/optimism-package"
 
 # Devnet template recipe
-devnet TEMPLATE_FILE DATA_FILE="":
+devnet TEMPLATE_FILE DATA_FILE="" NAME="":
+    #!/usr/bin/env bash
+    export DEVNET_NAME={{NAME}}
+    if [ -z "{{NAME}}" ]; then
+        export DEVNET_NAME=`basename {{TEMPLATE_FILE}} .yaml`
+        if [ -n "{{DATA_FILE}}" ]; then
+            export DATA_FILE_NAME=`basename {{DATA_FILE}} .json`
+            export DEVNET_NAME="$DEVNET_NAME-$DATA_FILE_NAME"
+        fi
+    fi
+    export ENCL_NAME="$DEVNET_NAME"-devnet
     go run cmd/main.go -kurtosis-package {{KURTOSIS_PACKAGE}} \
-        -environment tests/`basename {{TEMPLATE_FILE}} .yaml`-devnet.json \
+        -environment "tests/$ENCL_NAME.json" \
         -template "{{TEMPLATE_FILE}}" \
         -data "{{DATA_FILE}}" \
-        -enclave `basename {{TEMPLATE_FILE}} .yaml`-devnet
-    cat tests/`basename {{TEMPLATE_FILE}} .yaml`-devnet.json
+        -enclave "$ENCL_NAME" \
+    && cat "tests/$ENCL_NAME.json"
 
 devnet-test DEVNET *TEST:
     #!/usr/bin/env bash
@@ -72,3 +82,7 @@ simple-devnet: (devnet "simple.yaml")
 # Interop devnet
 interop-devnet: (devnet "interop.yaml")
 interop-devnet-test: (devnet-test "interop-devnet" "interop-smoke-test.sh")
+
+# User devnet
+user-devnet DATA_FILE:
+    {{just_executable()}} devnet "user.yaml" {{DATA_FILE}} {{file_stem(DATA_FILE)}}

kurtosis-devnet/pkg/tmpl/cmd/main.go

 package main
 
 import (
+	"encoding/json"
 	"flag"
 	"fmt"
 	"os"
 	"strings"
 
+	"github.com/ethereum-optimism/optimism/kurtosis-devnet/pkg/tmpl"
 	"github.com/ethereum-optimism/optimism/kurtosis-devnet/pkg/tmpl/fake"
 )
 
 func main() {
-
 	// Parse command line flags
 	templateFile := flag.String("template", "", "Path to template file")
+	dataFile := flag.String("data", "", "Optional JSON data file")
 	flag.Parse()
 
 	if *templateFile == "" {
@@ -42,6 +44,23 @@ func main() {
 	// Create template context
 	ctx := fake.NewFakeTemplateContext(enclave)
 
+	// Load data file if provided
+	if *dataFile != "" {
+		dataBytes, err := os.ReadFile(*dataFile)
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "Error reading data file: %v\n", err)
+			os.Exit(1)
+		}
+
+		var data interface{}
+		if err := json.Unmarshal(dataBytes, &data); err != nil {
+			fmt.Fprintf(os.Stderr, "Error parsing data file as JSON: %v\n", err)
+			os.Exit(1)
+		}
+
+		tmpl.WithData(data)(ctx)
+	}
+
 	// Process template and write to stdout
 	if err := ctx.InstantiateTemplate(f, os.Stdout); err != nil {
 		fmt.Fprintf(os.Stderr, "Error processing template: %v\n", err)

kurtosis-devnet/pkg/tmpl/tmpl.go

 package tmpl
 
 import (
+	"bytes"
+	"encoding/json"
 	"fmt"
 	"io"
+	"os"
+	"path/filepath"
 	"text/template"
 
 	sprig "github.com/go-task/slim-sprig/v3"
+	"gopkg.in/yaml.v3"
 )
 
 // TemplateFunc represents a function that can be used in templates
@@ -13,12 +18,20 @@ type TemplateFunc any
 
 // TemplateContext contains data and functions to be passed to templates
 type TemplateContext struct {
-	Data      interface{}
-	Functions map[string]TemplateFunc
+	baseDir      string
+	Data         interface{}
+	Functions    map[string]TemplateFunc
+	includeStack []string // Track stack of included files to detect circular includes
 }
 
 type TemplateContextOptions func(*TemplateContext)
 
+func WithBaseDir(basedir string) TemplateContextOptions {
+	return func(ctx *TemplateContext) {
+		ctx.baseDir = basedir
+	}
+}
+
 func WithFunction(name string, fn TemplateFunc) TemplateContextOptions {
 	return func(ctx *TemplateContext) {
 		ctx.Functions[name] = fn
@@ -34,7 +47,9 @@ func WithData(data interface{}) TemplateContextOptions {
 // NewTemplateContext creates a new TemplateContext with default functions
 func NewTemplateContext(opts ...TemplateContextOptions) *TemplateContext {
 	ctx := &TemplateContext{
-		Functions: make(map[string]TemplateFunc),
+		baseDir:      ".",
+		Functions:    make(map[string]TemplateFunc),
+		includeStack: make([]string, 0),
 	}
 
 	for _, opt := range opts {
@@ -44,6 +59,73 @@ func NewTemplateContext(opts ...TemplateContextOptions) *TemplateContext {
 	return ctx
 }
 
+// includeFile reads and processes a template file relative to the given context's baseDir,
+// parses the content as YAML, and returns its JSON representation.
+// We use JSON because it can be inlined without worrying about indentation, while remaining valid YAML.
+// Note: to protect against infinite recursion, we check for circular includes.
+func (ctx *TemplateContext) includeFile(fname string, data ...interface{}) (string, error) {
+	// Resolve the file path relative to baseDir
+	path := filepath.Join(ctx.baseDir, fname)
+
+	// Check for circular includes
+	absPath, err := filepath.Abs(path)
+	if err != nil {
+		return "", fmt.Errorf("error resolving absolute path: %w", err)
+	}
+	for _, includedFile := range ctx.includeStack {
+		if includedFile == absPath {
+			return "", fmt.Errorf("circular include detected for file %s", fname)
+		}
+	}
+
+	// Read the included file
+	file, err := os.Open(path)
+	if err != nil {
+		return "", fmt.Errorf("error opening include file: %w", err)
+	}
+	defer file.Close()
+
+	// Create buffer for output
+	var buf bytes.Buffer
+
+	var tplData interface{}
+	switch len(data) {
+	case 0:
+		tplData = nil
+	case 1:
+		tplData = data[0]
+	default:
+		return "", fmt.Errorf("invalid number of arguments for includeFile: %d", len(data))
+	}
+
+	// Create new context with updated baseDir and include stack
+	includeCtx := &TemplateContext{
+		baseDir:      filepath.Dir(path),
+		Data:         tplData,
+		Functions:    ctx.Functions,
+		includeStack: append(append([]string{}, ctx.includeStack...), absPath),
+	}
+
+	// Process the included template
+	if err := includeCtx.InstantiateTemplate(file, &buf); err != nil {
+		return "", fmt.Errorf("error processing include file: %w", err)
+	}
+
+	// Parse the buffer content as YAML
+	var yamlData interface{}
+	if err := yaml.Unmarshal(buf.Bytes(), &yamlData); err != nil {
+		return "", fmt.Errorf("error parsing YAML: %w", err)
+	}
+
+	// Convert to JSON
+	jsonBytes, err := json.Marshal(yamlData)
+	if err != nil {
+		return "", fmt.Errorf("error converting to JSON: %w", err)
+	}
+
+	return string(jsonBytes), nil
+}
+
 // InstantiateTemplate reads a template from the reader, executes it with the context,
 // and writes the result to the writer
 func (ctx *TemplateContext) InstantiateTemplate(reader io.Reader, writer io.Writer) error {
@@ -54,7 +136,9 @@ func (ctx *TemplateContext) InstantiateTemplate(reader io.Reader, writer io.Writ
 	}
 
 	// Convert TemplateFunc map to FuncMap
-	funcMap := template.FuncMap{}
+	funcMap := template.FuncMap{
+		"include": ctx.includeFile,
+	}
 	for name, fn := range ctx.Functions {
 		funcMap[name] = fn
 	}
@@ -71,10 +155,35 @@ func (ctx *TemplateContext) InstantiateTemplate(reader io.Reader, writer io.Writ
 		return fmt.Errorf("failed to parse template: %w", err)
 	}
 
-	// Execute template with context
-	if err := tmpl.Execute(writer, ctx.Data); err != nil {
+	// Execute template into a buffer
+	var buf bytes.Buffer
+	if err := tmpl.Execute(&buf, ctx.Data); err != nil {
 		return fmt.Errorf("failed to execute template: %w", err)
 	}
 
+	// If this is the top-level rendering, we want to write the output as "pretty" YAML
+	if len(ctx.includeStack) == 0 {
+		var yamlData interface{}
+		// Parse the buffer content as YAML
+		if err := yaml.Unmarshal(buf.Bytes(), &yamlData); err != nil {
+			return fmt.Errorf("error parsing template output as YAML: %w. Template output: %s", err, buf.String())
+		}
+
+		// Create YAML encoder with default indentation
+		encoder := yaml.NewEncoder(writer)
+		encoder.SetIndent(2)
+
+		// Write the YAML document
+		if err := encoder.Encode(yamlData); err != nil {
+			return fmt.Errorf("error writing YAML output: %w", err)
+		}
+
+	} else {
+		// Otherwise, just write the buffer content to the writer
+		if _, err := buf.WriteTo(writer); err != nil {
+			return fmt.Errorf("failed to write template output: %w", err)
+		}
+	}
+
 	return nil
 }

kurtosis-devnet/pkg/tmpl/tmpl_test.go

@@ -44,7 +44,7 @@ func TestInstantiateTemplate(t *testing.T) {
 		err := ctx.InstantiateTemplate(input, &output)
 		require.NoError(t, err)
 
-		expected := "Hello world!"
+		expected := "Hello world!\n"
 		require.Equal(t, expected, output.String())
 	})
 
@@ -61,7 +61,7 @@ func TestInstantiateTemplate(t *testing.T) {
 		err := ctx.InstantiateTemplate(input, &output)
 		require.NoError(t, err)
 
-		expected := "Hello WORLD!"
+		expected := "Hello WORLD!\n"
 		require.Equal(t, expected, output.String())
 	})
 
@@ -104,7 +104,7 @@ func TestInstantiateTemplate(t *testing.T) {
 		err := ctx.InstantiateTemplate(input, &output)
 		require.NoError(t, err)
 
-		expected := "HELLO world!"
+		expected := "HELLO world!\n"
 		require.Equal(t, expected, output.String())
 	})
 }

kurtosis-devnet/templates/devnet.yaml

+{{- $context := or . (dict)}}
+{{- $default_l2s := dict 
+    "2151908" (dict "nodes" (list "op-geth")) 
+    "2151909" (dict "nodes" (list "op-geth"))
+}}
+{{- $l2s := dig "l2s" $default_l2s $context }}
+{{- $overrides := dig "overrides" (dict) $context }}
+{{- $interop := dig "interop" false $context }}
+---
+optimism_package:
+{{ if $interop }}
+  interop:
+    enabled: true
+    supervisor_params:
+      image: {{ dig "overrides" "images" "op_supervisor" (localDockerImage "op-supervisor") $context }}
+      extra_params:
+      - {{ dig "overrides" "flags" "log_level" "!!str" $context }}
+{{ end }}
+  chains:
+  {{ range $l2_id, $l2 := $l2s }}
+    - {{ include "l2.yaml" (dict "chain_id" $l2_id "overrides" $overrides "nodes" $l2.nodes) }}
+  {{ end }}
+  op_contract_deployer_params:
+    image: {{ dig "overrides" "images" "op_deployer" (localDockerImage "op-deployer") $context }}
+    l1_artifacts_locator: {{ dig "overrides" "urls" "l1_artifacts" (localContractArtifacts "l1") $context }}
+    l2_artifacts_locator: {{ dig "overrides" "urls" "l2_artifacts" (localContractArtifacts "l2") $context }}
+{{ if $interop }}
+    global_deploy_overrides:
+      faultGameAbsolutePrestate: {{ dig "overrides" "deployer" "prestate" (localPrestate.Hashes.prestate) $context }}
+{{ end }}
+  global_log_level: "info"
+  global_node_selectors: {}
+  global_tolerations: []
+  persistent: false
+ethereum_package:
+  network_params:
+    preset: minimal
+    genesis_delay: 5
+    additional_preloaded_contracts: |
+      {
+        "0x4e59b44847b379578588920cA78FbF26c0B4956C": {
+          "balance": "0ETH",
+          "code": "0x7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe03601600081602082378035828234f58015156039578182fd5b8082525050506014600cf3",
+          "storage": {},
+          "nonce": "1"
+        }
+      }

kurtosis-devnet/templates/l2.yaml

+{{- $context := or . (dict)}}
+{{- $nodes := dig "nodes" (list "op-geth") $context -}}
+---
+participants:
+{{- range $node := $nodes }}
+- {{ include "local-op-node.yaml" (dict "overrides" $context.overrides "el_type" $node) }}
+{{- end }}
+network_params:
+  network: "kurtosis"
+  network_id: "{{ .chain_id }}"
+  seconds_per_slot: 2
+  name: "op-kurtosis-{{ .chain_id }}"
+  fjord_time_offset: 0
+  granite_time_offset: 0
+  holocene_time_offset: 0
+  interop_time_offset: 0
+  fund_dev_accounts: true
+batcher_params:
+  image: {{ dig "overrides" "images" "op_batcher" (localDockerImage "op-batcher") $context }}
+  extra_params:
+  - {{ dig "overrides" "flags" "log_level" "!!str" $context }}
+challenger_params:
+  image: {{ dig "overrides" "images" "op_challenger" (localDockerImage "op-challenger") $context }}
+  cannon_prestate_path: ""
+  cannon_prestates_url: {{ dig "overrides" "urls" "prestate" (localPrestate.URL) $context }}
+  extra_params:
+  - {{ dig "overrides" "flags" "log_level" "!!str" $context }}
+proposer_params:
+  image: {{ dig "overrides" "images" "op_proposer" (localDockerImage "op-proposer") $context }}
+  extra_params:
+  - {{ dig "overrides" "flags" "log_level" "!!str" $context }}
+  game_type: 1
+  proposal_interval: 10m
+mev_params:
+  rollup_boost_image: ""
+  builder_host: ""
+  builder_port: ""
+additional_services: []
\ No newline at end of file

kurtosis-devnet/templates/local-op-node.yaml

+{{- $context := or . (dict)}}
+{{- $el_type := dig "el_type" "op-geth" $context -}}
+---
+el_type: {{ $el_type }}
+el_image: {{ dig "overrides" "images" $el_type "!!str" $context }}
+el_log_level: ""
+el_extra_env_vars: {}
+el_extra_labels: {}
+el_extra_params: []
+el_tolerations: []
+el_volume_size: 0
+el_min_cpu: 0
+el_max_cpu: 0
+el_min_mem: 0
+el_max_mem: 0
+cl_type: op-node
+cl_image: {{ dig "overrides" "images" "op-node" (localDockerImage "op-node") $context }}
+cl_log_level: ""
+cl_extra_env_vars: {}
+cl_extra_labels: {}
+cl_extra_params: []
+cl_tolerations: []
+cl_volume_size: 0
+cl_min_cpu: 0
+cl_max_cpu: 0
+cl_min_mem: 0
+cl_max_mem: 0
+node_selectors: {}
+tolerations: []
+count: 1
\ No newline at end of file

kurtosis-devnet/tests/.gitignore

+*.json

kurtosis-devnet/user.yaml

+{{- $context := or . (dict)}}
+---
+{{ include "templates/devnet.yaml" $context }}