Merge pull request #7323 from ethereum-optimism/willc/migrate

✨ feat(indexer): Add migrate command to cli

indexer/Dockerfile

@@ -22,5 +22,11 @@ RUN make indexer
 FROM alpine:3.18
 
 COPY --from=builder /app/indexer/indexer /usr/local/bin
+COPY --from=builder /app/indexer/indexer.toml /app/indexer/indexer.toml
+COPY --from=builder /app/indexer/migrations /app/indexer/migrations
+
+WORKDIR /app
+
+ENV INDEXER_MIGRATIONS_DIR="/app/indexer/migrations"
 
 CMD ["indexer", "index", "--config", "/app/indexer/indexer.toml"]

indexer/cmd/indexer/cli.go

@@ -19,6 +19,12 @@ var (
 		Usage:   "path to config file",
 		EnvVars: []string{"INDEXER_CONFIG"},
 	}
+	MigrationsFlag = &cli.StringFlag{
+		Name:    "migrations-dir",
+		Value:   "./migrations",
+		Usage:   "path to migrations folder",
+		EnvVars: []string{"INDEXER_MIGRATIONS_DIR"},
+	}
 )
 
 func runIndexer(ctx *cli.Context) error {
@@ -64,9 +70,30 @@ func runApi(ctx *cli.Context) error {
 	return api.Start(ctx.Context)
 }
 
+func runMigrations(ctx *cli.Context) error {
+	log := log.NewLogger(log.ReadCLIConfig(ctx)).New("role", "api")
+	cfg, err := config.LoadConfig(log, ctx.String(ConfigFlag.Name))
+	migrationsDir := ctx.String(MigrationsFlag.Name)
+	if err != nil {
+		log.Error("failed to load config", "err", err)
+		return err
+	}
+
+	db, err := database.NewDB(cfg.DB)
+	if err != nil {
+		log.Error("failed to connect to database", "err", err)
+		return err
+	}
+	defer db.Close()
+
+	return db.ExecuteSQLMigration(migrationsDir)
+}
+
 func newCli(GitCommit string, GitDate string) *cli.App {
 	flags := []cli.Flag{ConfigFlag}
 	flags = append(flags, log.CLIFlags("INDEXER")...)
+	migrationFlags := []cli.Flag{MigrationsFlag, ConfigFlag}
+	migrationFlags = append(migrationFlags, log.CLIFlags("INDEXER")...)
 	return &cli.App{
 		Version:              params.VersionWithCommit(GitCommit, GitDate),
 		Description:          "An indexer of all optimism events with a serving api layer",
@@ -84,6 +111,12 @@ func newCli(GitCommit string, GitDate string) *cli.App {
 				Description: "Runs the indexing service",
 				Action:      runIndexer,
 			},
+			{
+				Name:        "migrate",
+				Flags:       migrationFlags,
+				Description: "Runs the database migrations",
+				Action:      runMigrations,
+			},
 			{
 				Name:        "version",
 				Description: "print version",

indexer/database/db.go

@@ -4,6 +4,8 @@ package database
 import (
 	"context"
 	"fmt"
+	"os"
+	"path/filepath"
 
 	"github.com/ethereum-optimism/optimism/indexer/config"
 	_ "github.com/ethereum-optimism/optimism/indexer/database/serializers"
@@ -62,7 +64,6 @@ func NewDB(dbConfig config.DBConfig) (*DB, error) {
 	if err != nil {
 		return nil, errors.Wrap(err, "failed to connect to database after multiple retries")
 	}
-
 	db := &DB{
 		gorm:               gorm,
 		Blocks:             newBlocksDB(gorm),
@@ -71,7 +72,6 @@ func NewDB(dbConfig config.DBConfig) (*DB, error) {
 		BridgeMessages:     newBridgeMessagesDB(gorm),
 		BridgeTransactions: newBridgeTransactionsDB(gorm),
 	}
-
 	return db, nil
 }
 
@@ -102,3 +102,33 @@ func dbFromGormTx(tx *gorm.DB) *DB {
 		BridgeTransactions: newBridgeTransactionsDB(tx),
 	}
 }
+
+func (db *DB) ExecuteSQLMigration(migrationsFolder string) error {
+	err := filepath.Walk(migrationsFolder, func(path string, info os.FileInfo, err error) error {
+		// Check for any walking error
+		if err != nil {
+			return errors.Wrap(err, fmt.Sprintf("Failed to process migration file: %s", path))
+		}
+
+		// Skip directories
+		if info.IsDir() {
+			return nil
+		}
+
+		// Read the migration file content
+		fileContent, readErr := os.ReadFile(path)
+		if readErr != nil {
+			return errors.Wrap(readErr, fmt.Sprintf("Error reading SQL file: %s", path))
+		}
+
+		// Execute the migration
+		execErr := db.gorm.Exec(string(fileContent)).Error
+		if execErr != nil {
+			return errors.Wrap(execErr, fmt.Sprintf("Error executing SQL script: %s", path))
+		}
+
+		return nil
+	})
+
+	return err
+}

indexer/docker-compose.yml

@@ -17,6 +17,27 @@ services:
       - postgres_data:/data/postgres
       - ./migrations:/docker-entrypoint-initdb.d/
 
+  migrations:
+    build:
+      context: ..
+      dockerfile: indexer/Dockerfile
+    command: ["indexer", "migrate"]
+    environment:
+      - INDEXER_RPC_URL_L1=$INDEXER_RPC_URL_L1
+      - INDEXER_RPC_URL_L2=$INDEXER_RPC_URL_L2
+      - INDEXER_CONFIG=/indexer/indexer.toml
+      - INDEXER_CHAIN_PRESET=$INDEXER_CHAIN_PRESET
+      - INDEXER_DB_PORT=5432
+      - INDEXER_DB_HOST=postgres
+      - INDEXER_DB_USER=db_username 
+      - INDEXER_DB_PASS=db_password 
+      - INDEXER_DB_NAME=db_name
+    volumes:
+      - ./indexer.toml:/indexer/indexer.toml
+    depends_on:
+      postgres:
+        condition: service_healthy
+
   indexer:
     build:
       context: ..
@@ -26,11 +47,21 @@ services:
       - INDEXER_RPC_URL_L1=$INDEXER_RPC_URL_L1
       - INDEXER_RPC_URL_L2=$INDEXER_RPC_URL_L2
       - INDEXER_CONFIG=/indexer/indexer.toml
+      - INDEXER_CHAIN_PRESET=$INDEXER_CHAIN_PRESET
+      - INDEXER_DB_PORT=5432
+      - INDEXER_DB_HOST=postgres
+      - INDEXER_DB_USER=db_username 
+      - INDEXER_DB_PASS=db_password 
+      - INDEXER_DB_NAME=db_name
     volumes:
       - ./indexer.toml:/indexer/indexer.toml
     depends_on:
       postgres:
         condition: service_healthy
+    depends_on:
+      migrations:
+        condition: service_started
+
 
   api:
     build:
@@ -45,6 +76,12 @@ services:
       - INDEXER_RPC_URL_L1=$INDEXER_RPC_URL_L1
       - INDEXER_RPC_URL_L2=$INDEXER_RPC_URL_L2
       - INDEXER_CONFIG=/indexer/indexer.toml
+      - INDEXER_CHAIN_PRESET=$INDEXER_CHAIN_PRESET
+      - INDEXER_DB_HOST=postgres
+      - INDEXER_DB_PORT=5432
+      - INDEXER_DB_USER=db_username 
+      - INDEXER_DB_PASS=db_password 
+      - INDEXER_DB_NAME=db_name
     volumes:
       - ./indexer.toml:/indexer/indexer.toml
     ports:
@@ -83,6 +120,7 @@ services:
 
   backend-goerli:
     image: ethereumoptimism/gateway-backend:latest
+    platform: linux/amd64
     environment:
       # this enables the backend to proxy history requests to the indexer
       - BRIDGE_INDEXER_URI=http://api

indexer/e2e_tests/setup.go

@@ -4,9 +4,7 @@ import (
 	"context"
 	"database/sql"
 	"fmt"
-	"io/fs"
 	"os"
-	"path/filepath"
 	"testing"
 	"time"
 
@@ -132,31 +130,20 @@ func setupTestDatabase(t *testing.T) string {
 		pg.Close()
 	})
 
-	// setup schema, migration files ware walked in lexical order
-	t.Logf("created database %s", dbName)
-	db, err := sql.Open("pgx", fmt.Sprintf("postgres://%s@localhost:5432/%s?sslmode=disable", user, dbName))
+	dbConfig := config.DBConfig{
+		Host:     "127.0.0.1",
+		Port:     5432,
+		Name:     dbName,
+		User:     user,
+		Password: "",
+	}
+	// NewDB will create the database schema
+	db, err := database.NewDB(dbConfig)
 	require.NoError(t, err)
-	require.NoError(t, db.Ping())
 	defer db.Close()
+	err = db.ExecuteSQLMigration("../migrations")
+	require.NoError(t, err)
 
-	t.Logf("running schema migrations...")
-	require.NoError(t, filepath.Walk("../migrations", func(path string, info fs.FileInfo, err error) error {
-		if err != nil {
-			return err
-		} else if info.IsDir() {
-			return nil
-		}
-
-		t.Logf("running schema migration: %s", path)
-		data, err := os.ReadFile(path)
-		if err != nil {
-			return err
-		}
-
-		_, err = db.Exec(string(data))
-		return err
-	}))
-
-	t.Logf("schema loaded")
+	t.Logf("database %s setup and migrations executed", dbName)
 	return dbName
 }