Merge pull request #7811 from ethereum-optimism/fix-docker-releases

ops,ci: docker buildx bake, fix publish/release flows

.circleci/config.yml

@@ -148,18 +148,13 @@ jobs:
     environment:
       DOCKER_BUILDKIT: 1
     parameters:
-      docker_name:
-        description: Docker image name
-        type: string
       docker_tags:
-        description: Docker image tags as csv
-        type: string
-      docker_file:
-        description: Path to Dockerfile
+        description: Docker image tags, comma-separated
         type: string
-      docker_context:
-        description: Docker build context
+      docker_name:
+        description: "Docker buildx bake target"
         type: string
+        default: ""
       registry:
         description: Docker registry
         type: string
@@ -168,42 +163,50 @@ jobs:
         description: Docker repo
         type: string
         default: "oplabs-tools-artifacts/images"
-      build_args:
-        description: Docker build args
-        type: string
-        default: ""
-      load_base_image:
-        description: Load docker image as base
-        type: string
-        default: ""
       save_image_tag:
-        description: Save docker image as
+        description: Save docker image with given tag
         type: string
         default: ""
+      platforms:
+        description: Platforms to build for, comma-separated
+        type: string
+        default: "linux/amd64"
+      publish:
+        description: Publish the docker image (multi-platform, all tags)
+        type: boolean
+        default: false
+      release:
+        description: Run the release script
+        type: boolean
+        default: false
     machine:
       image: ubuntu-2204:2022.07.1
       resource_class: medium
-      docker_layer_caching: true
+      docker_layer_caching: true  # we rely on this for faster builds, and actively warm it up for builds with common stages
     steps:
       - checkout
-      - restore_cache:
-          name: Restore docker build cache
-          key: docker-build-cache
       - attach_workspace:
           at: /tmp/docker_images
       - run:
           command: mkdir -p /tmp/docker_images
       - when:
-          condition: "<<parameters.load_base_image>>"
+          condition: "<<parameters.release>>"
           steps:
-            - run:
-                name: Load OP-Stack Go base image
-                command: |
-                  docker load < "/tmp/docker_images/<<parameters.load_base_image>>.tar"
-      - run:
-          name: build args
-          command: |
-            echo "build args: <<parameters.build_args>>"
+            - gcp-cli/install
+      - when:
+          condition:
+            or:
+              - "<<parameters.publish>>"
+              - "<<parameters.release>>"
+          steps:
+            - gcp-oidc-authenticate
+      # Below is CircleCI recommended way of specifying nameservers on an Ubuntu box:
+      # https://support.circleci.com/hc/en-us/articles/7323511028251-How-to-set-custom-DNS-on-Ubuntu-based-images-using-netplan
+      - run: sudo sed -i '13 i \ \ \ \ \ \ \ \ \ \ \ \ nameservers:' /etc/netplan/50-cloud-init.yaml
+      - run: sudo sed -i '14 i \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ addresses:' /etc/netplan/50-cloud-init.yaml
+      - run: sudo sed -i "s/addresses:/ addresses":" [8.8.8.8, 8.8.4.4] /g" /etc/netplan/50-cloud-init.yaml
+      - run: cat /etc/netplan/50-cloud-init.yaml
+      - run: sudo netplan apply
       - run:
           name: Build
           command: |
@@ -212,15 +215,43 @@ jobs:
             if [[ -v DOCKER_HUB_READ_ONLY_TOKEN ]]; then
               echo "$DOCKER_HUB_READ_ONLY_TOKEN" | docker login -u "$DOCKER_HUB_READ_ONLY_USER" --password-stdin
             fi
-            IMAGE_BASE="<<parameters.registry>>/<<parameters.repo>>/<<parameters.docker_name>>"
-            DOCKER_TAGS=$(echo -ne <<parameters.docker_tags>> | sed "s/,/\n/g" | sed "s/[^a-zA-Z0-9\n]/-/g" | sed -e "s|^|-t ${IMAGE_BASE}:|")
-            IMAGE_BASE_PREFIX="<<parameters.registry>>/<<parameters.repo>>"
-            IMAGE_BASE="$IMAGE_BASE_PREFIX/<<parameters.docker_name>>"
-            docker build --progress plain \
-            <<parameters.build_args>> \
-            $(echo -ne $DOCKER_TAGS | tr '\n' ' ') \
-            -f <<parameters.docker_file>> \
-            <<parameters.docker_context>>
+
+            export REGISTRY="<<parameters.registry>>"
+            export REPOSITORY="<<parameters.repo>>"
+            export IMAGE_TAGS="<<parameters.docker_tags>>"
+            export GIT_COMMIT="$(git rev-parse HEAD)"
+            export GIT_DATE="$(git show -s --format='%ct')"
+            export GIT_VERSION="<<pipeline.git.tag>>"
+            export PLATFORMS="<<parameters.platforms>>"
+
+            # Create, start (bootstrap) and use a *named* docker builder
+            # This allows us to cross-build multi-platform,
+            # and naming allows us to use the DLC (docker-layer-cache)
+            docker buildx create --driver=docker-container --name=buildx-build --bootstrap --use
+
+            DOCKER_OUTPUT_DESTINATION="--load"
+            # if we are publishing, change the destination
+            if [ "<<parameters.publish>>" == "true" ]; then
+              DOCKER_OUTPUT_DESTINATION="--push"
+              echo "Building for platforms $PLATFORMS and then publishing to registry"
+            else
+              if [[ $PLATFORMS == *,* ]]; then
+                echo "ERROR: cannot perform multi-arch build while also loading the result into regular docker"
+                exit 1
+              else
+                echo "Running single-platform $PLATFORMS build and loading into docker"
+              fi
+            fi
+
+            # Let them cook!
+            docker buildx bake \
+              --progress plain \
+              --builder=buildx-build \
+              -f docker-bake.hcl \
+              $DOCKER_OUTPUT_DESTINATION \
+              <<parameters.docker_name>>
+
+          no_output_timeout: 45m
       - when:
           condition: "<<parameters.save_image_tag>>"
           steps:
@@ -233,130 +264,28 @@ jobs:
                 root: /tmp/docker_images
                 paths:  # only write the one file, to avoid concurrent workspace-file additions
                   - "<<parameters.docker_name>>.tar"
-      - save_cache:
-          name: Save docker build cache
-          key: docker-build-cache
-          paths:
-            - "/tmp/docker-build-cache"
-
-  docker-publish:
-    environment:
-      DOCKER_BUILDKIT: 1
-    parameters:
-      docker_name:
-        description: Docker image name
-        type: string
-      docker_tags:
-        description: Docker image tags as csv
-        type: string
-      docker_file:
-        description: Path to Dockerfile
-        type: string
-      docker_context:
-        description: Docker build context
-        type: string
-        default: "."
-      docker_target:
-        description: "target build stage"
-        type: string
-        default: ""
-      registry:
-        description: Docker registry
-        type: string
-        default: "us-docker.pkg.dev"
-      repo:
-        description: Docker repo
-        type: string
-        default: "oplabs-tools-artifacts/images"
-      platforms:
-        description: Platforms to build for
-        type: string
-        default: "linux/amd64"
-    machine:
-      image: ubuntu-2204:2022.07.1
-      resource_class: medium
-    steps:
-      - gcp-oidc-authenticate
-      # Below is CircleCI recommended way of specifying nameservers on an Ubuntu box:
-      # https://support.circleci.com/hc/en-us/articles/7323511028251-How-to-set-custom-DNS-on-Ubuntu-based-images-using-netplan
-      - run: sudo sed -i '13 i \ \ \ \ \ \ \ \ \ \ \ \ nameservers:' /etc/netplan/50-cloud-init.yaml
-      - run: sudo sed -i '14 i \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ addresses:' /etc/netplan/50-cloud-init.yaml
-      - run: sudo sed -i "s/addresses:/ addresses":" [8.8.8.8, 8.8.4.4] /g" /etc/netplan/50-cloud-init.yaml
-      - run: cat /etc/netplan/50-cloud-init.yaml
-      - run: sudo netplan apply
-      - checkout
-      - run:
-          name: Build & Publish
-          command: |
-            gcloud auth configure-docker <<parameters.registry>>
-            IMAGE_BASE="<<parameters.registry>>/<<parameters.repo>>/<<parameters.docker_name>>"
-            DOCKER_TAGS=$(echo -ne <<parameters.docker_tags>> | sed "s/,/\n/g" | sed "s/[^a-zA-Z0-9\n]/-/g" | sed -e "s|^|-t ${IMAGE_BASE}:|")
-            docker context create buildx-build
-            docker buildx create --use buildx-build
-            docker buildx build --progress plain --platform=<<parameters.platforms>> --target "<<parameters.docker_target>>" --push \
-              $(echo -ne $DOCKER_TAGS | tr '\n' ' ') \
-              -f <<parameters.docker_file>> \
-              <<parameters.docker_context>>
+      - when:
+          condition: "<<parameters.publish>>"
+          steps:
+            - run:
+                name: Publish
+                command: |
+                  gcloud auth configure-docker <<parameters.registry>>
+                  IMAGE_BASE="<<parameters.registry>>/<<parameters.repo>>/<<parameters.docker_name>>"
+                  # tags, without the '-t ' here, so we can loop over them
+                  DOCKER_TAGS=$(echo -ne <<parameters.docker_tags>> | sed "s/,/\n/g" | sed "s/[^a-zA-Z0-9\n]/-/g" | sed -e "s|^|${IMAGE_BASE}:|")
+                  for docker_image_tag in $DOCKER_TAGS; do
+                    docker image push $docker_image_tag
+                  done
+                no_output_timeout: 45m
+      - when:
+          condition: "<<parameters.release>>"
+          steps:
+            - run:
+                name: Tag
+                command: |
+                  ./ops/scripts/ci-docker-tag-op-stack-release.sh <<parameters.registry>>/<<parameters.repo>> $CIRCLE_TAG $CIRCLE_SHA1
 
-  docker-release:
-    environment:
-      DOCKER_BUILDKIT: 1
-    parameters:
-      docker_name:
-        description: Docker image name
-        type: string
-      docker_tags:
-        description: Docker image tags as csv
-        type: string
-      docker_file:
-        description: Path to Dockerfile
-        type: string
-      docker_context:
-        description: Docker build context
-        type: string
-      docker_target:
-        description: "target build stage"
-        type: string
-        default: ""
-      registry:
-        description: Docker registry
-        type: string
-        default: "us-docker.pkg.dev"
-      repo:
-        description: Docker repo
-        type: string
-        default: "oplabs-tools-artifacts/images"
-      platforms:
-        description: Platforms to build for
-        type: string
-        default: "linux/amd64"
-    machine:
-      image: ubuntu-2204:2022.07.1
-      resource_class: medium
-    steps:
-      - gcp-cli/install
-      - gcp-oidc-authenticate
-      - checkout
-      - run:
-          name: Configure Docker
-          command: |
-            gcloud auth configure-docker <<parameters.registry>>
-      - run:
-          name: Build & Publish
-          command: |
-            IMAGE_BASE="<<parameters.registry>>/<<parameters.repo>>/<<parameters.docker_name>>"
-            DOCKER_TAGS=$(echo -ne <<parameters.docker_tags>> | sed "s/,/\n/g" | sed "s/[^a-zA-Z0-9\n]/-/g" | sed -e "s|^|-t ${IMAGE_BASE}:|")
-            docker context create buildx-build
-            docker buildx create --use buildx-build
-            docker buildx build --progress plain --platform=<<parameters.platforms>> --target "<<parameters.docker_target>>" --push \
-              $(echo -ne $DOCKER_TAGS | tr '\n' ' ') \
-              -f <<parameters.docker_file>> \
-              <<parameters.docker_context>>
-          no_output_timeout: 45m
-      - run:
-          name: Tag
-          command: |
-            ./ops/scripts/ci-docker-tag-op-stack-release.sh <<parameters.registry>>/<<parameters.repo>> $CIRCLE_TAG $CIRCLE_SHA1
 
   contracts-bedrock-coverage:
     docker:
@@ -1099,12 +1028,12 @@ jobs:
           command: |
             IMAGE_BASE_PREFIX="us-docker.pkg.dev/oplabs-tools-artifacts/images"
             # Load from previous docker-build job
-            docker load < "/tmp/workspace/op_stack_go.tar"
+            docker load < "/tmp/workspace/op-stack-go.tar"
             docker load < "/tmp/workspace/op-node.tar"
             docker load < "/tmp/workspace/op-proposer.tar"
             docker load < "/tmp/workspace/op-batcher.tar"
             # rename to the tags that the docker-compose of the devnet expects
-            docker tag "$IMAGE_BASE_PREFIX/op_stack_go:<<pipeline.git.revision>>" "$IMAGE_BASE_PREFIX/op_stack_go:devnet"
+            docker tag "$IMAGE_BASE_PREFIX/op-stack-go:<<pipeline.git.revision>>" "$IMAGE_BASE_PREFIX/op-stack-go:devnet"
             docker tag "$IMAGE_BASE_PREFIX/op-node:<<pipeline.git.revision>>" "$IMAGE_BASE_PREFIX/op-node:devnet"
             docker tag "$IMAGE_BASE_PREFIX/op-proposer:<<pipeline.git.revision>>" "$IMAGE_BASE_PREFIX/op-proposer:devnet"
             docker tag "$IMAGE_BASE_PREFIX/op-batcher:<<pipeline.git.revision>>" "$IMAGE_BASE_PREFIX/op-batcher:devnet"
@@ -1473,71 +1402,47 @@ workflows:
             - op-e2e-WS-tests
             - op-e2e-HTTP-tests
             - op-e2e-ext-geth-tests
-      - docker-build:
+      - docker-build: # just to warm up the cache (other jobs run in parallel)
           name: op-stack-go-docker-build
-          docker_file: ops/docker/op-stack-go/Dockerfile
-          docker_name: op_stack_go
+          docker_name: op-stack-go
           docker_tags: <<pipeline.git.revision>>,<<pipeline.git.branch>>
-          docker_context: .
-          build_args: "--build-arg GIT_COMMIT=$(git rev-parse HEAD) --build-arg GIT_DATE=$(git show -s --format='%ct')"
-          save_image_tag: <<pipeline.git.revision>> # other images builds below depend on this image, so we save it to the workspace
+          save_image_tag: <<pipeline.git.revision>> # for devnet later
       - docker-build:
           name: op-node-docker-build
-          docker_file: op-node/Dockerfile
           docker_name: op-node
           docker_tags: <<pipeline.git.revision>>,<<pipeline.git.branch>>
-          load_base_image: "op_stack_go"
-          docker_context: .
           requires: ['op-stack-go-docker-build']
-          build_args: --build-arg OP_STACK_GO_BUILDER="$IMAGE_BASE_PREFIX/op_stack_go:<<pipeline.git.revision>>"
           save_image_tag: <<pipeline.git.revision>> # for devnet later
       - docker-build:
           name: op-batcher-docker-build
-          docker_file: op-batcher/Dockerfile
           docker_name: op-batcher
           docker_tags: <<pipeline.git.revision>>,<<pipeline.git.branch>>
-          load_base_image: "op_stack_go"
-          docker_context: .
           requires: ['op-stack-go-docker-build']
-          build_args: --build-arg OP_STACK_GO_BUILDER="$IMAGE_BASE_PREFIX/op_stack_go:<<pipeline.git.revision>>"
           save_image_tag: <<pipeline.git.revision>> # for devnet later
       - docker-build:
           name: op-program-docker-build
-          docker_file: op-program/Dockerfile
           docker_name: op-program
           docker_tags: <<pipeline.git.revision>>,<<pipeline.git.branch>>
-          load_base_image: "op_stack_go"
-          docker_context: .
           requires: ['op-stack-go-docker-build']
-          build_args: --build-arg OP_STACK_GO_BUILDER="$IMAGE_BASE_PREFIX/op_stack_go:<<pipeline.git.revision>>"
+          save_image_tag: <<pipeline.git.revision>> # for devnet later
       - docker-build:
           name: op-proposer-docker-build
-          docker_file: op-proposer/Dockerfile
           docker_name: op-proposer
           docker_tags: <<pipeline.git.revision>>,<<pipeline.git.branch>>
-          load_base_image: "op_stack_go"
-          docker_context: .
           requires: ['op-stack-go-docker-build']
-          build_args: --build-arg OP_STACK_GO_BUILDER="$IMAGE_BASE_PREFIX/op_stack_go:<<pipeline.git.revision>>"
           save_image_tag: <<pipeline.git.revision>> # for devnet later
       - docker-build:
           name: op-challenger-docker-build
-          docker_file: op-challenger/Dockerfile
           docker_name: op-challenger
           docker_tags: <<pipeline.git.revision>>,<<pipeline.git.branch>>
-          load_base_image: "op_stack_go"
-          docker_context: .
           requires: ['op-stack-go-docker-build']
-          build_args: --build-arg OP_STACK_GO_BUILDER="$IMAGE_BASE_PREFIX/op_stack_go:<<pipeline.git.revision>>"
+          save_image_tag: <<pipeline.git.revision>> # for devnet later
       - docker-build:
           name: op-heartbeat-docker-build
-          docker_file: op-heartbeat/Dockerfile
           docker_name: op-heartbeat
           docker_tags: <<pipeline.git.revision>>,<<pipeline.git.branch>>
-          load_base_image: "op_stack_go"
-          docker_context: .
           requires: ['op-stack-go-docker-build']
-          build_args: --build-arg OP_STACK_GO_BUILDER="$IMAGE_BASE_PREFIX/op_stack_go:<<pipeline.git.revision>>"
+          save_image_tag: <<pipeline.git.revision>> # for devnet later
       - cannon-prestate:
           requires: ["op-stack-go-lint"]
       - devnet:
@@ -1550,16 +1455,12 @@ workflows:
             - cannon-prestate
       - docker-build:
           name: indexer-docker-build
-          docker_file: indexer/Dockerfile
           docker_name: indexer
           docker_tags: <<pipeline.git.revision>>,<<pipeline.git.branch>>
-          docker_context: .
       - docker-build:
           name: ufm-metamask-docker-build
-          docker_file: ufm-test-services/metamask/Dockerfile
           docker_name: ufm-metamask
           docker_tags: <<pipeline.git.revision>>,<<pipeline.git.branch>>
-          docker_context: ufm-test-services/metamask
       - check-generated-mocks-op-node
       - check-generated-mocks-op-service
       - cannon-go-lint-and-test
@@ -1577,172 +1478,159 @@ workflows:
               only: /^(proxyd|indexer|ci-builder|ufm-[a-z0-9\-]*|op-[a-z0-9\-]*)\/v.*/
             branches:
               ignore: /.*/
-      - docker-release:
+      - docker-build: # just to warm up the cache (other jobs run in parallel)
+          name: op-stack-go-docker-build-release
+          docker_name: op-stack-go
+          docker_tags: <<pipeline.git.revision>>
+          platforms: "linux/amd64,linux/arm64"
+          requires:
+            - hold
+      - docker-build:
           name: op-heartbeat-release
           filters:
             tags:
               only: /^op-heartbeat\/v.*/
             branches:
               ignore: /.*/
-          docker_file: op-heartbeat/Dockerfile
           docker_name: op-heartbeat
           docker_tags: <<pipeline.git.revision>>,<<pipeline.git.branch>>
-          docker_context: .
+          requires: ['op-stack-go-docker-build-release']
           platforms: "linux/amd64,linux/arm64"
+          publish: true
+          release: true
           context:
             - oplabs-gcr-release
-          requires:
-            - hold
-      - docker-release:
+      - docker-build:
           name: op-node-docker-release
           filters:
             tags:
               only: /^op-node\/v.*/
             branches:
               ignore: /.*/
-          docker_file: op-node/Dockerfile
           docker_name: op-node
           docker_tags: <<pipeline.git.revision>>,<<pipeline.git.branch>>
-          docker_context: .
+          requires: ['op-stack-go-docker-build-release']
           platforms: "linux/amd64,linux/arm64"
+          publish: true
+          release: true
           context:
             - oplabs-gcr-release
-          requires:
-            - hold
-      - docker-release:
+      - docker-build:
           name: op-batcher-docker-release
           filters:
             tags:
               only: /^op-batcher\/v.*/
             branches:
               ignore: /.*/
-          docker_file: op-batcher/Dockerfile
           docker_name: op-batcher
           docker_tags: <<pipeline.git.revision>>,<<pipeline.git.branch>>
-          docker_context: .
+          requires: ['op-stack-go-docker-build-release']
           platforms: "linux/amd64,linux/arm64"
+          publish: true
+          release: true
           context:
             - oplabs-gcr-release
-          requires:
-            - hold
-      - docker-release:
+      - docker-build:
           name: op-proposer-docker-release
           filters:
             tags:
               only: /^op-proposer\/v.*/
             branches:
               ignore: /.*/
-          docker_file: op-proposer/Dockerfile
           docker_name: op-proposer
           docker_tags: <<pipeline.git.revision>>,<<pipeline.git.branch>>
-          docker_context: .
+          requires: ['op-stack-go-docker-build-release']
           platforms: "linux/amd64,linux/arm64"
+          publish: true
+          release: true
           context:
             - oplabs-gcr-release
-          requires:
-            - hold
-      - docker-release:
+      - docker-build:
           name: op-challenger-docker-release
           filters:
             tags:
               only: /^op-challenger\/v.*/
             branches:
               ignore: /.*/
-          docker_file: op-challenger/Dockerfile
           docker_name: op-challenger
           docker_tags: <<pipeline.git.revision>>,<<pipeline.git.branch>>
-          docker_context: .
+          requires: ['op-stack-go-docker-build-release']
           platforms: "linux/amd64,linux/arm64"
+          publish: true
+          release: true
           context:
             - oplabs-gcr-release
-          requires:
-            - hold
       - docker-build:
-          name: op-migrate-docker-release
-          filters:
-            tags:
-              only: /^op-migrate\/v.*/
-            branches:
-              ignore: /.*/
-          docker_file: op-chain-ops/Dockerfile
-          docker_name: op-migrate
-          docker_tags: <<pipeline.git.revision>>,<<pipeline.git.branch>>
-          docker_context: .
-          context:
-            - oplabs-gcr-release
-          requires:
-            - hold
-      - docker-release:
           name: op-ufm-docker-release
           filters:
             tags:
               only: /^op-ufm\/v.*/
             branches:
               ignore: /.*/
-          docker_file: op-ufm/Dockerfile
           docker_name: op-ufm
           docker_tags: <<pipeline.git.revision>>,<<pipeline.git.branch>>
-          docker_context: .
+          publish: true
+          release: true
           context:
             - oplabs-gcr-release
           requires:
             - hold
-      - docker-release:
+      - docker-build:
           name: proxyd-docker-release
           filters:
             tags:
               only: /^proxyd\/v.*/
             branches:
               ignore: /.*/
-          docker_file: proxyd/Dockerfile
           docker_name: proxyd
           docker_tags: <<pipeline.git.revision>>,<<pipeline.git.branch>>
-          docker_context: .
+          publish: true
+          release: true
           context:
             - oplabs-gcr-release
           requires:
             - hold
-      - docker-release:
+      - docker-build:
           name: indexer-docker-release
           filters:
             tags:
               only: /^indexer\/v.*/
             branches:
               ignore: /.*/
-          docker_file: indexer/Dockerfile
           docker_name: indexer
           docker_tags: <<pipeline.git.revision>>,<<pipeline.git.branch>>
-          docker_context: .
+          publish: true
+          release: true
           context:
             - oplabs-gcr-release
           requires:
             - hold
-      - docker-release:
+      - docker-build:
           name: ci-builder-docker-release
           filters:
             tags:
               only: /^ci-builder\/v.*/
             branches:
               ignore: /.*/
-          docker_file: ./ops/docker/ci-builder/Dockerfile
           docker_name: ci-builder
           docker_tags: <<pipeline.git.revision>>,latest
-          docker_context: .
+          publish: true
+          release: true
           context:
             - oplabs-gcr
           requires:
             - hold
-      - docker-release:
+      - docker-build:
           name: ufm-metamask-docker-release
           filters:
             tags:
               only: /^ufm-metamask\/v.*/
             branches:
               ignore: /.*/
-          docker_file: ./ufm-test-services/metamask/Dockerfile
           docker_name: ufm-metamask
           docker_tags: <<pipeline.git.revision>>,latest
-          docker_context: ./ufm-test-services/metamask
+          publish: true
+          release: true
           context:
             - oplabs-gcr
           requires:
@@ -1777,75 +1665,86 @@ workflows:
     when:
       equal: [ build_hourly, <<pipeline.schedule.name>> ]
     jobs:
-      - docker-publish:
+      - docker-build: # just to warm up the cache (other jobs run in parallel)
+          name: op-stack-go-docker-build-publish
+          docker_name: op-stack-go
+          docker_tags: <<pipeline.git.revision>>,<<pipeline.git.branch>>
+          platforms: "linux/amd64,linux/arm64"
+          context:
+            - oplabs-gcr
+      - docker-build:
           name: op-node-docker-publish
           docker_name: op-node
-          docker_file: op-node/Dockerfile
           docker_tags: <<pipeline.git.revision>>,<<pipeline.git.branch>>
+          requires: [ 'op-stack-go-docker-build-publish' ]
+          platforms: "linux/amd64,linux/arm64"
+          publish: true
           context:
             - oplabs-gcr
-          platforms: "linux/amd64,linux/arm64"
-      - docker-publish:
+      - docker-build:
           name: op-batcher-docker-publish
-          docker_file: op-batcher/Dockerfile
           docker_name: op-batcher
           docker_tags: <<pipeline.git.revision>>,<<pipeline.git.branch>>
+          requires: [ 'op-stack-go-docker-build-publish' ]
+          platforms: "linux/amd64,linux/arm64"
+          publish: true
           context:
             - oplabs-gcr
-          platforms: "linux/amd64,linux/arm64"
-      - docker-publish:
+      - docker-build:
           name: op-program-docker-publish
-          docker_file: op-program/Dockerfile
           docker_name: op-program
           docker_tags: <<pipeline.git.revision>>,<<pipeline.git.branch>>
+          requires: [ 'op-stack-go-docker-build-publish' ]
+          platforms: "linux/amd64,linux/arm64"
+          publish: true
           context:
             - oplabs-gcr
-          platforms: "linux/amd64,linux/arm64"
-      - docker-publish:
+      - docker-build:
           name: op-proposer-docker-publish
-          docker_file: op-proposer/Dockerfile
           docker_name: op-proposer
           docker_tags: <<pipeline.git.revision>>,<<pipeline.git.branch>>
+          requires: [ 'op-stack-go-docker-build-publish' ]
+          platforms: "linux/amd64,linux/arm64"
+          publish: true
           context:
             - oplabs-gcr
-          platforms: "linux/amd64,linux/arm64"
-      - docker-publish:
+      - docker-build:
           name: op-challenger-docker-publish
-          docker_file: op-challenger/Dockerfile
           docker_name: op-challenger
           docker_tags: <<pipeline.git.revision>>,<<pipeline.git.branch>>
+          requires: [ 'op-stack-go-docker-build-publish' ]
+          platforms: "linux/amd64,linux/arm64"
+          publish: true
           context:
             - oplabs-gcr
-          platforms: "linux/amd64,linux/arm64"
-      - docker-publish:
+      - docker-build:
           name: op-heartbeat-docker-publish
-          docker_file: op-heartbeat/Dockerfile
           docker_name: op-heartbeat
           docker_tags: <<pipeline.git.revision>>,<<pipeline.git.branch>>
+          requires: [ 'op-stack-go-docker-build-publish' ]
+          platforms: "linux/amd64,linux/arm64"
+          publish: true
           context:
             - oplabs-gcr
-          platforms: "linux/amd64,linux/arm64"
-      - docker-publish:
+      - docker-build:
           name: indexer-docker-publish
-          docker_file: indexer/Dockerfile
           docker_name: indexer
           docker_tags: <<pipeline.git.revision>>,<<pipeline.git.branch>>
+          publish: true
           context:
             - oplabs-gcr
           platforms: "linux/amd64,linux/arm64"
-      - docker-publish:
+      - docker-build:
           name: chain-mon-docker-publish
-          docker_file: ./ops/docker/Dockerfile.packages
           docker_name: chain-mon
           docker_tags: <<pipeline.git.revision>>,<<pipeline.git.branch>>
-          docker_target: wd-mon
+          publish: true
           context:
             - oplabs-gcr
-      - docker-publish:
+      - docker-build:
           name: ufm-metamask-docker-publish
-          docker_file: ufm-test-services/metamask/Dockerfile
           docker_name: ufm-metamask
-          docker_context: ufm-test-services/metamask
           docker_tags: <<pipeline.git.revision>>,<<pipeline.git.branch>>
+          publish: true
           context:
             - oplabs-gcr

Makefile

 COMPOSEFLAGS=-d
 ITESTS_L2_HOST=http://localhost:9545
 BEDROCK_TAGS_REMOTE?=origin
-OP_STACK_GO_BUILDER?=us-docker.pkg.dev/oplabs-tools-artifacts/images/op_stack_go:latest
+OP_STACK_GO_BUILDER?=us-docker.pkg.dev/oplabs-tools-artifacts/images/op-stack-go:latest
 
 build: build-go build-ts
 .PHONY: build
@@ -25,10 +25,15 @@ ci-builder:
 	docker build -t ci-builder -f ops/docker/ci-builder/Dockerfile .
 
 golang-docker:
-	DOCKER_BUILDKIT=1 docker build -t op-stack-go \
-		--build-arg GIT_COMMIT=$$(git rev-parse HEAD) \
-		--build-arg GIT_DATE=$$(git show -s --format='%ct') \
- 		-f ops/docker/op-stack-go/Dockerfile .
+	# We don't use a buildx builder here, and just load directly into regular docker, for convenience.
+	GIT_COMMIT=$$(git rev-parse HEAD) \
+	GIT_DATE=$$(git show -s --format='%ct') \
+	IMAGE_TAGS=$$GIT_COMMIT,latest \
+	docker buildx bake \
+			--progress plain \
+			--load \
+			-f docker-bake.hcl \
+			op-node op-batcher op-proposer op-challenger
 .PHONY: golang-docker
 
 submodules:

docker-bake.hcl

+variable "REGISTRY" {
+  default = "us-docker.pkg.dev"
+}
+
+variable "REPOSITORY" {
+  default = "oplabs-tools-artifacts/images"
+}
+
+variable "GIT_COMMIT" {
+  default = "dev"
+}
+
+variable "GIT_DATE" {
+  default = "0"
+}
+
+variable "GIT_VERSION" {
+  default = "docker"  // original default as set in proxyd file, not used by full go stack, yet
+}
+
+variable "IMAGE_TAGS" {
+  default = "${GIT_COMMIT}" // split by ","
+}
+
+variable "PLATFORMS" {
+  // You can override this as "linux/amd64,linux/arm64".
+  // Only a specify a single platform when `--load` ing into docker.
+  // Multi-platform is supported when outputting to disk or pushing to a registry.
+  // Multi-platform builds can be tested locally with:  --set="*.output=type=image,push=false"
+  default = "linux/amd64"
+}
+
+target "op-stack-go" {
+  dockerfile = "ops/docker/op-stack-go/Dockerfile"
+  context = "."
+  args = {
+    GIT_COMMIT = "${GIT_COMMIT}"
+    GIT_DATE = "${GIT_DATE}"
+  }
+  platforms = split(",", PLATFORMS)
+  tags = [for tag in split(",", IMAGE_TAGS) : "${REGISTRY}/${REPOSITORY}/op-stack-go:${tag}"]
+}
+
+target "op-node" {
+  dockerfile = "Dockerfile"
+  context = "./op-node"
+  args = {
+    OP_STACK_GO_BUILDER = "op-stack-go"
+  }
+  contexts = {
+    op-stack-go: "target:op-stack-go"
+  }
+  platforms = split(",", PLATFORMS)
+  tags = [for tag in split(",", IMAGE_TAGS) : "${REGISTRY}/${REPOSITORY}/op-node:${tag}"]
+}
+
+target "op-batcher" {
+  dockerfile = "Dockerfile"
+  context = "./op-batcher"
+  args = {
+    OP_STACK_GO_BUILDER = "op-stack-go"
+  }
+  contexts = {
+    op-stack-go: "target:op-stack-go"
+  }
+  platforms = split(",", PLATFORMS)
+  tags = [for tag in split(",", IMAGE_TAGS) : "${REGISTRY}/${REPOSITORY}/op-batcher:${tag}"]
+}
+
+target "op-proposer" {
+  dockerfile = "Dockerfile"
+  context = "./op-proposer"
+  args = {
+    OP_STACK_GO_BUILDER = "op-stack-go"
+  }
+  contexts = {
+    op-stack-go: "target:op-stack-go"
+  }
+  platforms = split(",", PLATFORMS)
+  tags = [for tag in split(",", IMAGE_TAGS) : "${REGISTRY}/${REPOSITORY}/op-proposer:${tag}"]
+}
+
+target "op-challenger" {
+  dockerfile = "Dockerfile"
+  context = "./op-challenger"
+  args = {
+    OP_STACK_GO_BUILDER = "op-stack-go"
+  }
+  contexts = {
+    op-stack-go: "target:op-stack-go"
+  }
+  platforms = split(",", PLATFORMS)
+  tags = [for tag in split(",", IMAGE_TAGS) : "${REGISTRY}/${REPOSITORY}/op-challenger:${tag}"]
+}
+
+target "op-heartbeat" {
+  dockerfile = "Dockerfile"
+  context = "./op-heartbeat"
+  args = {
+    OP_STACK_GO_BUILDER = "op-stack-go"
+  }
+  contexts = {
+    op-stack-go: "target:op-stack-go"
+  }
+  platforms = split(",", PLATFORMS)
+  tags = [for tag in split(",", IMAGE_TAGS) : "${REGISTRY}/${REPOSITORY}/op-heartbeat:${tag}"]
+}
+
+target "op-program" {
+  dockerfile = "Dockerfile"
+  context = "./op-program"
+  args = {
+    OP_STACK_GO_BUILDER = "op-stack-go"
+  }
+  contexts = {
+    op-stack-go: "target:op-stack-go"
+  }
+  platforms = split(",", PLATFORMS)
+  tags = [for tag in split(",", IMAGE_TAGS) : "${REGISTRY}/${REPOSITORY}/op-program:${tag}"]
+}
+
+target "proxyd" {
+  dockerfile = "Dockerfile"
+  context = "./proxyd"
+  args = {
+    // proxyd dockerfile has no _ in the args
+    GITCOMMIT = "${GIT_COMMIT}"
+    GITDATE = "${GIT_DATE}"
+    GITVERSION = "${GIT_VERSION}"
+  }
+  platforms = split(",", PLATFORMS)
+  tags = [for tag in split(",", IMAGE_TAGS) : "${REGISTRY}/${REPOSITORY}/proxyd:${tag}"]
+}
+
+target "indexer" {
+  dockerfile = "./indexer/Dockerfile"
+  context = "./"
+  args = {
+    // proxyd dockerfile has no _ in the args
+    GITCOMMIT = "${GIT_COMMIT}"
+    GITDATE = "${GIT_DATE}"
+    GITVERSION = "${GIT_VERSION}"
+  }
+  platforms = split(",", PLATFORMS)
+  tags = [for tag in split(",", IMAGE_TAGS) : "${REGISTRY}/${REPOSITORY}/proxyd:${tag}"]
+}
+
+target "ufm-metamask" {
+  dockerfile = "Dockerfile"
+  context = "./ufm-test-services/metamask"
+  args = {
+    // proxyd dockerfile has no _ in the args
+    GITCOMMIT = "${GIT_COMMIT}"
+    GITDATE = "${GIT_DATE}"
+    GITVERSION = "${GIT_VERSION}"
+  }
+  platforms = split(",", PLATFORMS)
+  tags = [for tag in split(",", IMAGE_TAGS) : "${REGISTRY}/${REPOSITORY}/ufm-metamask:${tag}"]
+}
+
+type "chain-mon" {
+  dockerfile = "./ops/docker/Dockerfile.packages"
+  context = "."
+  args = {
+    // proxyd dockerfile has no _ in the args
+    GITCOMMIT = "${GIT_COMMIT}"
+    GITDATE = "${GIT_DATE}"
+    GITVERSION = "${GIT_VERSION}"
+  }
+  // this is a multi-stage build, where each stage is a possible output target, but wd-mon is all we publish
+  target = "wd-mon"
+  platforms = split(",", PLATFORMS)
+  tags = [for tag in split(",", IMAGE_TAGS) : "${REGISTRY}/${REPOSITORY}/chain-mon:${tag}"]
+}
+
+type "ci-builder" {
+  dockerfile = "Dockerfile"
+  context = "ops/docker/ci-builder"
+  platforms = split(",", PLATFORMS)
+  tags = [for tag in split(",", IMAGE_TAGS) : "${REGISTRY}/${REPOSITORY}/chain-mon:${tag}"]
+}
+
+

op-batcher/Dockerfile

-ARG OP_STACK_GO_BUILDER=us-docker.pkg.dev/oplabs-tools-artifacts/images/op_stack_go:latest
+ARG OP_STACK_GO_BUILDER=us-docker.pkg.dev/oplabs-tools-artifacts/images/op-stack-go:latest
 FROM $OP_STACK_GO_BUILDER as builder
 # See "make golang-docker" and /ops/docker/op-stack-go
 

op-challenger/Dockerfile

-ARG OP_STACK_GO_BUILDER=us-docker.pkg.dev/oplabs-tools-artifacts/images/op_stack_go:latest
+ARG OP_STACK_GO_BUILDER=us-docker.pkg.dev/oplabs-tools-artifacts/images/op-stack-go:latest
 FROM $OP_STACK_GO_BUILDER as builder
 # See "make golang-docker" and /ops/docker/op-stack-go
 

op-heartbeat/Dockerfile

-ARG OP_STACK_GO_BUILDER=us-docker.pkg.dev/oplabs-tools-artifacts/images/op_stack_go:latest
+ARG OP_STACK_GO_BUILDER=us-docker.pkg.dev/oplabs-tools-artifacts/images/op-stack-go:latest
 FROM $OP_STACK_GO_BUILDER as builder
 # See "make golang-docker" and /ops/docker/op-stack-go
 

op-node/Dockerfile

-ARG OP_STACK_GO_BUILDER=us-docker.pkg.dev/oplabs-tools-artifacts/images/op_stack_go:latest
+ARG OP_STACK_GO_BUILDER=us-docker.pkg.dev/oplabs-tools-artifacts/images/op-stack-go:latest
 FROM $OP_STACK_GO_BUILDER as builder
 # See "make golang-docker" and /ops/docker/op-stack-go
 

op-program/Dockerfile

-ARG OP_STACK_GO_BUILDER=us-docker.pkg.dev/oplabs-tools-artifacts/images/op_stack_go:latest
+ARG OP_STACK_GO_BUILDER=us-docker.pkg.dev/oplabs-tools-artifacts/images/op-stack-go:latest
 FROM $OP_STACK_GO_BUILDER as builder
 # See "make golang-docker" and /ops/docker/op-stack-go
 

op-proposer/Dockerfile

-ARG OP_STACK_GO_BUILDER=us-docker.pkg.dev/oplabs-tools-artifacts/images/op_stack_go:latest
+ARG OP_STACK_GO_BUILDER=us-docker.pkg.dev/oplabs-tools-artifacts/images/op-stack-go:latest
 FROM $OP_STACK_GO_BUILDER as builder
 # See "make golang-docker" and /ops/docker/op-stack-go
 

op-wheel/Dockerfile

-ARG OP_STACK_GO_BUILDER=us-docker.pkg.dev/oplabs-tools-artifacts/images/op_stack_go:latest
+ARG OP_STACK_GO_BUILDER=us-docker.pkg.dev/oplabs-tools-artifacts/images/op-stack-go:latest
 FROM $OP_STACK_GO_BUILDER as builder
 # See "make golang-docker" and /ops/docker/op-stack-go
 FROM alpine:3.18

ops-bedrock/docker-compose.yml

@@ -18,7 +18,7 @@ services:
       args:
         GIT_COMMIT: "dev"
         GIT_DATE: "0"
-    image: us-docker.pkg.dev/oplabs-tools-artifacts/images/op_stack_go:devnet
+    image: us-docker.pkg.dev/oplabs-tools-artifacts/images/op-stack-go:devnet
     entrypoint: ["echo", "build complete"]
 
   l1:
@@ -59,7 +59,7 @@ services:
       context: ../
       dockerfile: ./op-node/Dockerfile
       args:
-        OP_STACK_GO_BUILDER: us-docker.pkg.dev/oplabs-tools-artifacts/images/op_stack_go:devnet
+        OP_STACK_GO_BUILDER: us-docker.pkg.dev/oplabs-tools-artifacts/images/op-stack-go:devnet
     image: us-docker.pkg.dev/oplabs-tools-artifacts/images/op-node:devnet
     command: >
       op-node
@@ -107,7 +107,7 @@ services:
       context: ../
       dockerfile: ./op-proposer/Dockerfile
       args:
-        OP_STACK_GO_BUILDER: us-docker.pkg.dev/oplabs-tools-artifacts/images/op_stack_go:devnet
+        OP_STACK_GO_BUILDER: us-docker.pkg.dev/oplabs-tools-artifacts/images/op-stack-go:devnet
     image: us-docker.pkg.dev/oplabs-tools-artifacts/images/op-proposer:devnet
     ports:
       - "6062:6060"
@@ -136,7 +136,7 @@ services:
       context: ../
       dockerfile: ./op-batcher/Dockerfile
       args:
-        OP_STACK_GO_BUILDER: us-docker.pkg.dev/oplabs-tools-artifacts/images/op_stack_go:devnet
+        OP_STACK_GO_BUILDER: us-docker.pkg.dev/oplabs-tools-artifacts/images/op-stack-go:devnet
     image: us-docker.pkg.dev/oplabs-tools-artifacts/images/op-batcher:devnet
     ports:
       - "6061:6060"

proxyd/Dockerfile.ignore

+# ignore everything but proxyd, proxyd defines all its dependencies in the go.mod
+*
+!/proxyd