Merge pull request #1559 from mslipper/feature/layer-caching

Introduce Docker layer caching

.github/workflows/ext-test-snx.yml

@@ -5,6 +5,11 @@ on: workflow_dispatch
 jobs:
   integration:
     runs-on: ubuntu-latest
+    services:
+      registry:
+        image: registry:2
+        ports:
+          - 5000:5000
     env:
       DOCKER_BUILDKIT: 1
       COMPOSE_DOCKER_CLI_BUILD: 1
@@ -28,9 +33,23 @@ jobs:
           restore-keys: |
             ${{ runner.os }}-yarn-
 
+      - uses: actions/cache@v2
+        name: Set up layer cache
+        with:
+          path: /tmp/.buildx-cache
+          key: ${{ runner.os }}-buildx-1-${{ github.sha }}
+          restore-keys: |
+            ${{ runner.os }}-buildx-1-
+
+      - uses: docker/setup-buildx-action@master
+        name: Set up Docker Buildx
+        id: buildx
+        with:
+          version: latest
+          driver-opts: image=moby/buildkit:master,network=host
+
       - name: Build the services
-        working-directory: ./ops
-        run: ./scripts/build-ci.sh
+        run: ./ops/scripts/build-ci.sh
 
       - name: Bring the stack up
         working-directory: ./ops
@@ -63,3 +82,8 @@ jobs:
         with:
           name: logs.tgz
           path: ./logs.tgz
+
+      - name: Move cache
+        run: |
+          rm -rf /tmp/.buildx-cache
+          mv /tmp/.buildx-cache-new /tmp/.buildx-cache

.github/workflows/integration.yml

@@ -13,6 +13,11 @@ on:
 jobs:
   integration:
     runs-on: ubuntu-latest
+    services:
+      registry:
+        image: registry:2
+        ports:
+          - 5000:5000
     env:
         DOCKER_BUILDKIT: 1
         COMPOSE_DOCKER_CLI_BUILD: 1
@@ -32,9 +37,23 @@ jobs:
           restore-keys: |
             ${{ runner.os }}-yarn-
 
+      - uses: actions/cache@v2
+        name: Set up layer cache
+        with:
+          path: /tmp/.buildx-cache
+          key: ${{ runner.os }}-buildx-1-${{ github.sha }}
+          restore-keys: |
+            ${{ runner.os }}-buildx-1-
+
+      - uses: docker/setup-buildx-action@master
+        name: Set up Docker Buildx
+        id: buildx
+        with:
+          version: latest
+          driver-opts: image=moby/buildkit:master,network=host
+
       - name: Build the services
-        working-directory: ./ops
-        run: ./scripts/build-ci.sh
+        run: ./ops/scripts/build-ci.sh
 
       - name: Bring the stack up
         working-directory: ./ops
@@ -67,3 +86,11 @@ jobs:
         with:
           name: logs.tgz
           path: ./logs.tgz
+
+      # Needed to address the following bugs:
+      # https://github.com/docker/build-push-action/issues/252
+      # https://github.com/moby/buildkit/issues/1896
+      - name: Move cache
+        run: |
+          rm -rf /tmp/.buildx-cache
+          mv /tmp/.buildx-cache-new /tmp/.buildx-cache

.github/workflows/sync-tests.yml

@@ -5,6 +5,11 @@ on: workflow_dispatch
 jobs:
   integration-sync-test:
     runs-on: ubuntu-latest
+    services:
+      registry:
+        image: registry:2
+        ports:
+          - 5000:5000
     env:
         DOCKER_BUILDKIT: 1
         COMPOSE_DOCKER_CLI_BUILD: 1
@@ -22,9 +27,24 @@ jobs:
           key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }}
           restore-keys: |
             ${{ runner.os }}-yarn-
+
+      - uses: actions/cache@v2
+        name: Set up layer cache
+        with:
+          path: /tmp/.buildx-cache
+          key: ${{ runner.os }}-buildx-1-${{ github.sha }}
+          restore-keys: |
+            ${{ runner.os }}-buildx-1-
+
+      - uses: docker/setup-buildx-action@master
+        name: Set up Docker Buildx
+        id: buildx
+        with:
+          version: latest
+          driver-opts: image=moby/buildkit:master,network=host
+
       - name: Build the services
-        working-directory: ./ops
-        run: ./scripts/build-ci.sh
+        run: ./ops/scripts/build-ci.sh
 
       - name: Bring the stack up
         working-directory: ./ops
@@ -54,3 +74,8 @@ jobs:
         with:
           name: logs.tgz
           path: ./logs.tgz
+
+      - name: Move cache
+        run: |
+          rm -rf /tmp/.buildx-cache
+          mv /tmp/.buildx-cache-new /tmp/.buildx-cache

ops/docker/Dockerfile.batch-submitter

-FROM ethereumoptimism/builder AS builder
+ARG LOCAL_REGISTRY=docker.io
+FROM ${LOCAL_REGISTRY}/ethereumoptimism/builder AS builder
 
 FROM node:14-alpine
 

ops/docker/Dockerfile.data-transport-layer

-FROM ethereumoptimism/builder AS builder
+ARG LOCAL_REGISTRY=docker.io
+FROM ${LOCAL_REGISTRY}/ethereumoptimism/builder AS builder
 
 FROM node:14-alpine
 

ops/docker/Dockerfile.deployer

-FROM ethereumoptimism/builder AS builder
+ARG LOCAL_REGISTRY=docker.io
+FROM ${LOCAL_REGISTRY}/ethereumoptimism/builder AS builder
 
 FROM node:14-alpine
 

ops/docker/Dockerfile.integration-tests

-FROM ethereumoptimism/builder AS builder
+ARG LOCAL_REGISTRY=docker.io
+FROM ${LOCAL_REGISTRY}/ethereumoptimism/builder AS builder
 
 FROM node:14-alpine
 

ops/docker/Dockerfile.message-relayer

-FROM ethereumoptimism/builder AS builder
+ARG LOCAL_REGISTRY=docker.io
+FROM ${LOCAL_REGISTRY}/ethereumoptimism/builder AS builder
 
 FROM node:14-alpine
 

ops/scripts/build-ci.sh

-# build in 2 steps
-function build_images() {
-    docker-compose build --parallel -- builder l2geth l1_chain
-    docker-compose build --parallel -- deployer dtl batch_submitter relayer integration_tests
+# Builds an image using Buildx. Usage:
+# build <name> <tag> <dockerfile> <context>
+function build() {
+    echo "Building $1."
+    echo "Tag: $2"
+    echo "Dockerfile: $3"
+    echo "Context: $4"
+    docker buildx build \
+        --tag "$2" \
+        --build-arg LOCAL_REGISTRY=localhost:5000 \
+        --cache-from "type=local,src=/tmp/.buildx-cache/$1" \
+        --cache-to="type=local,dest=/tmp/.buildx-cache-new/$1" \
+        --file "$3" \
+        --load "$4" \
+        &
 }
 
-function build_dependencies() {
-    yarn
-    yarn build
-}
+# Split across two build stages:
+#
+# 1. Build the builder and everything that doesn't depend on it, then
+# 2. Build everything else.
+#
+# Each individual build is executed in parallel, so we use wait block all builds
+# in each stage are complete.
+
+mkdir -p /tmp/.buildx-cache-new
+docker buildx build --tag "localhost:5000/ethereumoptimism/builder:latest" --cache-from "type=local,src=/tmp/.buildx-cache/builder" --cache-to="type=local,mode=max,dest=/tmp/.buildx-cache-new/builder" --file "./ops/docker/Dockerfile.monorepo" --push . &
+build l2geth "ethereumoptimism/l2geth:latest" "./ops/docker/Dockerfile.geth" .
+build l1chain "ethereumoptimism/hardhat:latest" "./ops/docker/hardhat/Dockerfile" ./ops/docker/hardhat
+
+wait
+
+# BuildX builds everything in a container when docker-container is selected as
+# the backend. Unfortunately, this means that the built image must be pushed
+# then re-pulled in order to make the container accessible to the Docker daemon.
+# We have to use the docker-container backend since the the docker backend does
+# not support cache-from and cache-to.
+docker pull localhost:5000/ethereumoptimism/builder:latest
+
+# Re-tag the local registry version of the builder so that docker-compose and
+# friends can see it.
+docker tag localhost:5000/ethereumoptimism/builder:latest ethereumoptimism/builder:latest
 
-build_images &
-build_dependencies &
+build deployer "ethereumoptimism/deployer:latest" "./ops/docker/Dockerfile.deployer" .
+build dtl "ethereumoptimism/data-transport-layer:latest" "./ops/docker/Dockerfile.data-transport-layer" .
+build batch_submitter "ethereumoptimism/batch-submitter:latest" "./ops/docker/Dockerfile.batch-submitter" .
+build relayer "ethereumoptimism/message-relayer:latest" "./ops/docker/Dockerfile.message-relayer" .
+build integration-tests "ethereumoptimism/integration-tests:latest" "./ops/docker/Dockerfile.integration-tests" .
 
 wait