Implement CI release pipeline (#464)

* chore: rename dockerfiles for convenience

* ci: auto-publish versioned docker images

* fix: generate correct arguments for images

.github/workflows/release.yml

@@ -9,6 +9,14 @@ jobs:
   release:
     name: Release
     runs-on: ubuntu-latest
+    # map the step outputs to job outputs
+    outputs:
+      builder: ${{ steps.packages.outputs.builder }}
+      l2geth: ${{ steps.packages.outputs.l2geth }}
+      batch-submitter: ${{ steps.packages.outputs.batch-submitter }}
+      message-relayer: ${{ steps.packages.outputs.message-relayer }}
+      data-transport-layer: ${{ steps.packages.outputs.data-transport-layer }}
+
     steps:
       - name: Checkout Repo
         uses: actions/checkout@master
@@ -36,10 +44,159 @@ jobs:
       - name: Install Dependencies
         run: yarn
 
-      - name: Create Release Pull Request
+      - name: Publish To NPM or Create Release Pull Request
         uses: changesets/action@master
+        id: changesets
         with:
           publish: yarn release
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           NPM_TOKEN: ${{ secrets.NPM_TOKEN }}
+
+      # Conditional on the release being executed, we unbundle the publishedPackages to specific
+      # job outputs
+      - name: Get version tags from each published version
+        id: packages
+        if: steps.changesets.outputs.published == 'true'
+        run: |
+          node ops/scripts/ci-versions.js ${{ toJSON(steps.changesets.outputs.publishedPackages) }}
+
+  # The below code is duplicated, would be ideal if we could use a matrix with a
+  # key/value being dynamically generated from the `publishedPackages` output
+  # while also allowing for parallelization (i.e. `l2geth` not depending on `builder`)
+  # and all jobs executing in parallel once `builder` is built
+  l2geth:
+    name: Publish L2Geth Version ${{ needs.release.outputs.l2geth }}
+    needs: release
+    if: needs.release.outputs.l2geth != ''
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+
+      - name: Login to Docker Hub
+        uses: docker/login-action@v1
+        with:
+          username: ${{ secrets.DOCKERHUB_ACCESS_TOKEN_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_ACCESS_TOKEN_SECRET }}
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+
+      - name: Publish L2Geth
+        uses: docker/build-push-action@v2
+        with:
+          context: .
+          file: ./ops/docker/Dockerfile.geth
+          push: true
+          tags: ethereumoptimism/l2geth:${{ needs.release.outputs.l2geth }}
+
+  # pushes the base builder image to dockerhub
+  builder:
+    name: Prepare the base builder image for the services
+    needs: release
+    if: needs.release.outputs.builder == 'true'
+    runs-on: ubuntu-latest
+    # we re-output the variables so that the child jobs can access them
+    outputs:
+      batch-submitter: ${{ needs.release.outputs.batch-submitter }}
+      message-relayer: ${{ needs.release.outputs.message-relayer }}
+      data-transport-layer: ${{ needs.release.outputs.data-transport-layer }}
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+
+      - name: Login to Docker Hub
+        uses: docker/login-action@v1
+        with:
+          username: ${{ secrets.DOCKERHUB_ACCESS_TOKEN_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_ACCESS_TOKEN_SECRET }}
+
+      - name: Build and push
+        uses: docker/build-push-action@v2
+        with:
+          context: .
+          file: ./ops/docker/Dockerfile.monorepo
+          push: true
+          tags: ethereumoptimism/builder
+
+  message-relayer:
+    name: Publish Message Relayer Version ${{ needs.builder.outputs.message-relayer }}
+    needs: builder
+    if: needs.builder.outputs.message-relayer != ''
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+
+      - name: Login to Docker Hub
+        uses: docker/login-action@v1
+        with:
+          username: ${{ secrets.DOCKERHUB_ACCESS_TOKEN_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_ACCESS_TOKEN_SECRET }}
+
+      - name: Build and push
+        uses: docker/build-push-action@v2
+        with:
+          context: .
+          file: ./ops/docker/Dockerfile.message-relayer
+          push: true
+          tags: ethereumoptimism/message-relayer:${{ needs.builder.outputs.message-relayer }}
+
+  batch-submitter:
+    name: Publish Batch Submitter Version ${{ needs.builder.outputs.batch-submitter }}
+    needs: builder
+    if: needs.builder.outputs.batch-submitter != ''
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+
+      - name: Login to Docker Hub
+        uses: docker/login-action@v1
+        with:
+          username: ${{ secrets.DOCKERHUB_ACCESS_TOKEN_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_ACCESS_TOKEN_SECRET }}
+
+      - name: Build and push
+        uses: docker/build-push-action@v2
+        with:
+          context: .
+          file: ./ops/docker/Dockerfile.batch-submitter
+          push: true
+          tags: ethereumoptimism/batch-submitter:${{ needs.builder.outputs.batch-submitter }}
+
+  data-transport-layer:
+    name: Publish Data Transport Layer Version ${{ needs.builder.outputs.data-transport-layer }}
+    needs: builder
+    if: needs.builder.outputs.data-transport-layer != ''
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+
+      - name: Login to Docker Hub
+        uses: docker/login-action@v1
+        with:
+          username: ${{ secrets.DOCKERHUB_ACCESS_TOKEN_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_ACCESS_TOKEN_SECRET }}
+
+      - name: Build and push
+        uses: docker/build-push-action@v2
+        with:
+          context: .
+          file: ./ops/docker/Dockerfile.data-transport-layer
+          push: true
+          tags: ethereumoptimism/data-transport-layer:${{ needs.builder.outputs.data-transport-layer }}

ops/docker-compose.yml

@@ -43,7 +43,7 @@ services:
     image: ethereumoptimism/data-transport-layer
     build:
       context: ..
-      dockerfile: ./ops/docker/Dockerfile.dtl
+      dockerfile: ./ops/docker/Dockerfile.data-transport-layer
     # override with the dtl script and the env vars required for it
     entrypoint: ./dtl.sh
     env_file:
@@ -83,10 +83,10 @@ services:
       - ${L2GETH_WS_PORT:-8546}:8546
 
   relayer:
-    image: ethereumoptimism/relayer
+    image: ethereumoptimism/message-relayer
     build:
       context: ..
-      dockerfile: ./ops/docker/Dockerfile.relayer
+      dockerfile: ./ops/docker/Dockerfile.message-relayer
     entrypoint: ./relayer.sh
     environment:
         L1_NODE_WEB3_URL: http://l1_chain:8545
@@ -102,7 +102,7 @@ services:
     image: ethereumoptimism/batch-submitter
     build:
       context: ..
-      dockerfile: ./ops/docker/Dockerfile.batches
+      dockerfile: ./ops/docker/Dockerfile.batch-submitter
     entrypoint: ./batches.sh
     env_file:
         ./envs/batches.env

ops/scripts/ci-versions.js

+const os = require('os')
+
+// this script unbundles the published packages output
+// from changesets action to a key-value pair to be used
+// with our publishing CI workflow
+data = process.argv[2]
+data = JSON.parse(data)
+
+builder = false
+for (const i of data) {
+  const name = i.name.replace("@eth-optimism/", "")
+  if (name !== 'l2geth') {
+    builder = true
+  }
+  const version = i.version
+  process.stdout.write(`::set-output name=${name}::${version}` + os.EOL)
+}
+
+if (builder) {
+  process.stdout.write(`::set-output name=builder::true` + os.EOL)
+}
+