K3s infra tests (#1061)

.github/workflows/beekeeper.yml

@@ -16,9 +16,19 @@ jobs:
         uses: actions/checkout@v2
         with:
           fetch-depth: 1
+      - name: Cache
+        uses: actions/cache@v2
+        with:
+          path: |
+            /tmp/k3s-v1.19.5+k3s2
+          key: k3s-v1.19.5+k3s2
+      - name: Cache Go modules
+        uses: actions/cache@v2
+        with:
+          path: ~/go/pkg/mod
+          key: ${{ runner.os }}-build-${{ hashFiles('**/go.sum') }}
       - name: Install bee-local
         run: |
-          sudo mv $(command -v brew){,_disable}
           export URL=$(curl -s https://api.github.com/repos/ethersphere/bee-local/releases/latest | jq -r .tarball_url)
           curl -Ls ${URL} -o bee-local.tar.gz
           tar --strip-components=1 --wildcards -xzf bee-local.tar.gz ethersphere-bee-local-*/{beeinfra.sh,helm-values,hack}
@@ -29,23 +39,24 @@ jobs:
           chmod +x beekeeper
       - name: Prepare testing cluster (DNS discovery)
         run: |
-          ./beeinfra.sh prepare --dns-disco --geth
+          printf ${{ secrets.CR_PAT }} | docker login ghcr.io -u bee-worker --password-stdin
+          ./beeinfra.sh prepare --geth --k3s
       - name: Set kube config
         run: |
           mkdir -p ~/.kube
-          cp $(k3d get-kubeconfig --name='k3s-default') ~/.kube/config
+          cp /etc/rancher/k3s/k3s.yaml ~/.kube/config
           echo "kubeconfig: ${HOME}/.kube/config" > ~/.beekeeper.yaml
       - name: Set testing cluster (DNS discovery)
         run: |
           echo -e "127.0.0.10\tregistry.localhost" | sudo tee -a /etc/hosts
           for ((i=0; i<REPLICA; i++)); do echo -e "127.0.1.$((i+1))\tbee-${i}.localhost bee-${i}-debug.localhost"; done | sudo tee -a /etc/hosts
-          ./beeinfra.sh install --dns-disco --local -r "${REPLICA}" --bootnode /dnsaddr/localhost --geth
+          ./beeinfra.sh install --local -r "${REPLICA}" --bootnode /dnsaddr/localhost --geth --k3s
+      - name: Test pingpong
+        id: pingpong-1
+        run: until ./beekeeper check pingpong --api-scheme http --debug-api-scheme http --disable-namespace --debug-api-domain localhost --api-domain localhost --node-count "${REPLICA}"; do echo "waiting for pingpong..."; sleep .3; done
       - name: Test fullconnectivity
         id: fullconnectivity-1
         run: ./beekeeper check fullconnectivity --api-scheme http --debug-api-scheme http --disable-namespace --debug-api-domain localhost --api-domain localhost --node-count "${REPLICA}"
-      - name: Test pingpong
-        id: pingpong-1
-        run: ./beekeeper check pingpong --api-scheme http --debug-api-scheme http --disable-namespace --debug-api-domain localhost --api-domain localhost --node-count "${REPLICA}"
       - name: Test settlements
         id: settlements-1
         run: ./beekeeper check settlements --api-scheme http --debug-api-scheme http --disable-namespace --debug-api-domain localhost --api-domain localhost --node-count "${REPLICA}" --upload-node-count "${REPLICA}" -t 100000
@@ -58,15 +69,12 @@ jobs:
       - name: Test retrieval
         id: retrieval-1
         run: ./beekeeper check retrieval --api-scheme http --debug-api-scheme http --disable-namespace --debug-api-domain localhost --api-domain localhost --node-count "${REPLICA}" --upload-node-count "${REPLICA}" --chunks-per-node 3
-      # - name: Test chunkrepair
-      #   id: chunkrepair-1
-      #   run: ./beekeeper check chunkrepair --api-scheme http --debug-api-scheme http --disable-namespace --debug-api-domain localhost --api-domain localhost --node-count "${REPLICA}"
-      - name: Test manifest
-        id: manifest-1
-        run: ./beekeeper check manifest --api-scheme http --debug-api-scheme http --disable-namespace --debug-api-domain localhost --api-domain localhost --node-count "${REPLICA}"
       - name: Test gc
         id: gc-chunk-1
         run: ./beekeeper check gc --db-capacity 2000 --api-scheme http --debug-api-scheme http --disable-namespace --debug-api-domain localhost --api-domain localhost --node-count "${REPLICA}"
+      - name: Test manifest
+        id: manifest-1
+        run: ./beekeeper check manifest --api-scheme http --debug-api-scheme http --disable-namespace --debug-api-domain localhost --api-domain localhost --node-count "${REPLICA}"
       - name: Test pin chunk
         id: pin-chunk-1
         run: ./beekeeper check pin-chunk --db-capacity 2000 --api-scheme http --debug-api-scheme http --disable-namespace --debug-api-domain localhost --api-domain localhost --node-count "${REPLICA}"
@@ -78,23 +86,23 @@ jobs:
         run: ./beekeeper check pin-remote --api-scheme http --debug-api-scheme http --disable-namespace --debug-api-domain localhost --api-domain localhost --node-count "${REPLICA}"
       - name: Destroy the cluster
         run: |
-          ./beeinfra.sh destroy
+          ./beeinfra.sh uninstall
       - name: Prepare testing cluster (Node connection and clef enabled)
         run: |
-          ./beeinfra.sh prepare --geth --clef
+          ./beeinfra.sh prepare --geth --clef --k3s
       - name: Set kube config
         run: |
           mkdir -p ~/.kube
-          cp $(k3d get-kubeconfig --name='k3s-default') ~/.kube/config
+          cp /etc/rancher/k3s/k3s.yaml ~/.kube/config
       - name: Set testing cluster (Node connection and clef enabled)
         run: |
-          ./beeinfra.sh install --local -r "${REPLICA}" --geth --clef
+          ./beeinfra.sh install --local -r "${REPLICA}" --geth --clef --k3s
+      - name: Test pingpong
+        id: pingpong-2
+        run: until ./beekeeper check pingpong --api-scheme http --debug-api-scheme http --disable-namespace --debug-api-domain localhost --api-domain localhost --node-count "${REPLICA}"; do echo "waiting for pingpong..."; sleep .3; done
       - name: Test fullconnectivity
         id: fullconnectivity-2
         run: ./beekeeper check fullconnectivity --api-scheme http --debug-api-scheme http --disable-namespace --debug-api-domain localhost --api-domain localhost --node-count "${REPLICA}"
-      - name: Test pingpong
-        id: pingpong-2
-        run: ./beekeeper check pingpong --api-scheme http --debug-api-scheme http --disable-namespace --debug-api-domain localhost --api-domain localhost --node-count "${REPLICA}"
       - name: Test settlements
         id: settlements-2
         run: ./beekeeper check settlements --api-scheme http --debug-api-scheme http --disable-namespace --debug-api-domain localhost --api-domain localhost --node-count "${REPLICA}" --upload-node-count "${REPLICA}" -t 100000
@@ -107,9 +115,11 @@ jobs:
       - name: Test retrieval
         id: retrieval-2
         run: ./beekeeper check retrieval --api-scheme http --debug-api-scheme http --disable-namespace --debug-api-domain localhost --api-domain localhost --node-count "${REPLICA}" --upload-node-count "${REPLICA}" --chunks-per-node 3
-      # - name: Test chunkrepair
-      #   id: chunkrepair-2
-      #   run: ./beekeeper check chunkrepair --api-scheme http --debug-api-scheme http --disable-namespace --debug-api-domain localhost --api-domain localhost --node-count "${REPLICA}"
+      - name: Retag Docker image and push for cache
+        if: success()
+        run: |
+          docker tag registry.localhost:5000/ethersphere/bee:latest ghcr.io/ethersphere/bee
+          docker push ghcr.io/ethersphere/bee
       - name: Retag Docker image and push
         if: github.ref == 'refs/heads/master' && success()
         run: |
@@ -132,24 +142,19 @@ jobs:
         run: |
           export FAILED='no-test'
           if ${{ steps.fullconnectivity-1.outcome=='failure' }}; then FAILED=fullconnectivity-1; fi
-          if ${{ steps.pingpong-1.outcome=='failure' }}; then FAILED=pingpong-1; fi
           if ${{ steps.settlements-1.outcome=='failure' }}; then FAILED=settlements-1; fi
           if ${{ steps.pushsync-bytes-1.outcome=='failure' }}; then FAILED=pushsync-bytes-1; fi
           if ${{ steps.pushsync-chunks-1.outcome=='failure' }}; then FAILED=pushsync-chunks-1; fi
           if ${{ steps.retrieval-1.outcome=='failure' }}; then FAILED=retrieval-1; fi
-          # if ${{ steps.chunkrepair-1.outcome=='failure' }}; then FAILED=chunkrepair-1; fi
           if ${{ steps.manifest-1.outcome=='failure' }}; then FAILED=manifest-1; fi
           if ${{ steps.gc-chunk-1.outcome=='failure' }}; then FAILED=gc-chunk-1; fi
           if ${{ steps.pin-chunk-1.outcome=='failure' }}; then FAILED=pin-chunk-1; fi
           if ${{ steps.pin-bytes-1.outcome=='failure' }}; then FAILED=pin-bytes-1; fi
-          if ${{ steps.pin-remote-1.outcome=='failure' }}; then FAILED=pin-remote-1; fi
           if ${{ steps.fullconnectivity-2.outcome=='failure' }}; then FAILED=fullconnectivity-2; fi
-          if ${{ steps.pingpong-2.outcome=='failure' }}; then FAILED=pingpong-2; fi
           if ${{ steps.settlements-2.outcome=='failure' }}; then FAILED=settlements-2; fi
           if ${{ steps.pushsync-bytes-2.outcome=='failure' }}; then FAILED=pushsync-bytes-2; fi
           if ${{ steps.pushsync-chunks-2.outcome=='failure' }}; then FAILED=pushsync-chunks-2; fi
           if ${{ steps.retrieval-2.outcome=='failure' }}; then FAILED=retrieval-2; fi
-          # if ${{ steps.chunkrepair-2.outcome=='failure' }}; then FAILED=chunkrepair-2; fi
           KEYS=$(curl -sSf -X POST https://eu.relay.tunshell.com/api/sessions)
           curl -sSf -X POST -H "Content-Type: application/json" -d "{\"text\": \"**Merge run**\nFailed -> \`${FAILED}\`\nDebug -> \`sh <(curl -sSf https://lets.tunshell.com/init.sh) L $(echo $KEYS | jq -r .peer2_key) \${TUNSHELL_SECRET} eu.relay.tunshell.com\`\"}" https://beehive.ethswarm.org/hooks/${{ secrets.WEBHOOK_KEY }}
           echo "Failed test: ${FAILED}"

.github/workflows/slash-beekeeper.yml

@@ -58,9 +58,6 @@ jobs:
       - name: Test retrieval
         id: retrieval-1
         run: ./beekeeper check retrieval --api-scheme http --debug-api-scheme http --disable-namespace --debug-api-domain localhost --api-domain localhost --node-count "${REPLICA}" --upload-node-count "${REPLICA}" --chunks-per-node 3
-      # - name: Test chunkrepair
-      #   id: chunkrepair-1
-      #   run: ./beekeeper check chunkrepair --api-scheme http --debug-api-scheme http --disable-namespace --debug-api-domain localhost --api-domain localhost --node-count "${REPLICA}"
       - name: Test manifest
         id: manifest-1
         run: ./beekeeper check manifest --api-scheme http --debug-api-scheme http --disable-namespace --debug-api-domain localhost --api-domain localhost --node-count "${REPLICA}"
@@ -73,9 +70,6 @@ jobs:
       - name: Test pin bytes
         id: pin-bytes-1
         run: ./beekeeper check pin-bytes --db-capacity 2000 --api-scheme http --debug-api-scheme http --disable-namespace --debug-api-domain localhost --api-domain localhost --node-count "${REPLICA}"
-      - name: Test pin remote chunks
-        id: pin-remote-1
-        run: ./beekeeper check pin-remote --api-scheme http --debug-api-scheme http --disable-namespace --debug-api-domain localhost --api-domain localhost --node-count "${REPLICA}"
       - name: Destroy the cluster
         run: |
           ./beeinfra.sh destroy
@@ -107,9 +101,18 @@ jobs:
       - name: Test retrieval
         id: retrieval-2
         run: ./beekeeper check retrieval --api-scheme http --debug-api-scheme http --disable-namespace --debug-api-domain localhost --api-domain localhost --node-count "${REPLICA}" --upload-node-count "${REPLICA}" --chunks-per-node 3
-      # - name: Test chunkrepair
-      #   id: chunkrepair-2
-      #   run: ./beekeeper check chunkrepair --api-scheme http --debug-api-scheme http --disable-namespace --debug-api-domain localhost --api-domain localhost --node-count "${REPLICA}"
+      - name: Test manifest
+        id: manifest-2
+        run: ./beekeeper check manifest --api-scheme http --debug-api-scheme http --disable-namespace --debug-api-domain localhost --api-domain localhost --node-count "${REPLICA}"
+      - name: Test gc
+        id: gc-chunk-2
+        run: ./beekeeper check gc --db-capacity 2000 --api-scheme http --debug-api-scheme http --disable-namespace --debug-api-domain localhost --api-domain localhost --node-count "${REPLICA}"
+      - name: Test pin chunk
+        id: pin-chunk-2
+        run: ./beekeeper check pin-chunk --db-capacity 2000 --api-scheme http --debug-api-scheme http --disable-namespace --debug-api-domain localhost --api-domain localhost --node-count "${REPLICA}"
+      - name: Test pin bytes
+        id: pin-bytes-2
+        run: ./beekeeper check pin-bytes --db-capacity 2000 --api-scheme http --debug-api-scheme http --disable-namespace --debug-api-domain localhost --api-domain localhost --node-count "${REPLICA}"
       - name: Debug workflow if failed
         if: failure()
         run: |
@@ -125,7 +128,6 @@ jobs:
           if ${{ steps.gc-chunk-1.outcome=='failure' }}; then FAILED=gc-chunk-1; fi
           if ${{ steps.pin-chunk-1.outcome=='failure' }}; then FAILED=pin-chunk-1; fi
           if ${{ steps.pin-bytes-1.outcome=='failure' }}; then FAILED=pin-bytes-1; fi
-          if ${{ steps.pin-remote-1.outcome=='failure' }}; then FAILED=pin-remote-1; fi
           if ${{ steps.fullconnectivity-2.outcome=='failure' }}; then FAILED=fullconnectivity-2; fi
           if ${{ steps.pingpong-2.outcome=='failure' }}; then FAILED=pingpong-2; fi
           if ${{ steps.settlements-2.outcome=='failure' }}; then FAILED=settlements-2; fi
@@ -133,6 +135,10 @@ jobs:
           if ${{ steps.pushsync-chunks-2.outcome=='failure' }}; then FAILED=pushsync-chunks-2; fi
           if ${{ steps.retrieval-2.outcome=='failure' }}; then FAILED=retrieval-2; fi
           # if ${{ steps.chunkrepair-2.outcome=='failure' }}; then FAILED=chunkrepair-2; fi
+          if ${{ steps.manifest-2.outcome=='failure' }}; then FAILED=manifest-2; fi
+          if ${{ steps.gc-chunk-2.outcome=='failure' }}; then FAILED=gc-chunk-2; fi
+          if ${{ steps.pin-chunk-2.outcome=='failure' }}; then FAILED=pin-chunk-2; fi
+          if ${{ steps.pin-bytes-2.outcome=='failure' }}; then FAILED=pin-bytes-2; fi
           KEYS=$(curl -sSf -X POST https://eu.relay.tunshell.com/api/sessions)
           curl -sSf -X POST -H "Content-Type: application/json" -d "{\"text\": \"**PR run** - > \`${{ github.event.client_payload.ref }}\`\nFailed -> \`${FAILED}\`\nDebug -> \`sh <(curl -sSf https://lets.tunshell.com/init.sh) L $(echo $KEYS | jq -r .peer2_key) \${TUNSHELL_SECRET} eu.relay.tunshell.com\`\"}" https://beehive.ethswarm.org/hooks/${{ secrets.WEBHOOK_KEY }}
           echo "Failed test: ${FAILED}"

Dockerfile.goreleaser

+FROM debian:10.2-slim
+
+ENV DEBIAN_FRONTEND noninteractive
+
+RUN apt-get update && apt-get install -y \
+        ca-certificates; \
+    apt-get clean; \
+    rm -rf /var/lib/apt/lists/*; \
+    groupadd -r bee --gid 999; \
+    useradd -r -g bee --uid 999 --no-log-init -m bee;
+
+# make sure mounted volumes have correct permissions
+RUN mkdir -p /home/bee/.bee && chown 999:999 /home/bee/.bee
+
+COPY bee /usr/local/bin/bee
+
+EXPOSE 1633 1634 1635
+USER bee
+WORKDIR /home/bee
+VOLUME /home/bee/.bee
+
+ENTRYPOINT ["bee"]