feat: makefile beekeeper (#1931)

b7e8a354 · Ivan Vandot · GitHub · 27205bb6 · b7e8a354 · b7e8a354
Commit b7e8a354 authored Jun 02, 2021 by Ivan Vandot Committed by GitHub Jun 02, 2021
Show whitespace changes
Inline Side-by-side

Showing with 55 additions and 46 deletions

beekeeper.yml .github/workflows/beekeeper.yml +36 -44

Makefile Makefile +19 -2

No files found.
--- a/.github/workflows/beekeeper.yml
+++ b/.github/workflows/beekeeper.yml
@@ -37,115 +37,106 @@ jobs:
        with:
          path: ~/go/pkg/mod
          key: ${{ runner.os }}-build-${{ hashFiles('**/go.sum') }}
-      - name: Install bee-local
-        run: |
-          export URL=$(curl -s https://api.github.com/repos/ethersphere/bee-local/releases/latest | jq -r .tarball_url)
-          curl -Ls ${URL} -o bee-local.tar.gz
-          tar --strip-components=1 --wildcards -xzf bee-local.tar.gz ethersphere-bee-local-*/{beeinfra.sh,helm-values,hack}
-      - name: Install latest beekeeper
-        run: |
-          export TAG=$(curl -s https://api.github.com/repos/ethersphere/beekeeper/releases/latest | jq -r .tag_name)
-          curl -Ls https://github.com/ethersphere/beekeeper/releases/download/${TAG}/beekeeper-linux-amd64 -o beekeeper
-          chmod +x beekeeper
-      - name: Prepare testing cluster (DNS discovery)
+      - name: Prepare local cluster
        run: |
          printf ${{ secrets.CR_PAT }} | docker login ghcr.io -u bee-worker --password-stdin
-          timeout 30m ./beeinfra.sh prepare --geth --k3s
+          make beekeeper BEEKEEPER_INSTALL_DIR=/usr/local/bin BEEKEEPER_USE_SUDO=true
+          timeout 10m make beelocal OPTS='ci skip-vet'
      - name: Set kube config
        run: |
          mkdir -p ~/.kube
          cp /etc/rancher/k3s/k3s.yaml ~/.kube/config
-          echo "kubeconfig: ${HOME}/.kube/config" > ~/.beekeeper.yaml
      - name: Apply patches
        run: |
          patch pkg/postage/batchstore/reserve.go .github/patches/postagereserve.patch
          patch pkg/postage/postagecontract/contract.go .github/patches/postagecontract.patch
          patch pkg/postage/listener/listener.go .github/patches/listener.patch
-      - name: Set testing cluster (DNS discovery)
+      - name: Set local cluster
        run: |
-          echo -e "127.0.0.10\tregistry.localhost" | sudo tee -a /etc/hosts
-          for ((i=0; i<REPLICA; i++)); do echo -e "127.0.1.$((i+1))\tbee-${i}.localhost bee-${i}-debug.localhost"; done | sudo tee -a /etc/hosts
-          timeout 30m ./beeinfra.sh install --local -r "${REPLICA}" --bootnode /dnsaddr/localhost --geth --k3s --pay-threshold 1500000000000 --postage
+          make beelocal ACTION=add-hosts
+          timeout 10m make deploylocal BEEKEEPER_CLUSTER=local-dns
      - name: Test pingpong
        id: pingpong-1
-        run: until ./beekeeper check pingpong --api-scheme http --debug-api-scheme http --disable-namespace --debug-api-domain localhost --api-domain localhost --node-count "${REPLICA}"; do echo "waiting for pingpong..."; sleep .3; done
+        run: until beekeeper check --cluster-name local-dns --checks ci-pingpong; do echo "waiting for pingpong..."; sleep .3; done
      - name: Test fullconnectivity
        id: fullconnectivity-1
-        run: ./beekeeper check fullconnectivity --api-scheme http --debug-api-scheme http --disable-namespace --debug-api-domain localhost --api-domain localhost --node-count "${REPLICA}" 
+        run: beekeeper check --cluster-name local-dns --checks=ci-full-connectivity
      - name: Test settlements
        id: settlements-1
-        run: |
-          ./beekeeper check settlements --api-scheme http --debug-api-scheme http --disable-namespace --debug-api-domain localhost --api-domain localhost --node-count "${REPLICA}" --upload-node-count "${REPLICA}" -t 1500000000000
-          sleep 2
+        run: beekeeper check --cluster-name local-dns --checks=ci-settlements
+      - name: Sleep for time allowance to replenish
+        run: sleep 2
      - name: Test pss
        id: pss
-        run: ./beekeeper check pss --api-scheme http --debug-api-scheme http --disable-namespace --debug-api-domain localhost --api-domain localhost --node-count "${REPLICA}" --timeout 5m
+        run: beekeeper check --cluster-name local-dns --checks=ci-pss
      - name: Test soc
        id: soc
-        run: ./beekeeper check soc --api-scheme http --debug-api-scheme http --disable-namespace --debug-api-domain localhost --api-domain localhost --node-count "${REPLICA}"
+        run: beekeeper check --cluster-name local-dns --checks=ci-soc
      - name: Test pushsync (chunks)
        id: pushsync-chunks-1
-        run: ./beekeeper check pushsync --api-scheme http --debug-api-scheme http --disable-namespace --debug-api-domain localhost --api-domain localhost --node-count "${REPLICA}" --upload-node-count "${REPLICA}" --chunks-per-node 3 --upload-chunks --retry-delay 15s
+        run: beekeeper check --cluster-name local-dns --checks=ci-pushsync-chunks
      - name: Test retrieval
        id: retrieval-1
-        run: ./beekeeper check retrieval --api-scheme http --debug-api-scheme http --disable-namespace --debug-api-domain localhost --api-domain localhost --node-count "${REPLICA}" --upload-node-count "${REPLICA}" --chunks-per-node 3
+        run: beekeeper check --cluster-name local-dns --checks=ci-retrieval
      - name: Test manifest
        id: manifest-1
-        run: ./beekeeper check manifest --api-scheme http --debug-api-scheme http --disable-namespace --debug-api-domain localhost --api-domain localhost --node-count "${REPLICA}"
+        run: beekeeper check --cluster-name local-dns --checks=ci-manifest
      - name: Destroy the cluster
        run: |
-          ./beeinfra.sh uninstall
+          beekeeper delete bee-cluster --cluster-name local-dns
+          make beelocal ACTION=uninstall
      - name: Prepare testing cluster (Node connection and clef enabled)
        run: |
-          timeout 30m ./beeinfra.sh prepare --geth --clef --k3s
+            timeout 10m make beelocal OPTS='ci skip-vet'
      - name: Set kube config
        run: |
          mkdir -p ~/.kube
          cp /etc/rancher/k3s/k3s.yaml ~/.kube/config
      - name: Set testing cluster (Node connection and clef enabled)
        run: |
-          timeout 30m ./beeinfra.sh install --local -r "${REPLICA}" --geth --clef --k3s --pay-threshold 1500000000000 --postage
+          timeout 10m make deploylocal BEEKEEPER_CLUSTER=local-clef
      - name: Test pingpong
        id: pingpong-2
-        run: until ./beekeeper check pingpong --api-scheme http --debug-api-scheme http --disable-namespace --debug-api-domain localhost --api-domain localhost --node-count "${REPLICA}"; do echo "waiting for pingpong..."; sleep .3; done
+        run: until beekeeper check --cluster-name local-clef --checks ci-pingpong; do echo "waiting for pingpong..."; sleep .3; done
      - name: Test fullconnectivity
        id: fullconnectivity-2
-        run: ./beekeeper check fullconnectivity --api-scheme http --debug-api-scheme http --disable-namespace --debug-api-domain localhost --api-domain localhost --node-count "${REPLICA}"
+        run: beekeeper check --cluster-name local-clef --checks=ci-full-connectivity
      - name: Test settlements
        id: settlements-2
-        run: ./beekeeper check settlements --api-scheme http --debug-api-scheme http --disable-namespace --debug-api-domain localhost --api-domain localhost --node-count "${REPLICA}" --upload-node-count "${REPLICA}" -t 1500000000000
+        run: beekeeper check --cluster-name local-clef --checks=ci-settlements-clef
      - name: Destroy the cluster
        run: |
-          ./beeinfra.sh uninstall
+          beekeeper delete bee-cluster --cluster-name local-clef
+          make beelocal ACTION=uninstall
      - name: Apply patches
        run: |
          patch pkg/postage/batchstore/reserve.go .github/patches/postagereserve_gc.patch
      - name: Prepare testing cluster (storage incentives setup)
        run: |
-          timeout 10m ./beeinfra.sh prepare --geth --k3s
+          timeout 10m make beelocal OPTS='ci skip-vet'
      - name: Set kube config
        run: |
          mkdir -p ~/.kube
          cp /etc/rancher/k3s/k3s.yaml ~/.kube/config
      - name: Set testing cluster (storage incentives setup)
        run: |
-          timeout 10m ./beeinfra.sh install --local -r "${REPLICA}" --geth --k3s --pay-threshold 1500000000000 --postage --db-capacity 100
+          timeout 10m make deploylocal BEEKEEPER_CLUSTER=local-gc
+      - name: Test pingpong
+        id: pingpong-3
+        run: until beekeeper check --cluster-name local-gc --checks ci-pingpong; do echo "waiting for pingpong..."; sleep .3; done
      - name: Test gc
        id: gc-chunk-1
-        run: ./beekeeper check gc --cache-capacity 100 --api-scheme http --debug-api-scheme http --disable-namespace --debug-api-domain localhost --api-domain localhost --node-count "${REPLICA}"
-      - name: Destroy the cluster
-        run: |
-          ./beeinfra.sh uninstall
+        run: beekeeper check --cluster-name local-gc --checks=ci-gc
      - name: Retag Docker image and push for cache
        if: success()
        run: |
-          docker tag registry.localhost:5000/ethersphere/bee:latest ghcr.io/ethersphere/bee
+          docker tag k3d-registry.localhost:5000/ethersphere/bee:latest ghcr.io/ethersphere/bee
          docker push ghcr.io/ethersphere/bee
      - name: Retag Docker image and push
        if: github.ref == 'refs/heads/master' && github.event.action != 'beekeeper' && success()
        run: |
-          docker tag registry.localhost:5000/ethersphere/bee:latest ethersphere/bee:latest
-          docker tag registry.localhost:5000/ethersphere/bee:latest quay.io/ethersphere/bee:latest
+          docker tag k3d-registry.localhost:5000/ethersphere/bee:latest ethersphere/bee:latest
+          docker tag k3d-registry.localhost:5000/ethersphere/bee:latest quay.io/ethersphere/bee:latest
          printf ${{ secrets.DOCKERHUB_PASSWORD }} | docker login --username ${{ secrets.DOCKERHUB_USERNAME }} --password-stdin
          printf ${{ secrets.QUAY_PASSWORD }} | docker login --username ${{ secrets.QUAY_USERNAME }} quay.io --password-stdin
          docker push ethersphere/bee:latest
@@ -172,12 +163,13 @@ jobs:
          if ${{ steps.pushsync-chunks-1.outcome=='failure' }}; then FAILED=pushsync-chunks-1; fi
          if ${{ steps.retrieval-1.outcome=='failure' }}; then FAILED=retrieval-1; fi
          if ${{ steps.manifest-1.outcome=='failure' }}; then FAILED=manifest-1; fi
-          if ${{ steps.gc-chunk-1.outcome=='failure' }}; then FAILED=gc-chunk-1; fi
          if ${{ steps.pingpong-2.outcome=='failure' }}; then FAILED=pingpong-2; fi
          if ${{ steps.fullconnectivity-2.outcome=='failure' }}; then FAILED=fullconnectivity-2; fi
          if ${{ steps.settlements-2.outcome=='failure' }}; then FAILED=settlements-2; fi
          if ${{ steps.pss.outcome=='failure' }}; then FAILED=pss; fi
          if ${{ steps.soc.outcome=='failure' }}; then FAILED=soc; fi
+          if ${{ steps.pingpong-3.outcome=='failure' }}; then FAILED=pingpong-3; fi
+          if ${{ steps.gc-chunk-1.outcome=='failure' }}; then FAILED=gc-chunk-1; fi
          KEYS=$(curl -sSf -X POST https://eu.relay.tunshell.com/api/sessions)
          curl -sSf -X POST -H "Content-Type: application/json" -d "{\"text\": \"**${RUN_TYPE}** ${{ github.head_ref }}\nFailed -> \`${FAILED}\`\nDebug -> \`sh <(curl -sSf https://lets.tunshell.com/init.sh) L $(echo $KEYS | jq -r .peer2_key) \${TUNSHELL_SECRET} eu.relay.tunshell.com\`\"}" https://beehive.ethswarm.org/hooks/${{ secrets.WEBHOOK_KEY }}
          echo "Failed test: ${FAILED}"

--- a/Makefile
+++ b/Makefile
@@ -3,8 +3,11 @@ GOLANGCI_LINT ?= $$($(GO) env GOPATH)/bin/golangci-lint
 GOLANGCI_LINT_VERSION ?= v1.30.0
 GOGOPROTOBUF ?= protoc-gen-gogofaster
 GOGOPROTOBUF_VERSION ?= v1.3.1
-BEEKEEPER ?= $$($(GO) env GOPATH)/bin/beekeeper
+BEEKEEPER_INSTALL_DIR ?= $$($(GO) env GOPATH)/bin
+BEEKEEPER_USE_SUDO ?= false
+BEEKEEPER_CLUSTER ?= local
 BEELOCAL_BRANCH ?= main
+BEEKEEPER_BRANCH ?= master

 COMMIT ?= "$(shell git describe --long --dirty --always --match "" || true)"
 LDFLAGS ?= -s -w -X github.com/ethersphere/bee.commit="$(COMMIT)"
@@ -23,12 +26,26 @@ dist:

 .PHONY: beekeeper
 beekeeper:
-	test -f $(BEEKEEPER) || curl -sSfL https://raw.githubusercontent.com/ethersphere/beekeeper/master/install.sh | BEEKEEPER_INSTALL_DIR=$$($(GO) env GOPATH)/bin USE_SUDO=false bash
+	curl -sSfL https://raw.githubusercontent.com/ethersphere/beekeeper/master/scripts/install.sh | BEEKEEPER_INSTALL_DIR=$(BEEKEEPER_INSTALL_DIR) USE_SUDO=$(BEEKEEPER_USE_SUDO) bash
+	test -f ~/.beekeeper.yaml || curl -sSfL https://raw.githubusercontent.com/ethersphere/beekeeper/$(BEEKEEPER_BRANCH)/config/beekeeper-local.yaml -o ~/.beekeeper.yaml
+	mkdir -p ~/.beekeeper && curl -sSfL https://raw.githubusercontent.com/ethersphere/beekeeper/$(BEEKEEPER_BRANCH)/config/local.yaml -o ~/.beekeeper/local.yaml

 .PHONY: beelocal
 beelocal:
 	curl -sSfL https://raw.githubusercontent.com/ethersphere/beelocal/$(BEELOCAL_BRANCH)/beelocal.sh | bash

+.PHONY: deploylocal
+deploylocal:
+	beekeeper create bee-cluster --cluster-name $(BEEKEEPER_CLUSTER)
+
+.PHONY: testlocal
+testlocal:
+	export PATH=${PATH}:$$($(GO) env GOPATH)/bin
+	beekeeper check --cluster-name local --checks=ci-full-connectivity,ci-gc,ci-manifest,ci-pingpong,ci-pss,ci-pushsync-chunks,ci-retrieval,ci-settlements,ci-soc
+
+.PHONY: testlocal-all
+all: beekeeper beelocal deploylocal testlocal
+
 .PHONY: lint
 lint: linter
 	$(GOLANGCI_LINT) run