fix: shutdown node on errors from blockchain endpoint (#1868)

pkg/node/node.go

@@ -18,8 +18,10 @@ import (
 	"math/big"
 	"net"
 	"net/http"
+	"os"
 	"path/filepath"
 	"strings"
+	"syscall"
 	"time"
 
 	"github.com/ethereum/go-ethereum/common"
@@ -365,7 +367,7 @@ func NewBee(addr string, swarmAddress swarm.Address, publicKey ecdsa.PublicKey,
 			postageSyncStart = startBlock
 		}
 
-		eventListener = listener.New(logger, swapBackend, postageContractAddress, o.BlockTime)
+		eventListener = listener.New(logger, swapBackend, postageContractAddress, o.BlockTime, &pidKiller{node: b})
 		b.listenerCloser = eventListener
 
 		batchSvc = batchservice.New(stateStore, batchStore, logger, eventListener)
@@ -791,3 +793,26 @@ func getTxHash(stateStore storage.StateStorer, logger logging.Logger, o Options)
 	logger.Infof("using the chequebook transaction hash %x", txHash)
 	return txHash.Bytes(), nil
 }
+
+// pidKiller is used to issue a forced shut down of the node from sub modules. The issue with using the
+// node's Shutdown method is that it only shuts down the node and does not exit the start process
+// which is waiting on the os.Signals. This is not desirable, but currently bee node cannot handle
+// rate-limiting blockchain API calls properly. We will shut down the node in this case to allow the
+// user to rectify the API issues (by adjusting limits or using a different one). There is no platform
+// agnostic way to trigger os.Signals in go unfortunately. Which is why we will use the process.Kill
+// approach which works on windows as well.
+type pidKiller struct {
+	node *Bee
+}
+
+func (p *pidKiller) Shutdown(ctx context.Context) error {
+	err := p.node.Shutdown(ctx)
+	if err != nil {
+		return err
+	}
+	ps, err := os.FindProcess(syscall.Getpid())
+	if err != nil {
+		return err
+	}
+	return ps.Kill()
+}

pkg/postage/listener/listener.go

@@ -47,6 +47,12 @@ type BlockHeightContractFilterer interface {
 	BlockNumber(context.Context) (uint64, error)
 }
 
+// Shutdowner interface is passed to the listener to shutdown the node if we hit
+// error while listening for blockchain events.
+type Shutdowner interface {
+	Shutdown(context.Context) error
+}
+
 type listener struct {
 	logger    logging.Logger
 	ev        BlockHeightContractFilterer
@@ -56,6 +62,7 @@ type listener struct {
 	quit                chan struct{}
 	wg                  sync.WaitGroup
 	metrics             metrics
+	shutdowner          Shutdowner
 }
 
 func New(
@@ -63,6 +70,7 @@ func New(
 	ev BlockHeightContractFilterer,
 	postageStampAddress common.Address,
 	blockTime uint64,
+	shutdowner Shutdowner,
 ) postage.Listener {
 	return &listener{
 		logger:              logger,
@@ -71,6 +79,7 @@ func New(
 		postageStampAddress: postageStampAddress,
 		quit:                make(chan struct{}),
 		metrics:             newMetrics(),
+		shutdowner:          shutdowner,
 	}
 }
 
@@ -243,7 +252,13 @@ func (l *listener) Listen(from uint64, updater postage.EventUpdater) <-chan stru
 	go func() {
 		err := listenf()
 		if err != nil {
-			l.logger.Errorf("event listener sync: %v", err)
+			l.logger.Errorf("failed syncing event listener, shutting down node err: %v", err)
+			if l.shutdowner != nil {
+				err = l.shutdowner.Shutdown(context.Background())
+				if err != nil {
+					l.logger.Errorf("failed shutting down node: %v", err)
+				}
+			}
 		}
 	}()
 

pkg/postage/listener/listener_test.go

@@ -7,8 +7,10 @@ package listener_test
 import (
 	"bytes"
 	"context"
+	"errors"
 	"io/ioutil"
 	"math/big"
+	"sync"
 	"testing"
 	"time"
 
@@ -45,7 +47,7 @@ func TestListener(t *testing.T) {
 				c.toLog(496),
 			),
 		)
-		l := listener.New(logger, mf, postageStampAddress, 1)
+		l := listener.New(logger, mf, postageStampAddress, 1, nil)
 		l.Listen(0, ev)
 
 		select {
@@ -76,7 +78,7 @@ func TestListener(t *testing.T) {
 				topup.toLog(496),
 			),
 		)
-		l := listener.New(logger, mf, postageStampAddress, 1)
+		l := listener.New(logger, mf, postageStampAddress, 1, nil)
 		l.Listen(0, ev)
 
 		select {
@@ -107,7 +109,7 @@ func TestListener(t *testing.T) {
 				depthIncrease.toLog(496),
 			),
 		)
-		l := listener.New(logger, mf, postageStampAddress, 1)
+		l := listener.New(logger, mf, postageStampAddress, 1, nil)
 		l.Listen(0, ev)
 
 		select {
@@ -136,7 +138,7 @@ func TestListener(t *testing.T) {
 				priceUpdate.toLog(496),
 			),
 		)
-		l := listener.New(logger, mf, postageStampAddress, 1)
+		l := listener.New(logger, mf, postageStampAddress, 1, nil)
 		l.Listen(0, ev)
 		select {
 		case e := <-evC:
@@ -188,7 +190,7 @@ func TestListener(t *testing.T) {
 			),
 			WithBlockNumber(blockNumber),
 		)
-		l := listener.New(logger, mf, postageStampAddress, 1)
+		l := listener.New(logger, mf, postageStampAddress, 1, nil)
 		l.Listen(0, ev)
 
 		select {
@@ -250,6 +252,47 @@ func TestListener(t *testing.T) {
 			t.Fatal("timed out waiting for block number update")
 		}
 	})
+
+	t.Run("shutdown on error event", func(t *testing.T) {
+		shutdowner := &countShutdowner{}
+		ev, _ := newEventUpdaterMock()
+		mf := newMockFilterer(
+			WithBlockNumberError(errors.New("dummy error")),
+		)
+		l := listener.New(logger, mf, postageStampAddress, 1, shutdowner)
+		l.Listen(0, ev)
+
+		start := time.Now()
+		for {
+			time.Sleep(time.Millisecond * 100)
+			if shutdowner.NoOfCalls() == 1 {
+				break
+			}
+			if time.Since(start) > time.Second*5 {
+				t.Fatal("expected shutdown call by now")
+			}
+		}
+	})
+}
+
+type countShutdowner struct {
+	mtx           sync.Mutex
+	shutdownCalls int
+}
+
+func (c *countShutdowner) NoOfCalls() int {
+	c.mtx.Lock()
+	defer c.mtx.Unlock()
+
+	return c.shutdownCalls
+}
+
+func (c *countShutdowner) Shutdown(_ context.Context) error {
+	c.mtx.Lock()
+	defer c.mtx.Unlock()
+
+	c.shutdownCalls++
+	return nil
 }
 
 func newEventUpdaterMock() (*updater, chan interface{}) {
@@ -309,6 +352,7 @@ type mockFilterer struct {
 	subscriptionEvents []types.Log
 	sub                *sub
 	blockNumber        uint64
+	blockNumberError   error
 }
 
 func newMockFilterer(opts ...Option) *mockFilterer {
@@ -333,6 +377,12 @@ func WithBlockNumber(blockNumber uint64) Option {
 	})
 }
 
+func WithBlockNumberError(err error) Option {
+	return optionFunc(func(s *mockFilterer) {
+		s.blockNumberError = err
+	})
+}
+
 func (m *mockFilterer) FilterLogs(ctx context.Context, query ethereum.FilterQuery) ([]types.Log, error) {
 	return m.filterLogEvents, nil
 }
@@ -352,6 +402,9 @@ func (m *mockFilterer) Close() {
 }
 
 func (m *mockFilterer) BlockNumber(context.Context) (uint64, error) {
+	if m.blockNumberError != nil {
+		return 0, m.blockNumberError
+	}
 	return m.blockNumber, nil
 }