Clean addressbook after failed connection attempts (#309)

* Forget about non connectable peers in kademlia

pkg/addressbook/addressbook.go

@@ -19,6 +19,7 @@ var _ Interface = (*store)(nil)
 
 type Interface interface {
 	GetPutter
+	Remover
 	Overlays() ([]swarm.Address, error)
 	Addresses() ([]bzz.Address, error)
 }
@@ -36,6 +37,10 @@ type Putter interface {
 	Put(overlay swarm.Address, addr bzz.Address) (err error)
 }
 
+type Remover interface {
+	Remove(overlay swarm.Address) error
+}
+
 type store struct {
 	store storage.StateStorer
 }
@@ -61,6 +66,10 @@ func (s *store) Put(overlay swarm.Address, addr bzz.Address) (err error) {
 	return s.store.Put(key, &addr)
 }
 
+func (s *store) Remove(overlay swarm.Address) error {
+	return s.store.Delete(keyPrefix + overlay.String())
+}
+
 func (s *store) Overlays() (overlays []swarm.Address, err error) {
 	err = s.store.Iterate(keyPrefix, func(key, _ []byte) (stop bool, err error) {
 		k := string(key)

pkg/kademlia/kademlia.go

@@ -23,8 +23,9 @@ import (
 )
 
 const (
-	maxBins        = 16
-	nnLowWatermark = 2 // the number of peers in consecutive deepest bins that constitute as nearest neighbours
+	maxBins         = 16
+	nnLowWatermark  = 2 // the number of peers in consecutive deepest bins that constitute as nearest neighbours
+	maxConnAttempts = 3 // when there is maxConnAttempts failed connect calls for a given peer it is considered non-connectable
 )
 
 var (
@@ -56,13 +57,18 @@ type Kad struct {
 	depth          uint8                 // current neighborhood depth
 	depthMu        sync.RWMutex          // protect depth changes
 	manageC        chan struct{}         // trigger the manage forever loop to connect to new peers
-	waitNext       map[string]time.Time  // sanction connections to a peer, key is overlay string and value is time to next retry
+	waitNext       map[string]retryInfo  // sanction connections to a peer, key is overlay string and value is a retry information
 	waitNextMu     sync.Mutex            // synchronize map
 	logger         logging.Logger        // logger
 	quit           chan struct{}         // quit channel
 	done           chan struct{}         // signal that `manage` has quit
 }
 
+type retryInfo struct {
+	tryAfter       time.Time
+	failedAttempts int
+}
+
 // New returns a new Kademlia.
 func New(o Options) *Kad {
 	if o.SaturationFunc == nil {
@@ -78,7 +84,7 @@ func New(o Options) *Kad {
 		connectedPeers: pslice.New(maxBins),
 		knownPeers:     pslice.New(maxBins),
 		manageC:        make(chan struct{}, 1),
-		waitNext:       make(map[string]time.Time),
+		waitNext:       make(map[string]retryInfo),
 		logger:         o.Logger,
 		quit:           make(chan struct{}),
 		done:           make(chan struct{}),
@@ -115,7 +121,7 @@ func (k *Kad) manage() {
 				}
 
 				k.waitNextMu.Lock()
-				if next, ok := k.waitNext[peer.String()]; ok && time.Now().Before(next) {
+				if next, ok := k.waitNext[peer.String()]; ok && time.Now().Before(next.tryAfter) {
 					k.waitNextMu.Unlock()
 					return false, false, nil
 				}
@@ -248,12 +254,33 @@ func (k *Kad) connect(ctx context.Context, peer swarm.Address, ma ma.Multiaddr,
 		if errors.Is(err, p2p.ErrAlreadyConnected) {
 			return nil
 		}
+
 		k.logger.Debugf("error connecting to peer %s: %v", peer, err)
+		retryTime := time.Now().Add(timeToRetry)
+		var e *p2p.ConnectionBackoffError
 		k.waitNextMu.Lock()
-		k.waitNext[peer.String()] = time.Now().Add(timeToRetry)
-		k.waitNextMu.Unlock()
+		failedAttempts := 0
+		if errors.As(err, &e) {
+			retryTime = e.TryAfter()
+		} else {
+			info, ok := k.waitNext[peer.String()]
+			if ok {
+				failedAttempts = info.failedAttempts
+			}
 
-		// TODO: somehow keep track of attempts and at some point forget about the peer
+			failedAttempts++
+		}
+
+		if failedAttempts > maxConnAttempts {
+			delete(k.waitNext, peer.String())
+			if err := k.addressBook.Remove(peer); err != nil {
+				k.logger.Debugf("could not remove peer from addressbook: %s", peer.String())
+			}
+		} else {
+			k.waitNext[peer.String()] = retryInfo{tryAfter: retryTime, failedAttempts: failedAttempts}
+		}
+
+		k.waitNextMu.Unlock()
 		return err
 	}
 
@@ -335,7 +362,7 @@ func (k *Kad) Disconnected(addr swarm.Address) {
 	k.connectedPeers.Remove(addr, po)
 
 	k.waitNextMu.Lock()
-	k.waitNext[addr.String()] = time.Now().Add(timeToRetry)
+	k.waitNext[addr.String()] = retryInfo{tryAfter: time.Now().Add(timeToRetry), failedAttempts: 0}
 	k.waitNextMu.Unlock()
 
 	k.depthMu.Lock()