From fed5af68e69da70e4a63b5519dea71a1d2ea1934 Mon Sep 17 00:00:00 2001
From: Milos Gajdos <milosgajdos83@gmail.com>
Date: Tue, 15 Oct 2019 15:03:47 +0100
Subject: [PATCH 1/2] Handle Accept errors gracefully.

Originally when Accept fails we log the error and let the program flow
continue. This can lead to us spawning handling connection go routines
on nil connections which in turn leads to Go panics.
---
 network/default.go | 26 ++++++++++++++++++++------
 1 file changed, 20 insertions(+), 6 deletions(-)

diff --git a/network/default.go b/network/default.go
index 6952aaa2..a6c759c7 100644
--- a/network/default.go
+++ b/network/default.go
@@ -269,14 +269,21 @@ func (n *network) acceptNetConn(l tunnel.Listener, recv chan *transport.Message)
 	for {
 		// accept a connection
 		conn, err := l.Accept()
-		if err != nil {
-			log.Debugf("Network tunnel [%s] accept error: %v", NetworkChannel, err)
-		}
 
 		select {
 		case <-n.closed:
+			// only try to close the connection if it has been successfully opened
+			if err != nil {
+				if closeErr := conn.Close(); closeErr != nil {
+					log.Debugf("Network tunnel [%s] failed to close connection: %v", NetworkChannel, closeErr)
+				}
+			}
 			return
 		default:
+			if err != nil {
+				log.Debugf("Network tunnel [%s] accept error: %v", NetworkChannel, err)
+				continue
+			}
 			// go handle NetworkChannel connection
 			go n.handleNetConn(conn, recv)
 		}
@@ -558,14 +565,21 @@ func (n *network) acceptCtrlConn(l tunnel.Listener, recv chan *transport.Message
 	for {
 		// accept a connection
 		conn, err := l.Accept()
-		if err != nil {
-			log.Debugf("Network tunnel [%s] accept error: %v", ControlChannel, err)
-		}
 
 		select {
 		case <-n.closed:
+			// only try to close the connection if it has been successfully opened
+			if err != nil {
+				if closeErr := conn.Close(); closeErr != nil {
+					log.Debugf("Network tunnel [%s] failed to close connection: %v", ControlChannel, closeErr)
+				}
+			}
 			return
 		default:
+			if err != nil {
+				log.Debugf("Network tunnel [%s] accept error: %v", ControlChannel, err)
+				continue
+			}
 			// go handle ControlChannel connection
 			go n.handleCtrlConn(conn, recv)
 		}

From 4936a2e1a5181d9836c9d6f44649e2fa5eb812b3 Mon Sep 17 00:00:00 2001
From: Milos Gajdos <milosgajdos83@gmail.com>
Date: Tue, 15 Oct 2019 15:58:33 +0100
Subject: [PATCH 2/2] Exponential backoff for failed accept connections

---
 network/default.go | 46 ++++++++++++++++++++++++++++------------------
 1 file changed, 28 insertions(+), 18 deletions(-)

diff --git a/network/default.go b/network/default.go
index a6c759c7..535b9a83 100644
--- a/network/default.go
+++ b/network/default.go
@@ -19,6 +19,7 @@ import (
 	"github.com/micro/go-micro/transport"
 	"github.com/micro/go-micro/tunnel"
 	tun "github.com/micro/go-micro/tunnel/transport"
+	"github.com/micro/go-micro/util/backoff"
 	"github.com/micro/go-micro/util/log"
 )
 
@@ -266,24 +267,28 @@ func (n *network) handleNetConn(sess tunnel.Session, msg chan *transport.Message
 
 // acceptNetConn accepts connections from NetworkChannel
 func (n *network) acceptNetConn(l tunnel.Listener, recv chan *transport.Message) {
+	var i int
 	for {
 		// accept a connection
 		conn, err := l.Accept()
+		if err != nil {
+			sleep := backoff.Do(i)
+			log.Debugf("Network tunnel [%s] accept error: %v, backing off for %v", ControlChannel, err, sleep)
+			time.Sleep(sleep)
+			if i > 5 {
+				i = 0
+			}
+			i++
+			continue
+		}
 
 		select {
 		case <-n.closed:
-			// only try to close the connection if it has been successfully opened
-			if err != nil {
-				if closeErr := conn.Close(); closeErr != nil {
-					log.Debugf("Network tunnel [%s] failed to close connection: %v", NetworkChannel, closeErr)
-				}
+			if err := conn.Close(); err != nil {
+				log.Debugf("Network tunnel [%s] failed to close connection: %v", NetworkChannel, err)
 			}
 			return
 		default:
-			if err != nil {
-				log.Debugf("Network tunnel [%s] accept error: %v", NetworkChannel, err)
-				continue
-			}
 			// go handle NetworkChannel connection
 			go n.handleNetConn(conn, recv)
 		}
@@ -562,24 +567,29 @@ func (n *network) handleCtrlConn(sess tunnel.Session, msg chan *transport.Messag
 
 // acceptCtrlConn accepts connections from ControlChannel
 func (n *network) acceptCtrlConn(l tunnel.Listener, recv chan *transport.Message) {
+	var i int
 	for {
 		// accept a connection
 		conn, err := l.Accept()
+		if err != nil {
+			sleep := backoff.Do(i)
+			log.Debugf("Network tunnel [%s] accept error: %v, backing off for %v", ControlChannel, err, sleep)
+			time.Sleep(sleep)
+			if i > 5 {
+				// reset the counter
+				i = 0
+			}
+			i++
+			continue
+		}
 
 		select {
 		case <-n.closed:
-			// only try to close the connection if it has been successfully opened
-			if err != nil {
-				if closeErr := conn.Close(); closeErr != nil {
-					log.Debugf("Network tunnel [%s] failed to close connection: %v", ControlChannel, closeErr)
-				}
+			if err := conn.Close(); err != nil {
+				log.Debugf("Network tunnel [%s] failed to close connection: %v", ControlChannel, err)
 			}
 			return
 		default:
-			if err != nil {
-				log.Debugf("Network tunnel [%s] accept error: %v", ControlChannel, err)
-				continue
-			}
 			// go handle ControlChannel connection
 			go n.handleCtrlConn(conn, recv)
 		}