Merge pull request #859 from milosgajdos83/handle-channel-conn-errors

Handle tunnel session Accept errors gracefully

network/default.go

@@ -19,6 +19,7 @@ import (
 	"github.com/micro/go-micro/transport"
 	"github.com/micro/go-micro/tunnel"
 	tun "github.com/micro/go-micro/tunnel/transport"
+	"github.com/micro/go-micro/util/backoff"
 	"github.com/micro/go-micro/util/log"
 )
 
@@ -266,15 +267,26 @@ func (n *network) handleNetConn(sess tunnel.Session, msg chan *transport.Message
 
 // acceptNetConn accepts connections from NetworkChannel
 func (n *network) acceptNetConn(l tunnel.Listener, recv chan *transport.Message) {
+	var i int
 	for {
 		// accept a connection
 		conn, err := l.Accept()
 		if err != nil {
-			log.Debugf("Network tunnel [%s] accept error: %v", NetworkChannel, err)
+			sleep := backoff.Do(i)
+			log.Debugf("Network tunnel [%s] accept error: %v, backing off for %v", ControlChannel, err, sleep)
+			time.Sleep(sleep)
+			if i > 5 {
+				i = 0
+			}
+			i++
+			continue
 		}
 
 		select {
 		case <-n.closed:
+			if err := conn.Close(); err != nil {
+				log.Debugf("Network tunnel [%s] failed to close connection: %v", NetworkChannel, err)
+			}
 			return
 		default:
 			// go handle NetworkChannel connection
@@ -555,15 +567,27 @@ func (n *network) handleCtrlConn(sess tunnel.Session, msg chan *transport.Messag
 
 // acceptCtrlConn accepts connections from ControlChannel
 func (n *network) acceptCtrlConn(l tunnel.Listener, recv chan *transport.Message) {
+	var i int
 	for {
 		// accept a connection
 		conn, err := l.Accept()
 		if err != nil {
-			log.Debugf("Network tunnel [%s] accept error: %v", ControlChannel, err)
+			sleep := backoff.Do(i)
+			log.Debugf("Network tunnel [%s] accept error: %v, backing off for %v", ControlChannel, err, sleep)
+			time.Sleep(sleep)
+			if i > 5 {
+				// reset the counter
+				i = 0
+			}
+			i++
+			continue
 		}
 
 		select {
 		case <-n.closed:
+			if err := conn.Close(); err != nil {
+				log.Debugf("Network tunnel [%s] failed to close connection: %v", ControlChannel, err)
+			}
 			return
 		default:
 			// go handle ControlChannel connection