A few changes for the network / tunnel link state

This commit is contained in:
Asim Aslam 2019-10-24 17:51:41 +01:00
parent f5b8a12106
commit f26d470db1
4 changed files with 119 additions and 32 deletions

View File

@ -677,19 +677,19 @@ func (n *network) getHopCount(rtr string) int {
// the route origin is our peer
if _, ok := n.peers[rtr]; ok {
return 2
return 10
}
// the route origin is the peer of our peer
for _, peer := range n.peers {
for id := range peer.peers {
if rtr == id {
return 3
return 100
}
}
}
// otherwise we are three hops away
return 4
return 1000
}
// getRouteMetric calculates router metric and returns it
@ -721,11 +721,15 @@ func (n *network) getRouteMetric(router string, gateway string, link string) int
// make sure length is non-zero
length := link.Length()
if length == 0 {
length = 10e10
log.Debugf("Link length is 0 %v %v", link, link.Length())
length = 10e9
}
return (delay * length * int64(hops)) / 10e9
log.Debugf("Network calculated metric %v delay %v length %v distance %v", (delay*length*int64(hops))/10e6, delay, length, hops)
return (delay * length * int64(hops)) / 10e6
}
log.Debugf("Network failed to find a link to gateway: %s", gateway)
return math.MaxInt64
}
@ -783,12 +787,18 @@ func (n *network) processCtrlChan(listener tunnel.Listener) {
}
// calculate route metric and add to the advertised metric
// we need to make sure we do not overflow math.MaxInt64
log.Debugf("Network metric for router %s and gateway %s", event.Route.Router, event.Route.Gateway)
if metric := n.getRouteMetric(event.Route.Router, event.Route.Gateway, event.Route.Link); metric != math.MaxInt64 {
route.Metric += metric
metric := n.getRouteMetric(event.Route.Router, event.Route.Gateway, event.Route.Link)
log.Debugf("Network metric for router %s and gateway %s: %v", event.Route.Router, event.Route.Gateway, metric)
// check we don't overflow max int 64
if d := route.Metric + metric; d > math.MaxInt64 || d <= 0 {
// set to max int64 if we overflow
route.Metric = math.MaxInt64
} else {
route.Metric = metric
// set the combined value of metrics otherwise
route.Metric = d
}
// create router event
e := &router.Event{
Type: router.EventType(event.Type),

View File

@ -19,6 +19,7 @@ import (
"github.com/micro/go-micro/proxy"
"github.com/micro/go-micro/router"
"github.com/micro/go-micro/server"
"github.com/micro/go-micro/util/log"
)
// Proxy will transparently proxy requests to an endpoint.
@ -294,6 +295,8 @@ func (p *Proxy) ServeRequest(ctx context.Context, req server.Request, rsp server
continue
}
log.Debugf("Proxy using route %+v\n", route)
// set the address to call
addresses := toNodes([]router.Route{route})
opts = append(opts, client.WithAddress(addresses...))

View File

@ -746,8 +746,13 @@ func (t *tun) setupLink(node string) (*link, error) {
}
log.Debugf("Tunnel connected to %s", node)
// create a new link
link := newLink(c)
// set link id to remote side
link.id = c.Remote()
// send the first connect message
if err := c.Send(&transport.Message{
if err := link.Send(&transport.Message{
Header: map[string]string{
"Micro-Tunnel": "connect",
"Micro-Tunnel-Id": t.id,
@ -757,10 +762,6 @@ func (t *tun) setupLink(node string) (*link, error) {
return nil, err
}
// create a new link
link := newLink(c)
// set link id to remote side
link.id = c.Remote()
// we made the outbound connection
// and sent the connect message
link.connected = true

View File

@ -1,12 +1,14 @@
package tunnel
import (
"bytes"
"io"
"sync"
"time"
"github.com/google/uuid"
"github.com/micro/go-micro/transport"
"github.com/micro/go-micro/util/log"
)
type link struct {
@ -42,6 +44,9 @@ type link struct {
rate float64
// keep an error count on the link
errCount int
// link state channel
state chan *packet
}
// packet send over link
@ -56,21 +61,49 @@ type packet struct {
err error
}
var (
// the 4 byte 0 packet sent to determine the link state
linkRequest = []byte{0, 0, 0, 0}
// the 4 byte 1 filled packet sent to determine link state
linkResponse = []byte{1, 1, 1, 1}
)
func newLink(s transport.Socket) *link {
l := &link{
Socket: s,
id: uuid.New().String(),
lastKeepAlive: time.Now(),
closed: make(chan bool),
state: make(chan *packet, 64),
channels: make(map[string]time.Time),
sendQueue: make(chan *packet, 128),
recvQueue: make(chan *packet, 128),
}
// process inbound/outbound packets
go l.process()
go l.expiry()
// manage the link state
go l.manage()
return l
}
// setRate sets the bits per second rate as a float64
func (l *link) setRate(bits int64, delta time.Duration) {
// rate of send in bits per nanosecond
rate := float64(bits) / float64(delta.Nanoseconds())
// default the rate if its zero
if l.rate == 0 {
// rate per second
l.rate = rate * 1e9
} else {
// set new rate per second
l.rate = 0.8*l.rate + 0.2*(rate*1e9)
}
}
// setRTT sets a nanosecond based moving average roundtrip time for the link
func (l *link) setRTT(d time.Duration) {
l.Lock()
defer l.Unlock()
@ -101,8 +134,22 @@ func (l *link) process() {
// process new received message
pk := &packet{message: m, err: err}
// this is our link state packet
if m.Header["Micro-Method"] == "link" {
// process link state message
select {
case l.recvQueue <- &packet{message: m, err: err}:
case l.state <- pk:
default:
}
continue
}
// process all messages as is
select {
case l.recvQueue <- pk:
case <-l.closed:
return
}
@ -122,15 +169,49 @@ func (l *link) process() {
}
}
// watches the channel expiry
func (l *link) expiry() {
// manage manages the link state including rtt packets and channel mapping expiry
func (l *link) manage() {
// tick over every minute to expire and fire rtt packets
t := time.NewTicker(time.Minute)
defer t.Stop()
// used to send link state packets
send := func(b []byte) {
l.Send(&transport.Message{
Header: map[string]string{
"Micro-Method": "link",
}, Body: b,
})
}
// set time now
now := time.Now()
// send the initial rtt request packet
send(linkRequest)
for {
select {
// exit if closed
case <-l.closed:
return
// process link state rtt packets
case p := <-l.state:
if p.err != nil {
continue
}
// check the type of message
switch {
case bytes.Compare(p.message.Body, linkRequest) == 0:
log.Tracef("Link %s received link request %v", l.id, p.message.Body)
// send response
send(linkResponse)
case bytes.Compare(p.message.Body, linkResponse) == 0:
// set round trip time
d := time.Since(now)
log.Tracef("Link %s received link response in %v", p.message.Body, d)
l.setRTT(d)
}
case <-t.C:
// drop any channel mappings older than 2 minutes
var kill []string
@ -155,6 +236,10 @@ func (l *link) expiry() {
delete(l.channels, ch)
}
l.Unlock()
// fire off a link state rtt packet
now = time.Now()
send(linkRequest)
}
}
}
@ -278,23 +363,11 @@ func (l *link) Send(m *transport.Message) error {
// calculate based on data
if dataSent > 0 {
// measure time taken
delta := time.Since(now)
// bit sent
bits := dataSent * 1024
// rate of send in bits per nanosecond
rate := float64(bits) / float64(delta.Nanoseconds())
// default the rate if its zero
if l.rate == 0 {
// rate per second
l.rate = rate * 1e9
} else {
// set new rate per second
l.rate = 0.8*l.rate + 0.2*(rate*1e9)
}
// set the rate
l.setRate(int64(bits), time.Since(now))
}
return nil