A few changes for the network / tunnel link state
This commit is contained in:
parent
f5b8a12106
commit
f26d470db1
@ -677,19 +677,19 @@ func (n *network) getHopCount(rtr string) int {
|
|||||||
|
|
||||||
// the route origin is our peer
|
// the route origin is our peer
|
||||||
if _, ok := n.peers[rtr]; ok {
|
if _, ok := n.peers[rtr]; ok {
|
||||||
return 2
|
return 10
|
||||||
}
|
}
|
||||||
|
|
||||||
// the route origin is the peer of our peer
|
// the route origin is the peer of our peer
|
||||||
for _, peer := range n.peers {
|
for _, peer := range n.peers {
|
||||||
for id := range peer.peers {
|
for id := range peer.peers {
|
||||||
if rtr == id {
|
if rtr == id {
|
||||||
return 3
|
return 100
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
// otherwise we are three hops away
|
// otherwise we are three hops away
|
||||||
return 4
|
return 1000
|
||||||
}
|
}
|
||||||
|
|
||||||
// getRouteMetric calculates router metric and returns it
|
// getRouteMetric calculates router metric and returns it
|
||||||
@ -721,11 +721,15 @@ func (n *network) getRouteMetric(router string, gateway string, link string) int
|
|||||||
// make sure length is non-zero
|
// make sure length is non-zero
|
||||||
length := link.Length()
|
length := link.Length()
|
||||||
if length == 0 {
|
if length == 0 {
|
||||||
length = 10e10
|
log.Debugf("Link length is 0 %v %v", link, link.Length())
|
||||||
|
length = 10e9
|
||||||
}
|
}
|
||||||
return (delay * length * int64(hops)) / 10e9
|
log.Debugf("Network calculated metric %v delay %v length %v distance %v", (delay*length*int64(hops))/10e6, delay, length, hops)
|
||||||
|
return (delay * length * int64(hops)) / 10e6
|
||||||
}
|
}
|
||||||
|
|
||||||
log.Debugf("Network failed to find a link to gateway: %s", gateway)
|
log.Debugf("Network failed to find a link to gateway: %s", gateway)
|
||||||
|
|
||||||
return math.MaxInt64
|
return math.MaxInt64
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -783,12 +787,18 @@ func (n *network) processCtrlChan(listener tunnel.Listener) {
|
|||||||
}
|
}
|
||||||
// calculate route metric and add to the advertised metric
|
// calculate route metric and add to the advertised metric
|
||||||
// we need to make sure we do not overflow math.MaxInt64
|
// we need to make sure we do not overflow math.MaxInt64
|
||||||
log.Debugf("Network metric for router %s and gateway %s", event.Route.Router, event.Route.Gateway)
|
metric := n.getRouteMetric(event.Route.Router, event.Route.Gateway, event.Route.Link)
|
||||||
if metric := n.getRouteMetric(event.Route.Router, event.Route.Gateway, event.Route.Link); metric != math.MaxInt64 {
|
log.Debugf("Network metric for router %s and gateway %s: %v", event.Route.Router, event.Route.Gateway, metric)
|
||||||
route.Metric += metric
|
|
||||||
|
// check we don't overflow max int 64
|
||||||
|
if d := route.Metric + metric; d > math.MaxInt64 || d <= 0 {
|
||||||
|
// set to max int64 if we overflow
|
||||||
|
route.Metric = math.MaxInt64
|
||||||
} else {
|
} else {
|
||||||
route.Metric = metric
|
// set the combined value of metrics otherwise
|
||||||
|
route.Metric = d
|
||||||
}
|
}
|
||||||
|
|
||||||
// create router event
|
// create router event
|
||||||
e := &router.Event{
|
e := &router.Event{
|
||||||
Type: router.EventType(event.Type),
|
Type: router.EventType(event.Type),
|
||||||
|
@ -19,6 +19,7 @@ import (
|
|||||||
"github.com/micro/go-micro/proxy"
|
"github.com/micro/go-micro/proxy"
|
||||||
"github.com/micro/go-micro/router"
|
"github.com/micro/go-micro/router"
|
||||||
"github.com/micro/go-micro/server"
|
"github.com/micro/go-micro/server"
|
||||||
|
"github.com/micro/go-micro/util/log"
|
||||||
)
|
)
|
||||||
|
|
||||||
// Proxy will transparently proxy requests to an endpoint.
|
// Proxy will transparently proxy requests to an endpoint.
|
||||||
@ -294,6 +295,8 @@ func (p *Proxy) ServeRequest(ctx context.Context, req server.Request, rsp server
|
|||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
|
log.Debugf("Proxy using route %+v\n", route)
|
||||||
|
|
||||||
// set the address to call
|
// set the address to call
|
||||||
addresses := toNodes([]router.Route{route})
|
addresses := toNodes([]router.Route{route})
|
||||||
opts = append(opts, client.WithAddress(addresses...))
|
opts = append(opts, client.WithAddress(addresses...))
|
||||||
|
@ -746,8 +746,13 @@ func (t *tun) setupLink(node string) (*link, error) {
|
|||||||
}
|
}
|
||||||
log.Debugf("Tunnel connected to %s", node)
|
log.Debugf("Tunnel connected to %s", node)
|
||||||
|
|
||||||
|
// create a new link
|
||||||
|
link := newLink(c)
|
||||||
|
// set link id to remote side
|
||||||
|
link.id = c.Remote()
|
||||||
|
|
||||||
// send the first connect message
|
// send the first connect message
|
||||||
if err := c.Send(&transport.Message{
|
if err := link.Send(&transport.Message{
|
||||||
Header: map[string]string{
|
Header: map[string]string{
|
||||||
"Micro-Tunnel": "connect",
|
"Micro-Tunnel": "connect",
|
||||||
"Micro-Tunnel-Id": t.id,
|
"Micro-Tunnel-Id": t.id,
|
||||||
@ -757,10 +762,6 @@ func (t *tun) setupLink(node string) (*link, error) {
|
|||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
// create a new link
|
|
||||||
link := newLink(c)
|
|
||||||
// set link id to remote side
|
|
||||||
link.id = c.Remote()
|
|
||||||
// we made the outbound connection
|
// we made the outbound connection
|
||||||
// and sent the connect message
|
// and sent the connect message
|
||||||
link.connected = true
|
link.connected = true
|
||||||
|
109
tunnel/link.go
109
tunnel/link.go
@ -1,12 +1,14 @@
|
|||||||
package tunnel
|
package tunnel
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"bytes"
|
||||||
"io"
|
"io"
|
||||||
"sync"
|
"sync"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/google/uuid"
|
"github.com/google/uuid"
|
||||||
"github.com/micro/go-micro/transport"
|
"github.com/micro/go-micro/transport"
|
||||||
|
"github.com/micro/go-micro/util/log"
|
||||||
)
|
)
|
||||||
|
|
||||||
type link struct {
|
type link struct {
|
||||||
@ -42,6 +44,9 @@ type link struct {
|
|||||||
rate float64
|
rate float64
|
||||||
// keep an error count on the link
|
// keep an error count on the link
|
||||||
errCount int
|
errCount int
|
||||||
|
|
||||||
|
// link state channel
|
||||||
|
state chan *packet
|
||||||
}
|
}
|
||||||
|
|
||||||
// packet send over link
|
// packet send over link
|
||||||
@ -56,21 +61,49 @@ type packet struct {
|
|||||||
err error
|
err error
|
||||||
}
|
}
|
||||||
|
|
||||||
|
var (
|
||||||
|
// the 4 byte 0 packet sent to determine the link state
|
||||||
|
linkRequest = []byte{0, 0, 0, 0}
|
||||||
|
// the 4 byte 1 filled packet sent to determine link state
|
||||||
|
linkResponse = []byte{1, 1, 1, 1}
|
||||||
|
)
|
||||||
|
|
||||||
func newLink(s transport.Socket) *link {
|
func newLink(s transport.Socket) *link {
|
||||||
l := &link{
|
l := &link{
|
||||||
Socket: s,
|
Socket: s,
|
||||||
id: uuid.New().String(),
|
id: uuid.New().String(),
|
||||||
lastKeepAlive: time.Now(),
|
lastKeepAlive: time.Now(),
|
||||||
closed: make(chan bool),
|
closed: make(chan bool),
|
||||||
|
state: make(chan *packet, 64),
|
||||||
channels: make(map[string]time.Time),
|
channels: make(map[string]time.Time),
|
||||||
sendQueue: make(chan *packet, 128),
|
sendQueue: make(chan *packet, 128),
|
||||||
recvQueue: make(chan *packet, 128),
|
recvQueue: make(chan *packet, 128),
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// process inbound/outbound packets
|
||||||
go l.process()
|
go l.process()
|
||||||
go l.expiry()
|
// manage the link state
|
||||||
|
go l.manage()
|
||||||
|
|
||||||
return l
|
return l
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// setRate sets the bits per second rate as a float64
|
||||||
|
func (l *link) setRate(bits int64, delta time.Duration) {
|
||||||
|
// rate of send in bits per nanosecond
|
||||||
|
rate := float64(bits) / float64(delta.Nanoseconds())
|
||||||
|
|
||||||
|
// default the rate if its zero
|
||||||
|
if l.rate == 0 {
|
||||||
|
// rate per second
|
||||||
|
l.rate = rate * 1e9
|
||||||
|
} else {
|
||||||
|
// set new rate per second
|
||||||
|
l.rate = 0.8*l.rate + 0.2*(rate*1e9)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// setRTT sets a nanosecond based moving average roundtrip time for the link
|
||||||
func (l *link) setRTT(d time.Duration) {
|
func (l *link) setRTT(d time.Duration) {
|
||||||
l.Lock()
|
l.Lock()
|
||||||
defer l.Unlock()
|
defer l.Unlock()
|
||||||
@ -101,8 +134,22 @@ func (l *link) process() {
|
|||||||
|
|
||||||
// process new received message
|
// process new received message
|
||||||
|
|
||||||
|
pk := &packet{message: m, err: err}
|
||||||
|
|
||||||
|
// this is our link state packet
|
||||||
|
if m.Header["Micro-Method"] == "link" {
|
||||||
|
// process link state message
|
||||||
select {
|
select {
|
||||||
case l.recvQueue <- &packet{message: m, err: err}:
|
case l.state <- pk:
|
||||||
|
default:
|
||||||
|
}
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
// process all messages as is
|
||||||
|
|
||||||
|
select {
|
||||||
|
case l.recvQueue <- pk:
|
||||||
case <-l.closed:
|
case <-l.closed:
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
@ -122,15 +169,49 @@ func (l *link) process() {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// watches the channel expiry
|
// manage manages the link state including rtt packets and channel mapping expiry
|
||||||
func (l *link) expiry() {
|
func (l *link) manage() {
|
||||||
|
// tick over every minute to expire and fire rtt packets
|
||||||
t := time.NewTicker(time.Minute)
|
t := time.NewTicker(time.Minute)
|
||||||
defer t.Stop()
|
defer t.Stop()
|
||||||
|
|
||||||
|
// used to send link state packets
|
||||||
|
send := func(b []byte) {
|
||||||
|
l.Send(&transport.Message{
|
||||||
|
Header: map[string]string{
|
||||||
|
"Micro-Method": "link",
|
||||||
|
}, Body: b,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
// set time now
|
||||||
|
now := time.Now()
|
||||||
|
|
||||||
|
// send the initial rtt request packet
|
||||||
|
send(linkRequest)
|
||||||
|
|
||||||
for {
|
for {
|
||||||
select {
|
select {
|
||||||
|
// exit if closed
|
||||||
case <-l.closed:
|
case <-l.closed:
|
||||||
return
|
return
|
||||||
|
// process link state rtt packets
|
||||||
|
case p := <-l.state:
|
||||||
|
if p.err != nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
// check the type of message
|
||||||
|
switch {
|
||||||
|
case bytes.Compare(p.message.Body, linkRequest) == 0:
|
||||||
|
log.Tracef("Link %s received link request %v", l.id, p.message.Body)
|
||||||
|
// send response
|
||||||
|
send(linkResponse)
|
||||||
|
case bytes.Compare(p.message.Body, linkResponse) == 0:
|
||||||
|
// set round trip time
|
||||||
|
d := time.Since(now)
|
||||||
|
log.Tracef("Link %s received link response in %v", p.message.Body, d)
|
||||||
|
l.setRTT(d)
|
||||||
|
}
|
||||||
case <-t.C:
|
case <-t.C:
|
||||||
// drop any channel mappings older than 2 minutes
|
// drop any channel mappings older than 2 minutes
|
||||||
var kill []string
|
var kill []string
|
||||||
@ -155,6 +236,10 @@ func (l *link) expiry() {
|
|||||||
delete(l.channels, ch)
|
delete(l.channels, ch)
|
||||||
}
|
}
|
||||||
l.Unlock()
|
l.Unlock()
|
||||||
|
|
||||||
|
// fire off a link state rtt packet
|
||||||
|
now = time.Now()
|
||||||
|
send(linkRequest)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -278,23 +363,11 @@ func (l *link) Send(m *transport.Message) error {
|
|||||||
|
|
||||||
// calculate based on data
|
// calculate based on data
|
||||||
if dataSent > 0 {
|
if dataSent > 0 {
|
||||||
// measure time taken
|
|
||||||
delta := time.Since(now)
|
|
||||||
|
|
||||||
// bit sent
|
// bit sent
|
||||||
bits := dataSent * 1024
|
bits := dataSent * 1024
|
||||||
|
|
||||||
// rate of send in bits per nanosecond
|
// set the rate
|
||||||
rate := float64(bits) / float64(delta.Nanoseconds())
|
l.setRate(int64(bits), time.Since(now))
|
||||||
|
|
||||||
// default the rate if its zero
|
|
||||||
if l.rate == 0 {
|
|
||||||
// rate per second
|
|
||||||
l.rate = rate * 1e9
|
|
||||||
} else {
|
|
||||||
// set new rate per second
|
|
||||||
l.rate = 0.8*l.rate + 0.2*(rate*1e9)
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
|
Loading…
x
Reference in New Issue
Block a user