package tunnel import ( "errors" "math/rand" "strings" "sync" "time" "github.com/google/uuid" "github.com/micro/go-micro/transport" "github.com/micro/go-micro/util/log" ) var ( // DiscoverTime sets the time at which we fire discover messages DiscoverTime = 60 * time.Second // KeepAliveTime defines time interval we send keepalive messages to outbound links KeepAliveTime = 30 * time.Second // ReconnectTime defines time interval we periodically attempt to reconnect dead links ReconnectTime = 5 * time.Second ) // tun represents a network tunnel type tun struct { options Options sync.RWMutex // the unique id for this tunnel id string // tunnel token for session encryption token string // to indicate if we're connected or not connected bool // the send channel for all messages send chan *message // close channel closed chan bool // a map of sessions based on Micro-Tunnel-Channel sessions map[string]*session // outbound links links map[string]*link // listener listener transport.Listener } // create new tunnel on top of a link func newTunnel(opts ...Option) *tun { options := DefaultOptions() for _, o := range opts { o(&options) } return &tun{ options: options, id: options.Id, token: options.Token, send: make(chan *message, 128), closed: make(chan bool), sessions: make(map[string]*session), links: make(map[string]*link), } } // Init initializes tunnel options func (t *tun) Init(opts ...Option) error { t.Lock() defer t.Unlock() for _, o := range opts { o(&t.options) } return nil } // getSession returns a session from the internal session map. // It does this based on the Micro-Tunnel-Channel and Micro-Tunnel-Session func (t *tun) getSession(channel, session string) (*session, bool) { // get the session t.RLock() s, ok := t.sessions[channel+session] t.RUnlock() return s, ok } // delSession deletes a session if it exists func (t *tun) delSession(channel, session string) { t.Lock() if s, ok := t.sessions[channel+session]; ok { s.Close() } delete(t.sessions, channel+session) t.Unlock() } // listChannels returns a list of listening channels func (t *tun) listChannels() []string { t.RLock() defer t.RUnlock() //nolint:prealloc var channels []string for _, session := range t.sessions { if session.session != "listener" { continue } channels = append(channels, session.channel) } return channels } // newSession creates a new session and saves it func (t *tun) newSession(channel, sessionId string) (*session, bool) { // new session s := &session{ tunnel: t.id, channel: channel, session: sessionId, token: t.token, closed: make(chan bool), recv: make(chan *message, 128), send: t.send, errChan: make(chan error, 1), } // save session t.Lock() _, ok := t.sessions[channel+sessionId] if ok { // session already exists t.Unlock() return nil, false } t.sessions[channel+sessionId] = s t.Unlock() // return session return s, true } // TODO: use tunnel id as part of the session func (t *tun) newSessionId() string { return uuid.New().String() } // announce will send a message to the link to tell the other side of a channel mapping we have. // This usually happens if someone calls Dial and sends a discover message but otherwise we // periodically send these messages to asynchronously manage channel mappings. func (t *tun) announce(channel, session string, link *link) { // create the "announce" response message for a discover request msg := &transport.Message{ Header: map[string]string{ "Micro-Tunnel": "announce", "Micro-Tunnel-Id": t.id, "Micro-Tunnel-Channel": channel, "Micro-Tunnel-Session": session, "Micro-Tunnel-Link": link.id, }, } // if no channel is present we've been asked to discover all channels if len(channel) == 0 { // get the list of channels channels := t.listChannels() // if there are no channels continue if len(channels) == 0 { return } // create a list of channels as comma separated list channel = strings.Join(channels, ",") // set channels as header msg.Header["Micro-Tunnel-Channel"] = channel } else { // otherwise look for a single channel mapping // looking for existing mapping as a listener _, exists := t.getSession(channel, "listener") if !exists { return } } log.Debugf("Tunnel sending announce for discovery of channel(s) %s", channel) // send back the announcement if err := link.Send(msg); err != nil { log.Debugf("Tunnel failed to send announcement for channel(s) %s message: %v", channel, err) } } // monitor monitors outbound links and attempts to reconnect to the failed ones func (t *tun) monitor() { reconnect := time.NewTicker(ReconnectTime) defer reconnect.Stop() for { select { case <-t.closed: return case <-reconnect.C: t.RLock() var delLinks []string // check the link status and purge dead links for node, link := range t.links { // check link status switch link.State() { case "closed": delLinks = append(delLinks, node) case "error": delLinks = append(delLinks, node) } } t.RUnlock() // delete the dead links if len(delLinks) > 0 { t.Lock() for _, node := range delLinks { log.Debugf("Tunnel deleting dead link for %s", node) if link, ok := t.links[node]; ok { link.Close() delete(t.links, node) } } t.Unlock() } // check current link status var connect []string // build list of unknown nodes to connect to t.RLock() for _, node := range t.options.Nodes { if _, ok := t.links[node]; !ok { connect = append(connect, node) } } t.RUnlock() for _, node := range connect { // create new link link, err := t.setupLink(node) if err != nil { log.Debugf("Tunnel failed to setup node link to %s: %v", node, err) continue } // save the link t.Lock() t.links[node] = link t.Unlock() } } } } // process outgoing messages sent by all local sessions func (t *tun) process() { // manage the send buffer // all pseudo sessions throw everything down this for { select { case msg := <-t.send: newMsg := &transport.Message{ Header: make(map[string]string), } // set the data if msg.data != nil { for k, v := range msg.data.Header { newMsg.Header[k] = v } newMsg.Body = msg.data.Body } // set message head newMsg.Header["Micro-Tunnel"] = msg.typ // set the tunnel id on the outgoing message newMsg.Header["Micro-Tunnel-Id"] = msg.tunnel // set the tunnel channel on the outgoing message newMsg.Header["Micro-Tunnel-Channel"] = msg.channel // set the session id newMsg.Header["Micro-Tunnel-Session"] = msg.session // send the message via the interface t.RLock() if len(t.links) == 0 { log.Debugf("No links to send message type: %s channel: %s", msg.typ, msg.channel) } var sent bool var err error var sendTo []*link // build the list of links ot send to for node, link := range t.links { // get the values we need link.RLock() id := link.id connected := link.connected loopback := link.loopback _, exists := link.channels[msg.channel] link.RUnlock() // if the link is not connected skip it if !connected { log.Debugf("Link for node %s not connected", node) err = errors.New("link not connected") continue } // if the link was a loopback accepted connection // and the message is being sent outbound via // a dialled connection don't use this link if loopback && msg.outbound { log.Tracef("Link for node %s is loopback", node) err = errors.New("link is loopback") continue } // if the message was being returned by the loopback listener // send it back up the loopback link only if msg.loopback && !loopback { log.Tracef("Link for message %s is loopback", node) err = errors.New("link is not loopback") continue } // check the multicast mappings if msg.mode == Multicast { // channel mapping not found in link if !exists { continue } } else { // if we're picking the link check the id // this is where we explicitly set the link // in a message received via the listen method if len(msg.link) > 0 && id != msg.link { err = errors.New("link not found") continue } } // add to link list sendTo = append(sendTo, link) } t.RUnlock() // send the message for _, link := range sendTo { // send the message via the current link log.Tracef("Tunnel sending %+v to %s", newMsg.Header, link.Remote()) if errr := link.Send(newMsg); errr != nil { log.Debugf("Tunnel error sending %+v to %s: %v", newMsg.Header, link.Remote(), errr) err = errors.New(errr.Error()) t.delLink(link.Remote()) continue } // is sent sent = true // keep sending broadcast messages if msg.mode > Unicast { continue } // break on unicast break } var gerr error // set the error if not sent if !sent { gerr = err } // skip if its not been set if msg.errChan == nil { continue } // return error non blocking select { case msg.errChan <- gerr: default: } case <-t.closed: return } } } func (t *tun) delLink(remote string) { t.Lock() defer t.Unlock() // get the link for id, link := range t.links { if link.id != remote { continue } // close and delete log.Debugf("Tunnel deleting link node: %s remote: %s", id, link.Remote()) link.Close() delete(t.links, id) } } // process incoming messages func (t *tun) listen(link *link) { // remove the link on exit defer func() { t.delLink(link.Remote()) }() // let us know if its a loopback var loopback bool var connected bool // set the connected value link.RLock() connected = link.connected link.RUnlock() for { // process anything via the net interface msg := new(transport.Message) if err := link.Recv(msg); err != nil { log.Debugf("Tunnel link %s receive error: %v", link.Remote(), err) return } // TODO: figure out network authentication // for now we use tunnel token to encrypt/decrypt // session communication, but we will probably need // some sort of network authentication (token) to avoid // having rogue actors spamming the network // message type mtype := msg.Header["Micro-Tunnel"] // the tunnel id id := msg.Header["Micro-Tunnel-Id"] // the tunnel channel channel := msg.Header["Micro-Tunnel-Channel"] // the session id sessionId := msg.Header["Micro-Tunnel-Session"] // if its not connected throw away the link // the first message we process needs to be connect if !connected && mtype != "connect" { log.Debugf("Tunnel link %s not connected", link.id) return } // this state machine block handles the only message types // that we know or care about; connect, close, open, accept, // discover, announce, session, keepalive switch mtype { case "connect": log.Debugf("Tunnel link %s received connect message", link.Remote()) link.Lock() // check if we're connecting to ourselves? if id == t.id { link.loopback = true loopback = true } // set to remote node link.id = link.Remote() // set as connected link.connected = true connected = true link.Unlock() // save the link once connected t.Lock() t.links[link.Remote()] = link t.Unlock() // send back a discovery go t.announce("", "", link) // nothing more to do continue case "close": log.Debugf("Tunnel link %s received close message", link.Remote()) // if there is no channel then we close the link // as its a signal from the other side to close the connection if len(channel) == 0 { log.Debugf("Tunnel link %s received close message", link.Remote()) return } // the entire listener was closed by the remote side so we need to // remove the channel mapping for it. should we also close sessions? if sessionId == "listener" { link.delChannel(channel) continue } // assuming there's a channel and session // try get the dialing socket s, exists := t.getSession(channel, sessionId) if exists && !loopback { // only delete the session if its unicast // otherwise ignore close on the multicast if s.mode == Unicast { // only delete this if its unicast // but not if its a loopback conn t.delSession(channel, sessionId) continue } } // otherwise its a session mapping of sorts case "keepalive": log.Debugf("Tunnel link %s received keepalive", link.Remote()) // save the keepalive link.keepalive() continue // a new connection dialled outbound case "open": log.Debugf("Tunnel link %s received open %s %s", link.id, channel, sessionId) // we just let it pass through to be processed // an accept returned by the listener case "accept": s, exists := t.getSession(channel, sessionId) // just set accepted on anything not unicast if exists && s.mode > Unicast { s.accepted = true continue } // if its already accepted move on if exists && s.accepted { continue } // otherwise we're going to process to accept // a continued session case "session": // process message log.Tracef("Tunnel received %+v from %s", msg.Header, link.Remote()) // an announcement of a channel listener case "announce": log.Tracef("Tunnel received %+v from %s", msg.Header, link.Remote()) // process the announcement channels := strings.Split(channel, ",") // update mapping in the link link.setChannel(channels...) // this was an announcement not intended for anything // if the dialing side sent "discover" then a session // id would be present. We skip in case of multicast. switch sessionId { case "listener", "multicast", "": continue } // get the session that asked for the discovery s, exists := t.getSession(channel, sessionId) if exists { // don't bother it's already discovered if s.discovered { continue } msg := &message{ typ: "announce", tunnel: id, channel: channel, session: sessionId, link: link.id, } // send the announce back to the caller select { case <-s.closed: case s.recv <- msg: } } continue case "discover": // send back an announcement go t.announce(channel, sessionId, link) continue default: // blackhole it continue } // strip tunnel message header for k := range msg.Header { if strings.HasPrefix(k, "Micro-Tunnel") { delete(msg.Header, k) } } // if the session id is blank there's nothing we can do // TODO: check this is the case, is there any reason // why we'd have a blank session? Is the tunnel // used for some other purpose? if len(channel) == 0 || len(sessionId) == 0 { continue } var s *session var exists bool // If its a loopback connection then we've enabled link direction // listening side is used for listening, the dialling side for dialling switch { case loopback, mtype == "open": s, exists = t.getSession(channel, "listener") // only return accept to the session case mtype == "accept": log.Debugf("Tunnel received accept message for channel: %s session: %s", channel, sessionId) s, exists = t.getSession(channel, sessionId) if exists && s.accepted { continue } default: // get the session based on the tunnel id and session // this could be something we dialed in which case // we have a session for it otherwise its a listener s, exists = t.getSession(channel, sessionId) if !exists { // try get it based on just the tunnel id // the assumption here is that a listener // has no session but its set a listener session s, exists = t.getSession(channel, "listener") } } // bail if no session or listener has been found if !exists { log.Tracef("Tunnel skipping no channel: %s session: %s exists", channel, sessionId) // drop it, we don't care about // messages we don't know about continue } // is the session closed? select { case <-s.closed: // closed delete(t.sessions, channel) continue default: // otherwise process } log.Tracef("Tunnel using channel: %s session: %s type: %s", s.channel, s.session, mtype) // construct a new transport message tmsg := &transport.Message{ Header: msg.Header, Body: msg.Body, } // construct the internal message imsg := &message{ tunnel: id, typ: mtype, channel: channel, session: sessionId, mode: s.mode, data: tmsg, link: link.id, loopback: loopback, errChan: make(chan error, 1), } // append to recv backlog // we don't block if we can't pass it on select { case s.recv <- imsg: default: } } } // discover sends channel discover requests periodically func (t *tun) discover(link *link) { tick := time.NewTicker(DiscoverTime) defer tick.Stop() for { select { case <-tick.C: // send a discovery message to all links if err := link.Send(&transport.Message{ Header: map[string]string{ "Micro-Tunnel": "discover", "Micro-Tunnel-Id": t.id, }, }); err != nil { log.Debugf("Tunnel failed to send discover to link %s: %v", link.Remote(), err) } case <-link.closed: return case <-t.closed: return } } } // keepalive periodically sends keepalive messages to link func (t *tun) keepalive(link *link) { keepalive := time.NewTicker(KeepAliveTime) defer keepalive.Stop() for { select { case <-t.closed: return case <-link.closed: return case <-keepalive.C: // send keepalive message log.Debugf("Tunnel sending keepalive to link: %v", link.Remote()) if err := link.Send(&transport.Message{ Header: map[string]string{ "Micro-Tunnel": "keepalive", "Micro-Tunnel-Id": t.id, }, }); err != nil { log.Debugf("Tunnel error sending keepalive to link %v: %v", link.Remote(), err) t.delLink(link.Remote()) return } } } } // setupLink connects to node and returns link if successful // It returns error if the link failed to be established func (t *tun) setupLink(node string) (*link, error) { log.Debugf("Tunnel setting up link: %s", node) c, err := t.options.Transport.Dial(node) if err != nil { log.Debugf("Tunnel failed to connect to %s: %v", node, err) return nil, err } log.Debugf("Tunnel connected to %s", node) // create a new link link := newLink(c) // set link id to remote side link.id = c.Remote() // send the first connect message if err := link.Send(&transport.Message{ Header: map[string]string{ "Micro-Tunnel": "connect", "Micro-Tunnel-Id": t.id, }, }); err != nil { return nil, err } // we made the outbound connection // and sent the connect message link.connected = true // process incoming messages go t.listen(link) // start keepalive monitor go t.keepalive(link) // discover things on the remote side go t.discover(link) return link, nil } func (t *tun) setupLinks() { for _, node := range t.options.Nodes { // skip zero length nodes if len(node) == 0 { continue } // link already exists if _, ok := t.links[node]; ok { continue } // connect to node and return link link, err := t.setupLink(node) if err != nil { log.Debugf("Tunnel failed to establish node link to %s: %v", node, err) continue } // save the link t.links[node] = link } } // connect the tunnel to all the nodes and listen for incoming tunnel connections func (t *tun) connect() error { l, err := t.options.Transport.Listen(t.options.Address) if err != nil { return err } // save the listener t.listener = l go func() { // accept inbound connections err := l.Accept(func(sock transport.Socket) { log.Debugf("Tunnel accepted connection from %s", sock.Remote()) // create a new link link := newLink(sock) // start keepalive monitor go t.keepalive(link) // discover things on the remote side go t.discover(link) // listen for inbound messages. // only save the link once connected. // we do this inside liste t.listen(link) }) t.RLock() defer t.RUnlock() // still connected but the tunnel died if err != nil && t.connected { log.Logf("Tunnel listener died: %v", err) } }() return nil } // Connect the tunnel func (t *tun) Connect() error { t.Lock() defer t.Unlock() // already connected if t.connected { // setup links t.setupLinks() return nil } // send the connect message if err := t.connect(); err != nil { return err } // set as connected t.connected = true // create new close channel t.closed = make(chan bool) // setup links t.setupLinks() // process outbound messages to be sent // process sends to all links go t.process() // monitor links go t.monitor() return nil } func (t *tun) close() error { // close all the sessions for id, s := range t.sessions { s.Close() delete(t.sessions, id) } // close all the links for node, link := range t.links { link.Send(&transport.Message{ Header: map[string]string{ "Micro-Tunnel": "close", "Micro-Tunnel-Id": t.id, }, }) link.Close() delete(t.links, node) } // close the listener // this appears to be blocking return t.listener.Close() } // pickLink will pick the best link based on connectivity, delay, rate and length func (t *tun) pickLink(links []*link) *link { var metric float64 var chosen *link // find the best link for i, link := range links { // don't use disconnected or errored links if link.State() != "connected" { continue } // get the link state info d := float64(link.Delay()) l := float64(link.Length()) r := link.Rate() // metric = delay x length x rate m := d * l * r // first link so just and go if i == 0 { metric = m chosen = link continue } // we found a better metric if m < metric { metric = m chosen = link } } // if there's no link we're just going to mess around if chosen == nil { i := rand.Intn(len(links)) return links[i] } // we chose the link with; // the lowest delay e.g least messages queued // the lowest rate e.g the least messages flowing // the lowest length e.g the smallest roundtrip time return chosen } func (t *tun) Address() string { t.RLock() defer t.RUnlock() if !t.connected { return t.options.Address } return t.listener.Addr() } // Close the tunnel func (t *tun) Close() error { t.Lock() defer t.Unlock() if !t.connected { return nil } log.Debug("Tunnel closing") select { case <-t.closed: return nil default: close(t.closed) t.connected = false } // send a close message // we don't close the link // just the tunnel return t.close() } // Dial an address func (t *tun) Dial(channel string, opts ...DialOption) (Session, error) { log.Debugf("Tunnel dialing %s", channel) c, ok := t.newSession(channel, t.newSessionId()) if !ok { return nil, errors.New("error dialing " + channel) } // set remote c.remote = channel // set local c.local = "local" // outbound session c.outbound = true // get opts options := DialOptions{ Timeout: DefaultDialTimeout, } for _, o := range opts { o(&options) } // set the multicast option c.mode = options.Mode // set the dial timeout c.timeout = options.Timeout var links []*link // did we measure the rtt var measured bool t.RLock() // non multicast so we need to find the link for _, link := range t.links { // use the link specified it its available if id := options.Link; len(id) > 0 && link.id != id { continue } // get the channel lastMapped := link.getChannel(channel) // we have at least one channel mapping if !lastMapped.IsZero() { links = append(links, link) c.discovered = true } } t.RUnlock() // link not found and one was specified so error out if len(links) == 0 && len(options.Link) > 0 { // delete session and return error t.delSession(c.channel, c.session) log.Debugf("Tunnel deleting session %s %s: %v", c.session, c.channel, ErrLinkNotFound) return nil, ErrLinkNotFound } // discovered so set the link if not multicast if c.discovered && c.mode == Unicast { // pickLink will pick the best link link := t.pickLink(links) // set the link c.link = link.id } // if its not already discovered we need to attempt to do so if !c.discovered { // piggy back roundtrip nowRTT := time.Now() // attempt to discover the link err := c.Discover() if err != nil { t.delSession(c.channel, c.session) log.Debugf("Tunnel deleting session %s %s: %v", c.session, c.channel, err) return nil, err } // set roundtrip d := time.Since(nowRTT) // set the link time t.RLock() link, ok := t.links[c.link] t.RUnlock() if ok { // set the rountrip time link.setRTT(d) // set measured to true measured = true } } // return early if its not unicast // we will not call "open" for multicast if c.mode != Unicast { return c, nil } // Note: we go no further for multicast or broadcast. // This is a unicast session so we call "open" and wait // for an "accept" // reset now in case we use it now := time.Now() // try to open the session if err := c.Open(); err != nil { // delete the session t.delSession(c.channel, c.session) log.Debugf("Tunnel deleting session %s %s: %v", c.session, c.channel, err) return nil, err } // set time take to open d := time.Since(now) // if we haven't measured the roundtrip do it now if !measured { // set the link time t.RLock() link, ok := t.links[c.link] t.RUnlock() if ok { // set the rountrip time link.setRTT(d) } } return c, nil } // Accept a connection on the address func (t *tun) Listen(channel string, opts ...ListenOption) (Listener, error) { log.Debugf("Tunnel listening on %s", channel) var options ListenOptions for _, o := range opts { o(&options) } // create a new session by hashing the address c, ok := t.newSession(channel, "listener") if !ok { return nil, errors.New("already listening on " + channel) } // delete function removes the session when closed delFunc := func() { t.delSession(channel, "listener") } // set remote. it will be replaced by the first message received c.remote = "remote" // set local c.local = channel // set mode c.mode = options.Mode tl := &tunListener{ channel: channel, // tunnel token token: t.token, // the accept channel accept: make(chan *session, 128), // the channel to close closed: make(chan bool), // tunnel closed channel tunClosed: t.closed, // the listener session session: c, // delete session delFunc: delFunc, } // this kicks off the internal message processor // for the listener so it can create pseudo sessions // per session if they do not exist or pass messages // to the existign sessions go tl.process() // announces the listener channel to others go tl.announce() // return the listener return tl, nil } func (t *tun) Links() []Link { t.RLock() defer t.RUnlock() links := make([]Link, 0, len(t.links)) for _, link := range t.links { links = append(links, link) } return links } func (t *tun) String() string { return "mucp" }