micro/runtime/default.go

624 lines
13 KiB
Go
Raw Normal View History

2019-09-14 07:33:14 +03:00
package runtime
import (
"archive/tar"
"compress/gzip"
2019-09-14 07:33:14 +03:00
"errors"
"fmt"
"io"
"log"
"os"
"path/filepath"
"strings"
2019-09-14 07:33:14 +03:00
"sync"
"time"
"github.com/hpcloud/tail"
"github.com/micro/go-micro/v2/logger"
"github.com/micro/go-micro/v2/runtime/local/git"
2019-09-14 07:33:14 +03:00
)
type runtime struct {
sync.RWMutex
// options configure runtime
options Options
2019-09-14 18:07:36 +03:00
// used to stop the runtime
closed chan bool
// used to start new services
start chan *service
// indicates if we're running
running bool
// the service map
// TODO: track different versions of the same service
2019-09-14 07:33:14 +03:00
services map[string]*service
}
// NewRuntime creates new local runtime and returns it
func NewRuntime(opts ...Option) Runtime {
// get default options
options := Options{}
2019-09-14 07:33:14 +03:00
// apply requested options
for _, o := range opts {
o(&options)
}
2019-09-14 07:33:14 +03:00
2020-04-11 13:22:02 +03:00
// make the logs directory
path := filepath.Join(os.TempDir(), "micro", "logs")
_ = os.MkdirAll(path, 0755)
2019-09-14 07:33:14 +03:00
return &runtime{
options: options,
2019-09-14 07:33:14 +03:00
closed: make(chan bool),
2019-09-14 18:07:36 +03:00
start: make(chan *service, 128),
2019-09-14 07:33:14 +03:00
services: make(map[string]*service),
}
}
// @todo move this to runtime default
func (r *runtime) checkoutSourceIfNeeded(s *Service) error {
// Runtime service like config have no source.
// Skip checkout in that case
if len(s.Source) == 0 {
return nil
}
// @todo make this come from config
cpath := filepath.Join(os.TempDir(), "micro", "uploads", s.Source)
path := strings.ReplaceAll(cpath, ".tar.gz", "")
if ex, _ := exists(cpath); ex {
err := os.RemoveAll(path)
if err != nil {
return err
}
err = os.MkdirAll(path, 0777)
if err != nil {
return err
}
err = uncompress(cpath, path)
if err != nil {
return err
}
s.Source = path
return nil
}
source, err := git.ParseSourceLocal("", s.Source)
if err != nil {
return err
}
source.Ref = s.Version
err = git.CheckoutSource(os.TempDir(), source)
if err != nil {
return err
}
s.Source = source.FullPath
return nil
}
// modified version of: https://gist.github.com/mimoo/25fc9716e0f1353791f5908f94d6e726
func uncompress(src string, dst string) error {
file, err := os.OpenFile(src, os.O_RDWR|os.O_CREATE, 0666)
defer file.Close()
if err != nil {
return err
}
// ungzip
zr, err := gzip.NewReader(file)
if err != nil {
return err
}
// untar
tr := tar.NewReader(zr)
// uncompress each element
for {
header, err := tr.Next()
if err == io.EOF {
break // End of archive
}
if err != nil {
return err
}
target := header.Name
// validate name against path traversal
if !validRelPath(header.Name) {
return fmt.Errorf("tar contained invalid name error %q\n", target)
}
// add dst + re-format slashes according to system
target = filepath.Join(dst, header.Name)
// if no join is needed, replace with ToSlash:
// target = filepath.ToSlash(header.Name)
// check the type
switch header.Typeflag {
// if its a dir and it doesn't exist create it (with 0755 permission)
case tar.TypeDir:
if _, err := os.Stat(target); err != nil {
// @todo think about this:
// if we don't nuke the folder, we might end up with files from
// the previous decompress.
if err := os.MkdirAll(target, 0755); err != nil {
return err
}
}
// if it's a file create it (with same permission)
case tar.TypeReg:
// the truncating is probably unnecessary due to the `RemoveAll` of folders
// above
fileToWrite, err := os.OpenFile(target, os.O_TRUNC|os.O_CREATE|os.O_RDWR, os.FileMode(header.Mode))
if err != nil {
return err
}
// copy over contents
if _, err := io.Copy(fileToWrite, tr); err != nil {
return err
}
// manually close here after each file operation; defering would cause each file close
// to wait until all operations have completed.
fileToWrite.Close()
}
}
return nil
}
// check for path traversal and correct forward slashes
func validRelPath(p string) bool {
if p == "" || strings.Contains(p, `\`) || strings.HasPrefix(p, "/") || strings.Contains(p, "../") {
return false
}
return true
}
// Init initializes runtime options
func (r *runtime) Init(opts ...Option) error {
r.Lock()
defer r.Unlock()
2019-09-14 07:33:14 +03:00
for _, o := range opts {
o(&r.options)
2019-09-14 07:33:14 +03:00
}
return nil
}
// run runs the runtime management loop
func (r *runtime) run(events <-chan Event) {
2019-09-24 20:05:51 +03:00
t := time.NewTicker(time.Second * 5)
defer t.Stop()
// process event processes an incoming event
processEvent := func(event Event, service *service) error {
// get current vals
r.RLock()
name := service.Name
updated := service.updated
r.RUnlock()
// only process if the timestamp is newer
if !event.Timestamp.After(updated) {
return nil
}
if logger.V(logger.DebugLevel, logger.DefaultLogger) {
logger.Debugf("Runtime updating service %s", name)
}
// this will cause a delete followed by created
if err := r.Update(service.Service); err != nil {
return err
}
// update the local timestamp
r.Lock()
service.updated = updated
r.Unlock()
return nil
}
2019-09-24 20:05:51 +03:00
for {
select {
case <-t.C:
// check running services
r.RLock()
for _, service := range r.services {
if !service.ShouldStart() {
2019-09-24 20:05:51 +03:00
continue
}
// TODO: check service error
if logger.V(logger.DebugLevel, logger.DefaultLogger) {
logger.Debugf("Runtime starting %s", service.Name)
}
2019-09-24 20:05:51 +03:00
if err := service.Start(); err != nil {
if logger.V(logger.DebugLevel, logger.DefaultLogger) {
logger.Debugf("Runtime error starting %s: %v", service.Name, err)
}
2019-09-24 20:05:51 +03:00
}
}
r.RUnlock()
case service := <-r.start:
if !service.ShouldStart() {
2019-09-24 20:05:51 +03:00
continue
}
// TODO: check service error
if logger.V(logger.DebugLevel, logger.DefaultLogger) {
logger.Debugf("Runtime starting service %s", service.Name)
}
2019-09-24 20:05:51 +03:00
if err := service.Start(); err != nil {
if logger.V(logger.DebugLevel, logger.DefaultLogger) {
logger.Debugf("Runtime error starting service %s: %v", service.Name, err)
}
}
case event := <-events:
if logger.V(logger.DebugLevel, logger.DefaultLogger) {
logger.Debugf("Runtime received notification event: %v", event)
}
// NOTE: we only handle Update events for now
switch event.Type {
case Update:
if len(event.Service) > 0 {
r.RLock()
service, ok := r.services[fmt.Sprintf("%v:%v", event.Service, event.Version)]
r.RUnlock()
if !ok {
if logger.V(logger.DebugLevel, logger.DefaultLogger) {
logger.Debugf("Runtime unknown service: %s", event.Service)
}
continue
}
if err := processEvent(event, service); err != nil {
if logger.V(logger.DebugLevel, logger.DefaultLogger) {
logger.Debugf("Runtime error updating service %s: %v", event.Service, err)
}
}
continue
}
r.RLock()
services := r.services
r.RUnlock()
// if blank service was received we update all services
for _, service := range services {
if err := processEvent(event, service); err != nil {
if logger.V(logger.DebugLevel, logger.DefaultLogger) {
logger.Debugf("Runtime error updating service %s: %v", service.Name, err)
}
}
}
2019-09-24 20:05:51 +03:00
}
case <-r.closed:
if logger.V(logger.DebugLevel, logger.DefaultLogger) {
logger.Debugf("Runtime stopped")
}
2019-09-24 20:05:51 +03:00
return
}
}
}
func logFile(serviceName string) string {
2020-04-11 13:22:02 +03:00
// make the directory
name := strings.Replace(serviceName, "/", "-", -1)
2020-04-11 13:22:02 +03:00
path := filepath.Join(os.TempDir(), "micro", "logs")
return filepath.Join(path, fmt.Sprintf("%v.log", name))
}
func serviceKey(s *Service) string {
return fmt.Sprintf("%v:%v", s.Name, s.Version)
}
// Create creates a new service which is then started by runtime
2019-09-24 20:32:35 +03:00
func (r *runtime) Create(s *Service, opts ...CreateOption) error {
err := r.checkoutSourceIfNeeded(s)
if err != nil {
return err
}
2019-09-14 07:33:14 +03:00
r.Lock()
defer r.Unlock()
if _, ok := r.services[serviceKey(s)]; ok {
2020-01-17 17:14:47 +03:00
return errors.New("service already running")
2019-09-14 07:33:14 +03:00
}
2019-09-24 20:32:35 +03:00
var options CreateOptions
for _, o := range opts {
o(&options)
}
2019-11-29 14:35:00 +03:00
if len(options.Command) == 0 {
2020-03-20 01:38:37 +03:00
options.Command = []string{"go"}
options.Args = []string{"run", "."}
2019-09-24 20:32:35 +03:00
}
2019-12-24 20:51:30 +03:00
// create new service
service := newService(s, options)
f, err := os.OpenFile(logFile(service.Name), os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)
if err != nil {
log.Fatal(err)
}
if service.output != nil {
service.output = io.MultiWriter(service.output, f)
} else {
service.output = f
}
2019-12-24 20:51:30 +03:00
// start the service
if err := service.Start(); err != nil {
return err
}
2019-09-14 07:33:14 +03:00
2019-12-24 20:51:30 +03:00
// save service
r.services[serviceKey(s)] = service
2019-09-14 18:07:36 +03:00
2019-09-14 07:33:14 +03:00
return nil
}
2020-04-28 10:49:39 +03:00
// exists returns whether the given file or directory exists
func exists(path string) (bool, error) {
_, err := os.Stat(path)
if err == nil {
return true, nil
}
if os.IsNotExist(err) {
return false, nil
}
return true, err
}
// @todo: Getting existing lines is not supported yet.
// The reason for this is because it's hard to calculate line offset
// as opposed to character offset.
// This logger streams by default and only supports the `StreamCount` option.
func (r *runtime) Logs(s *Service, options ...LogsOption) (LogStream, error) {
lopts := LogsOptions{}
for _, o := range options {
o(&lopts)
}
ret := &logStream{
service: s.Name,
stream: make(chan LogRecord),
stop: make(chan bool),
}
2020-04-28 10:49:39 +03:00
fpath := logFile(s.Name)
if ex, err := exists(fpath); err != nil {
return nil, err
} else if !ex {
return nil, fmt.Errorf("Log file %v does not exists", fpath)
}
// have to check file size to avoid too big of a seek
fi, err := os.Stat(fpath)
if err != nil {
return nil, err
}
size := fi.Size()
2020-04-28 10:49:39 +03:00
whence := 2
// Multiply by length of an average line of log in bytes
offset := lopts.Count * 200
if offset > size {
offset = size
}
offset *= -1
2020-04-28 10:49:39 +03:00
t, err := tail.TailFile(fpath, tail.Config{Follow: lopts.Stream, Location: &tail.SeekInfo{
2020-04-28 10:49:39 +03:00
Whence: whence,
Offset: int64(offset),
}, Logger: tail.DiscardingLogger})
if err != nil {
return nil, err
}
2020-04-28 10:49:39 +03:00
ret.tail = t
go func() {
2020-04-28 10:49:39 +03:00
for {
select {
case line, ok := <-t.Lines:
if !ok {
ret.Stop()
return
}
ret.stream <- LogRecord{Message: line.Text}
case <-ret.stop:
return
}
}
2020-04-28 10:49:39 +03:00
}()
return ret, nil
}
type logStream struct {
tail *tail.Tail
service string
stream chan LogRecord
sync.Mutex
stop chan bool
err error
}
func (l *logStream) Chan() chan LogRecord {
return l.stream
}
func (l *logStream) Error() error {
return l.err
}
func (l *logStream) Stop() error {
l.Lock()
defer l.Unlock()
2020-04-28 10:49:39 +03:00
select {
case <-l.stop:
return nil
default:
close(l.stop)
2020-04-28 10:49:39 +03:00
close(l.stream)
err := l.tail.Stop()
if err != nil {
logger.Errorf("Error stopping tail: %v", err)
2020-04-28 10:49:39 +03:00
return err
}
}
return nil
}
// Read returns all instances of requested service
// If no service name is provided we return all the track services.
2019-11-29 14:35:00 +03:00
func (r *runtime) Read(opts ...ReadOption) ([]*Service, error) {
r.Lock()
defer r.Unlock()
gopts := ReadOptions{}
for _, o := range opts {
o(&gopts)
}
2019-11-29 14:35:00 +03:00
save := func(k, v string) bool {
if len(k) == 0 {
return true
}
return k == v
}
//nolint:prealloc
var services []*Service
2019-11-29 14:35:00 +03:00
for _, service := range r.services {
if !save(gopts.Service, service.Name) {
continue
}
2019-11-29 14:35:00 +03:00
if !save(gopts.Version, service.Version) {
continue
}
// TODO deal with service type
// no version has sbeen requested, just append the service
2019-11-29 14:35:00 +03:00
services = append(services, service.Service)
}
2019-11-29 14:35:00 +03:00
return services, nil
}
// Update attemps to update the service
func (r *runtime) Update(s *Service, opts ...UpdateOption) error {
err := r.checkoutSourceIfNeeded(s)
if err != nil {
return err
}
2020-04-16 18:50:24 +03:00
r.Lock()
service, ok := r.services[serviceKey(s)]
2020-04-16 18:50:24 +03:00
r.Unlock()
if !ok {
return errors.New("Service not found")
}
err = service.Stop()
2020-04-16 18:50:24 +03:00
if err != nil {
2019-10-29 15:29:21 +03:00
return err
}
2020-04-16 18:50:24 +03:00
return service.Start()
2019-10-29 15:29:21 +03:00
}
// Delete removes the service from the runtime and stops it
func (r *runtime) Delete(s *Service, opts ...DeleteOption) error {
r.Lock()
defer r.Unlock()
if logger.V(logger.DebugLevel, logger.DefaultLogger) {
logger.Debugf("Runtime deleting service %s", s.Name)
}
if s, ok := r.services[serviceKey(s)]; ok {
// check if running
if !s.Running() {
delete(r.services, s.key())
return nil
}
// otherwise stop it
if err := s.Stop(); err != nil {
return err
}
// delete it
delete(r.services, s.key())
return nil
}
return nil
}
// Start starts the runtime
func (r *runtime) Start() error {
2019-09-14 07:33:14 +03:00
r.Lock()
2019-09-24 20:05:51 +03:00
defer r.Unlock()
2019-09-14 07:33:14 +03:00
// already running
if r.running {
return nil
}
// set running
r.running = true
r.closed = make(chan bool)
2019-09-14 18:07:36 +03:00
var events <-chan Event
2020-01-16 16:34:04 +03:00
if r.options.Scheduler != nil {
var err error
2020-01-16 16:34:04 +03:00
events, err = r.options.Scheduler.Notify()
if err != nil {
// TODO: should we bail here?
if logger.V(logger.DebugLevel, logger.DefaultLogger) {
logger.Debugf("Runtime failed to start update notifier")
}
}
}
go r.run(events)
2019-09-14 07:33:14 +03:00
return nil
}
// Stop stops the runtime
2019-09-14 07:33:14 +03:00
func (r *runtime) Stop() error {
r.Lock()
defer r.Unlock()
if !r.running {
return nil
}
select {
case <-r.closed:
return nil
default:
close(r.closed)
// set not running
r.running = false
// stop all the services
for _, service := range r.services {
if logger.V(logger.DebugLevel, logger.DefaultLogger) {
logger.Debugf("Runtime stopping %s", service.Name)
}
2019-09-14 07:33:14 +03:00
service.Stop()
}
2020-01-16 16:34:04 +03:00
// stop the scheduler
if r.options.Scheduler != nil {
return r.options.Scheduler.Close()
}
2019-09-14 07:33:14 +03:00
}
return nil
}
// String implements stringer interface
func (r *runtime) String() string {
return "local"
}