From c567d1ceb31f3129cc958b2887cf231acc680742 Mon Sep 17 00:00:00 2001 From: Asim Aslam Date: Fri, 31 May 2019 00:26:34 +0100 Subject: [PATCH] Add runtime => run --- runtime/.travis.yml | 7 +++ runtime/README.md | 28 ++++++++++ runtime/package/docker/docker.go | 93 ++++++++++++++++++++++++++++++++ runtime/package/go/golang.go | 70 ++++++++++++++++++++++++ runtime/package/options.go | 15 ++++++ runtime/package/package.go | 34 ++++++++++++ runtime/process/options.go | 5 ++ runtime/process/os/os.go | 86 +++++++++++++++++++++++++++++ runtime/process/process.go | 37 +++++++++++++ runtime/source/git/git.go | 87 ++++++++++++++++++++++++++++++ runtime/source/go/golang.go | 93 ++++++++++++++++++++++++++++++++ runtime/source/options.go | 15 ++++++ runtime/source/source.go | 22 ++++++++ 13 files changed, 592 insertions(+) create mode 100644 runtime/.travis.yml create mode 100644 runtime/README.md create mode 100644 runtime/package/docker/docker.go create mode 100644 runtime/package/go/golang.go create mode 100644 runtime/package/options.go create mode 100644 runtime/package/package.go create mode 100644 runtime/process/options.go create mode 100644 runtime/process/os/os.go create mode 100644 runtime/process/process.go create mode 100644 runtime/source/git/git.go create mode 100644 runtime/source/go/golang.go create mode 100644 runtime/source/options.go create mode 100644 runtime/source/source.go diff --git a/runtime/.travis.yml b/runtime/.travis.yml new file mode 100644 index 00000000..1dbeee0b --- /dev/null +++ b/runtime/.travis.yml @@ -0,0 +1,7 @@ +language: go +go: +- 1.9.4 +- 1.10.x +notifications: + slack: + secure: fQJsxnChA2JDbdNti8cLiBlVj/ws38vaLyDJzkM+jhrU8NmdPbnjYu99lauDak0Yy9qdLNYUUO1KP6Kn8gl5VcbdC0++T3jPNCgIkJpvm/DJZKKtWlDXCavCDZ3EtCv8IhviD2dguHGtcX08/G6zc9V44GQu3sPWDdGdYnplh7OBIvicJZ5v8o6TR/WZ/4waGCLxeR84fIamXcZTlHGk3hJMKqCsb6urJX42rOlfSgjourb1yu8Tyg8uTXsO3Z7fgiZ73mYvZ1iD1v5thHB8T6bfB4g3U3KE53IjV9oV9bL6iDoiHbPWTIHD6yaZKXDbMxyw/sZ70GgxXH795pJeVBAWv9X6nH+aemwR72kgszvDLq64jxeAqJAmK5OqRs0YSQMKmvceSCIbEptCnpdl0KhhDZg/ojsfJOHyJb/SrEcntuvvig7O3Qyp9OtJdXJa5JnXi7Z2tPDXy+QnL/WSXUHU7JnDfQeQJC6QawxVGllxECcStIVb2/FrT0wnEinS66uhQhW48zLFcbJIFYNVsTlCMkTjXBkSvbjn3qSQr9IaMvpmDXhp9EC99Y5DgkPFP0S7aGjPCxfgrMHxqyxeEwmJQ+vYO6sZxnZbOzUMYVgPdAptQJ30QjovoF3lcQoBnZjvoKdunKE4qvSFYsMlxDV0SJhS5bc+eIXNIN5jYE8= diff --git a/runtime/README.md b/runtime/README.md new file mode 100644 index 00000000..600bdceb --- /dev/null +++ b/runtime/README.md @@ -0,0 +1,28 @@ +# Runtime + +A runtime for self governing services. + +## Overview + +In recent years we've started to develop complex architectures for the pipeline between writing code and running it. This +philosophy of build, run, manage or however many variations, has created a number of layers of abstraction that make it +all the more difficult to run code. + +Runtime manages the lifecycle of a service from source to running process. If the source is the *source of truth* then +everything in between running is wasted breath. Applications should be self governing and self sustaining. +To enable that we need libraries which make it possible. + +Runtime will fetch source code, build a binary and execute it. Any Go program that uses this library should be able +to run dependencies or itself with ease, with the ability to update itself as the source is updated. + +## Features + +- **Source** - Fetches source whether it be git, go, docker, etc +- **Package** - Compiles the source into a binary which can be executed +- **Process** - Executes a binary and creates a running process + +## Usage + +TODO + + diff --git a/runtime/package/docker/docker.go b/runtime/package/docker/docker.go new file mode 100644 index 00000000..e53010e5 --- /dev/null +++ b/runtime/package/docker/docker.go @@ -0,0 +1,93 @@ +// Package docker builds docker images +package docker + +import ( + "archive/tar" + "bytes" + "io/ioutil" + "os" + "path/filepath" + + "github.com/fsouza/go-dockerclient" + "github.com/micro/go-log" + "github.com/micro/go-run/package" +) + +type Packager struct { + Options packager.Options + Client *docker.Client +} + +func (d *Packager) Compile(s *packager.Source) (*packager.Binary, error) { + image := filepath.Join(s.Repository.Path, s.Repository.Name) + + buf := new(bytes.Buffer) + tw := tar.NewWriter(buf) + defer tw.Close() + + dockerFile := "Dockerfile" + + // open docker file + f, err := os.Open(filepath.Join(s.Repository.Path, s.Repository.Name, dockerFile)) + if err != nil { + return nil, err + } + // read docker file + by, err := ioutil.ReadAll(f) + if err != nil { + return nil, err + } + + tarHeader := &tar.Header{ + Name: dockerFile, + Size: int64(len(by)), + } + err = tw.WriteHeader(tarHeader) + if err != nil { + return nil, err + } + _, err = tw.Write(by) + if err != nil { + return nil, err + } + tr := bytes.NewReader(buf.Bytes()) + + err = d.Client.BuildImage(docker.BuildImageOptions{ + Name: image, + Dockerfile: dockerFile, + InputStream: tr, + OutputStream: ioutil.Discard, + RmTmpContainer: true, + SuppressOutput: true, + }) + if err != nil { + return nil, err + } + return &packager.Binary{ + Name: image, + Path: image, + Type: "docker", + Source: s, + }, nil +} + +func (d *Packager) Delete(b *packager.Binary) error { + image := filepath.Join(b.Path, b.Name) + return d.Client.RemoveImage(image) +} + +func NewPackager(opts ...packager.Option) packager.Packager { + options := packager.Options{} + for _, o := range opts { + o(&options) + } + endpoint := "unix:///var/run/docker.sock" + client, err := docker.NewClient(endpoint) + if err != nil { + log.Fatal(err) + } + return &Packager{ + Options: options, + Client: client, + } +} diff --git a/runtime/package/go/golang.go b/runtime/package/go/golang.go new file mode 100644 index 00000000..63572451 --- /dev/null +++ b/runtime/package/go/golang.go @@ -0,0 +1,70 @@ +// Package golang is a go package manager +package golang + +import ( + "os" + "os/exec" + "path/filepath" + + "github.com/micro/go-run/package" +) + +type Packager struct { + Options packager.Options + Cmd string + Path string +} + +// whichGo locates the go command +func whichGo() string { + // check GOROOT + if gr := os.Getenv("GOROOT"); len(gr) > 0 { + return filepath.Join(gr, "bin", "go") + } + + // check path + for _, p := range filepath.SplitList(os.Getenv("PATH")) { + bin := filepath.Join(p, "go") + if _, err := os.Stat(bin); err == nil { + return bin + } + } + + // best effort + return "go" +} + +func (g *Packager) Compile(s *packager.Source) (*packager.Binary, error) { + binary := filepath.Join(g.Path, s.Repository.Name) + source := filepath.Join(s.Repository.Path, s.Repository.Name) + + cmd := exec.Command(g.Cmd, "build", "-o", binary, source) + if err := cmd.Run(); err != nil { + return nil, err + } + return &packager.Binary{ + Name: s.Repository.Name, + Path: binary, + Type: "go", + Source: s, + }, nil +} + +func (g *Packager) Delete(b *packager.Binary) error { + binary := filepath.Join(b.Path, b.Name) + return os.Remove(binary) +} + +func NewPackager(opts ...packager.Option) packager.Packager { + options := packager.Options{ + Path: os.TempDir(), + } + for _, o := range opts { + o(&options) + } + return &Packager{ + Options: options, + Cmd: whichGo(), + Path: options.Path, + } +} diff --git a/runtime/package/options.go b/runtime/package/options.go new file mode 100644 index 00000000..a308c737 --- /dev/null +++ b/runtime/package/options.go @@ -0,0 +1,15 @@ +package packager + +type Options struct { + // local path to download source + Path string +} + +type Option func(o *Options) + +// Local path for repository +func Path(p string) Option { + return func(o *Options) { + o.Path = p + } +} diff --git a/runtime/package/package.go b/runtime/package/package.go new file mode 100644 index 00000000..39545544 --- /dev/null +++ b/runtime/package/package.go @@ -0,0 +1,34 @@ +// Package packager creates a binary image. Due to package being a reserved keyword we use packager. +package packager + +import ( + "github.com/micro/go-run/source" +) + +// Package builds binaries +type Packager interface { + // Compile builds a binary + Compile(*Source) (*Binary, error) + // Deletes the binary + Delete(*Binary) error +} + +// Source is the source of a build +type Source struct { + // Language is the language of code + Language string + // Location of the source + Repository *source.Repository +} + +// Binary is the representation of a binary +type Binary struct { + // Name of the binary + Name string + // Location of the binary + Path string + // Type of binary + Type string + // Source of the binary + Source *Source +} diff --git a/runtime/process/options.go b/runtime/process/options.go new file mode 100644 index 00000000..0dcc6882 --- /dev/null +++ b/runtime/process/options.go @@ -0,0 +1,5 @@ +package process + +type Options struct{} + +type Option func(o *Options) diff --git a/runtime/process/os/os.go b/runtime/process/os/os.go new file mode 100644 index 00000000..7f946e8c --- /dev/null +++ b/runtime/process/os/os.go @@ -0,0 +1,86 @@ +// Package os runs processes locally +package os + +import ( + "fmt" + "os" + "os/exec" + "strconv" + + "github.com/micro/go-run/process" +) + +type Process struct { +} + +func (p *Process) Exec(exe *process.Executable) error { + cmd := exec.Command(exe.Binary.Path) + return cmd.Run() +} + +func (p *Process) Fork(exe *process.Executable) (*process.PID, error) { + cmd := exec.Command(exe.Binary.Path) + if err := cmd.Start(); err != nil { + return nil, err + } + in, err := cmd.StdinPipe() + if err != nil { + return nil, err + } + out, err := cmd.StdoutPipe() + if err != nil { + return nil, err + } + er, err := cmd.StderrPipe() + if err != nil { + return nil, err + } + + return &process.PID{ + ID: fmt.Sprintf("%d", cmd.Process.Pid), + Input: in, + Output: out, + Error: er, + }, nil +} + +func (p *Process) Kill(pid *process.PID) error { + id, err := strconv.Atoi(pid.ID) + if err != nil { + return err + } + + pr, err := os.FindProcess(id) + if err != nil { + return err + } + + return pr.Kill() +} + +func (p *Process) Wait(pid *process.PID) error { + id, err := strconv.Atoi(pid.ID) + if err != nil { + return err + } + + pr, err := os.FindProcess(id) + if err != nil { + return err + } + + ps, err := pr.Wait() + if err != nil { + return err + } + + if ps.Success() { + return nil + } + + return fmt.Errorf(ps.String()) +} + +func NewProcess(opts ...process.Option) process.Process { + return &Process{} +} diff --git a/runtime/process/process.go b/runtime/process/process.go new file mode 100644 index 00000000..5306c79c --- /dev/null +++ b/runtime/process/process.go @@ -0,0 +1,37 @@ +// Package process executes a binary +package process + +import ( + "io" + + "github.com/micro/go-run/package" +) + +// Process manages a running process +type Process interface { + // Executes a process to completion + Exec(*Executable) error + // Creates a new process + Fork(*Executable) (*PID, error) + // Kills the process + Kill(*PID) error + // Waits for a process to exit + Wait(*PID) error +} + +type Executable struct { + // The executable binary + Binary *packager.Binary +} + +// PID is the running process +type PID struct { + // ID of the process + ID string + // Stdin + Input io.Writer + // Stdout + Output io.Reader + // Stderr + Error io.Reader +} diff --git a/runtime/source/git/git.go b/runtime/source/git/git.go new file mode 100644 index 00000000..a2590ec7 --- /dev/null +++ b/runtime/source/git/git.go @@ -0,0 +1,87 @@ +// Package git provides a git source +package git + +import ( + "os" + "path/filepath" + "strings" + + "github.com/micro/go-run/source" + "gopkg.in/src-d/go-git.v4" +) + +// Source retrieves source code +// An empty struct can be used +type Source struct { + Options source.Options +} + +func (g *Source) Fetch(url string) (*source.Repository, error) { + purl := url + + if parts := strings.Split(url, "://"); len(parts) > 1 { + purl = parts[len(parts)-1] + } + + name := filepath.Base(url) + path := filepath.Join(g.Options.Path, purl) + + _, err := git.PlainClone(path, false, &git.CloneOptions{ + URL: url, + }) + if err == nil { + return &source.Repository{ + Name: name, + Path: path, + URL: url, + }, nil + } + + // repo already exists + if err != git.ErrRepositoryAlreadyExists { + return nil, err + } + + // open repo + re, err := git.PlainOpen(path) + if err != nil { + return nil, err + } + + // update it + if err := re.Fetch(nil); err != nil { + return nil, err + } + + return &source.Repository{ + Name: name, + Path: path, + URL: url, + }, nil +} + +func (g *Source) Commit(r *source.Repository) error { + repo := filepath.Join(r.Path) + re, err := git.PlainOpen(repo) + if err != nil { + return err + } + return re.Push(nil) +} + +func (g *Source) String() string { + return "git" +} + +func NewSource(opts ...source.Option) *Source { + options := source.Options{ + Path: os.TempDir(), + } + for _, o := range opts { + o(&options) + } + + return &Source{ + Options: options, + } +} diff --git a/runtime/source/go/golang.go b/runtime/source/go/golang.go new file mode 100644 index 00000000..cbf42a9e --- /dev/null +++ b/runtime/source/go/golang.go @@ -0,0 +1,93 @@ +// Package golang is a source for Go +package golang + +import ( + "os" + "os/exec" + "path/filepath" + "strings" + + "github.com/micro/go-run/source" +) + +type Source struct { + Options source.Options + // Go Command + Cmd string + Path string +} + +func (g *Source) Fetch(url string) (*source.Repository, error) { + purl := url + + if parts := strings.Split(url, "://"); len(parts) > 1 { + purl = parts[len(parts)-1] + } + + // name of repo + name := filepath.Base(url) + // local path of repo + path := filepath.Join(g.Path, purl) + args := []string{"get", "-d", url, path} + + cmd := exec.Command(g.Cmd, args...) + if err := cmd.Run(); err != nil { + return nil, err + } + return &source.Repository{ + Name: name, + Path: path, + URL: url, + }, nil +} + +// Commit is not yet supported +func (g *Source) Commit(r *source.Repository) error { + return nil +} + +func (g *Source) String() string { + return "golang" +} + +// whichGo locates the go command +func whichGo() string { + // check GOROOT + if gr := os.Getenv("GOROOT"); len(gr) > 0 { + return filepath.Join(gr, "bin", "go") + } + + // check path + for _, p := range filepath.SplitList(os.Getenv("PATH")) { + bin := filepath.Join(p, "go") + if _, err := os.Stat(bin); err == nil { + return bin + } + } + + // best effort + return "go" +} + +func NewSource(opts ...source.Option) source.Source { + options := source.Options{ + Path: os.TempDir(), + } + for _, o := range opts { + o(&options) + } + + cmd := whichGo() + path := options.Path + + // point of no return + if len(cmd) == 0 { + panic("Could not find Go executable") + } + + return &Source{ + Options: options, + Cmd: cmd, + Path: path, + } +} diff --git a/runtime/source/options.go b/runtime/source/options.go new file mode 100644 index 00000000..2e842f33 --- /dev/null +++ b/runtime/source/options.go @@ -0,0 +1,15 @@ +package source + +type Options struct { + // local path to download source + Path string +} + +type Option func(o *Options) + +// Local path for repository +func Path(p string) Option { + return func(o *Options) { + o.Path = p + } +} diff --git a/runtime/source/source.go b/runtime/source/source.go new file mode 100644 index 00000000..2afa92e3 --- /dev/null +++ b/runtime/source/source.go @@ -0,0 +1,22 @@ +// Package source retrieves source code +package source + +// Source retrieves source code +type Source interface { + // Fetch repo from a url + Fetch(url string) (*Repository, error) + // Commit and upload repo + Commit(*Repository) error + // The sourcerer + String() string +} + +// Repository is the source repository +type Repository struct { + // Name or repo + Name string + // Local path where repo is stored + Path string + // URL from which repo was retrieved + URL string +}