flow improvements (#52)

* flow improvements

Signed-off-by: Vasiliy Tolstov <v.tolstov@unistack.org>

client/noop.go

@@ -221,5 +221,8 @@ func (n *noopClient) Publish(ctx context.Context, p Message, opts ...PublishOpti
 	return n.opts.Broker.Publish(ctx, topic, &broker.Message{
 		Header: md,
 		Body:   body,
-	}, broker.PublishContext(options.Context))
+	},
+		broker.PublishContext(options.Context),
+		broker.PublishBodyOnly(options.BodyOnly),
+	)
 }

client/options.go

@@ -118,6 +118,8 @@ func NewPublishOptions(opts ...PublishOption) PublishOptions {
 
 // PublishOptions holds publish options
 type PublishOptions struct {
+	// BodyOnly will publish only message body
+	BodyOnly bool
 	// Context used for external options
 	Context context.Context
 	// Exchange topic exchange name
@@ -377,6 +379,13 @@ func WithExchange(e string) PublishOption {
 	}
 }
 
+// WithBodyOnly publish only message body
+func WithBodyOnly(b bool) PublishOption {
+	return func(o *PublishOptions) {
+		o.BodyOnly = b
+	}
+}
+
 // PublishContext sets the context in publish options
 func PublishContext(ctx context.Context) PublishOption {
 	return func(o *PublishOptions) {

config/config.go

@@ -20,25 +20,27 @@ var (
 
 // Config is an interface abstraction for dynamic configuration
 type Config interface {
+	// Name returns name of config
 	Name() string
 	// Init the config
 	Init(opts ...Option) error
 	// Options in the config
 	Options() Options
 	// Load config from sources
-	Load(context.Context) error
+	Load(context.Context, ...LoadOption) error
 	// Save config to sources
-	Save(context.Context) error
+	Save(context.Context, ...SaveOption) error
 	// Watch a value for changes
-	//	Watch(interface{}) (Watcher, error)
+	//Watch(context.Context) (Watcher, error)
+	// String returns config type name
 	String() string
 }
 
 // Watcher is the config watcher
-// type Watcher interface {
-//	Next() (, error)
-//	Stop() error
-// }
+type Watcher interface {
+	// Next() (, error)
+	Stop() error
+}
 
 // Load loads config from config sources
 func Load(ctx context.Context, cs ...Config) error {

config/default.go

@@ -25,18 +25,27 @@ func (c *defaultConfig) Init(opts ...Option) error {
 	return nil
 }
 
-func (c *defaultConfig) Load(ctx context.Context) error {
+func (c *defaultConfig) Load(ctx context.Context, opts ...LoadOption) error {
 	for _, fn := range c.opts.BeforeLoad {
 		if err := fn(ctx, c); err != nil && !c.opts.AllowFail {
 			return err
 		}
 	}
 
+	options := NewLoadOptions(opts...)
+	mopts := []func(*mergo.Config){mergo.WithTypeCheck}
+	if options.Override {
+		mopts = append(mopts, mergo.WithOverride)
+	}
+	if options.Append {
+		mopts = append(mopts, mergo.WithAppendSlice)
+	}
+
 	src, err := rutil.Zero(c.opts.Struct)
 	if err == nil {
 		valueOf := reflect.ValueOf(src)
 		if err = c.fillValues(valueOf); err == nil {
-			err = mergo.Merge(c.opts.Struct, src, mergo.WithOverride, mergo.WithTypeCheck, mergo.WithAppendSlice)
+			err = mergo.Merge(c.opts.Struct, src, mopts...)
 		}
 	}
 
@@ -232,7 +241,7 @@ func (c *defaultConfig) fillValues(valueOf reflect.Value) error {
 	return nil
 }
 
-func (c *defaultConfig) Save(ctx context.Context) error {
+func (c *defaultConfig) Save(ctx context.Context, opts ...SaveOption) error {
 	for _, fn := range c.opts.BeforeSave {
 		if err := fn(ctx, c); err != nil && !c.opts.AllowFail {
 			return err

config/options.go

@@ -57,6 +57,52 @@ func NewOptions(opts ...Option) Options {
 	return options
 }
 
+// LoadOption function signature
+type LoadOption func(o *LoadOptions)
+
+// LoadOptions struct
+type LoadOptions struct {
+	Override bool
+	Append   bool
+}
+
+func NewLoadOptions(opts ...LoadOption) LoadOptions {
+	options := LoadOptions{}
+	for _, o := range opts {
+		o(&options)
+	}
+	return options
+}
+
+// LoadOverride override values when load
+func LoadOverride(b bool) LoadOption {
+	return func(o *LoadOptions) {
+		o.Override = b
+	}
+}
+
+// LoadAppend override values when load
+func LoadAppend(b bool) LoadOption {
+	return func(o *LoadOptions) {
+		o.Append = b
+	}
+}
+
+// SaveOption function signature
+type SaveOption func(o *SaveOptions)
+
+// SaveOptions struct
+type SaveOptions struct {
+}
+
+func NewSaveOptions(opts ...SaveOption) SaveOptions {
+	options := SaveOptions{}
+	for _, o := range opts {
+		o(&options)
+	}
+	return options
+}
+
 // AllowFail allows config source to fail
 func AllowFail(b bool) Option {
 	return func(o *Options) {

flow/context.go

@@ -0,0 +1,34 @@
+package flow
+
+import (
+	"context"
+)
+
+type flowKey struct{}
+
+// FromContext returns Flow from context
+func FromContext(ctx context.Context) (Flow, bool) {
+	if ctx == nil {
+		return nil, false
+	}
+	c, ok := ctx.Value(flowKey{}).(Flow)
+	return c, ok
+}
+
+// NewContext stores Flow to context
+func NewContext(ctx context.Context, f Flow) context.Context {
+	if ctx == nil {
+		ctx = context.Background()
+	}
+	return context.WithValue(ctx, flowKey{}, f)
+}
+
+// SetOption returns a function to setup a context with given value
+func SetOption(k, v interface{}) Option {
+	return func(o *Options) {
+		if o.Context == nil {
+			o.Context = context.Background()
+		}
+		o.Context = context.WithValue(o.Context, k, v)
+	}
+}

flow/default.go

@@ -0,0 +1,319 @@
+package flow
+
+import (
+	"context"
+	"fmt"
+	"sync"
+
+	"github.com/google/uuid"
+	"github.com/silas/dag"
+	"github.com/unistack-org/micro/v3/client"
+	"github.com/unistack-org/micro/v3/codec"
+)
+
+type microFlow struct {
+	opts Options
+}
+
+type microWorkflow struct {
+	id   string
+	g    *dag.AcyclicGraph
+	init bool
+	sync.RWMutex
+	opts  Options
+	steps map[string]Step
+}
+
+func (w *microWorkflow) ID() string {
+	return w.id
+}
+
+func (w *microWorkflow) Steps() [][]Step {
+	return nil
+}
+
+func (w *microWorkflow) AppendSteps(ctx context.Context, steps ...Step) error {
+	return nil
+}
+
+func (w *microWorkflow) RemoveSteps(ctx context.Context, steps ...Step) error {
+	return nil
+}
+
+func (w *microWorkflow) Execute(ctx context.Context, req interface{}, opts ...ExecuteOption) (string, error) {
+	w.Lock()
+	if !w.init {
+		if err := w.g.Validate(); err != nil {
+			w.Unlock()
+			return "", err
+		}
+		w.g.TransitiveReduction()
+		w.init = true
+	}
+	w.Unlock()
+
+	uid, err := uuid.NewRandom()
+	if err != nil {
+		return "", err
+	}
+
+	options := NewExecuteOptions(opts...)
+	var steps [][]Step
+	fn := func(n dag.Vertex, idx int) error {
+		if idx == 0 {
+			steps = make([][]Step, 1)
+			steps[0] = make([]Step, 0, 1)
+		} else if idx >= len(steps) {
+			tsteps := make([][]Step, idx+1)
+			copy(tsteps, steps)
+			steps = tsteps
+			steps[idx] = make([]Step, 0, 1)
+		}
+		steps[idx] = append(steps[idx], n.(Step))
+		return nil
+	}
+
+	var root dag.Vertex
+	if options.Start != "" {
+		var ok bool
+		w.RLock()
+		root, ok = w.steps[options.Start]
+		w.RUnlock()
+		if !ok {
+			return "", ErrStepNotExists
+		}
+	} else {
+		root, err = w.g.Root()
+		if err != nil {
+			return "", err
+		}
+	}
+	if options.Reverse {
+		err = w.g.SortedReverseDepthFirstWalk([]dag.Vertex{root}, fn)
+	} else {
+		err = w.g.SortedDepthFirstWalk([]dag.Vertex{root}, fn)
+	}
+	if err != nil {
+		return "", err
+	}
+
+	var wg sync.WaitGroup
+	cherr := make(chan error, 1)
+	defer close(cherr)
+
+	nctx, cancel := context.WithCancel(ctx)
+	defer cancel()
+	nopts := make([]ExecuteOption, 0, len(opts)+5)
+	nopts = append(nopts, ExecuteClient(w.opts.Client), ExecuteTracer(w.opts.Tracer), ExecuteLogger(w.opts.Logger), ExecuteMeter(w.opts.Meter), ExecuteStore(w.opts.Store))
+
+	go func() {
+		for idx := range steps {
+			wg.Add(len(steps[idx]))
+			for nidx := range steps[idx] {
+				go func(step Step) {
+					defer wg.Done()
+					if err = step.Execute(nctx, req, nopts...); err != nil {
+						cherr <- err
+						cancel()
+					}
+				}(steps[idx][nidx])
+			}
+			wg.Wait()
+		}
+		cherr <- nil
+	}()
+
+	err = <-cherr
+
+	return uid.String(), err
+}
+
+func NewFlow(opts ...Option) Flow {
+	options := NewOptions(opts...)
+	return &microFlow{opts: options}
+}
+
+func (f *microFlow) Options() Options {
+	return f.opts
+}
+
+func (f *microFlow) Init(opts ...Option) error {
+	for _, o := range opts {
+		o(&f.opts)
+	}
+	if err := f.opts.Client.Init(); err != nil {
+		return err
+	}
+	if err := f.opts.Tracer.Init(); err != nil {
+		return err
+	}
+	if err := f.opts.Logger.Init(); err != nil {
+		return err
+	}
+	if err := f.opts.Meter.Init(); err != nil {
+		return err
+	}
+	if err := f.opts.Store.Init(); err != nil {
+		return err
+	}
+	return nil
+}
+
+func (f *microFlow) WorkflowList(ctx context.Context) ([]Workflow, error) {
+	return nil, nil
+}
+
+func (f *microFlow) WorkflowCreate(ctx context.Context, id string, steps ...Step) (Workflow, error) {
+	w := &microWorkflow{opts: f.opts, id: id, g: &dag.AcyclicGraph{}, steps: make(map[string]Step, len(steps))}
+
+	for _, s := range steps {
+		w.steps[s.String()] = s
+		w.g.Add(s)
+	}
+
+	for _, dst := range steps {
+		for _, req := range dst.Requires() {
+			src, ok := w.steps[req]
+			if !ok {
+				return nil, ErrStepNotExists
+			}
+			w.g.Connect(dag.BasicEdge(src, dst))
+		}
+	}
+
+	if err := w.g.Validate(); err != nil {
+		return nil, err
+	}
+	w.g.TransitiveReduction()
+
+	w.init = true
+
+	return w, nil
+}
+
+func (f *microFlow) WorkflowRemove(ctx context.Context, id string) error {
+	return nil
+}
+
+func (f *microFlow) WorkflowSave(ctx context.Context, w Workflow) error {
+	return nil
+}
+
+func (f *microFlow) WorkflowLoad(ctx context.Context, id string) (Workflow, error) {
+	return nil, nil
+}
+
+type microCallStep struct {
+	opts    StepOptions
+	service string
+	method  string
+}
+
+func (s *microCallStep) ID() string {
+	return s.String()
+}
+
+func (s *microCallStep) Options() StepOptions {
+	return s.opts
+}
+
+func (s *microCallStep) Endpoint() string {
+	return s.method
+}
+
+func (s *microCallStep) Requires() []string {
+	return s.opts.Requires
+}
+
+func (s *microCallStep) Require(steps ...Step) error {
+	for _, step := range steps {
+		s.opts.Requires = append(s.opts.Requires, step.String())
+	}
+	return nil
+}
+
+func (s *microCallStep) String() string {
+	if s.opts.ID != "" {
+		return s.opts.ID
+	}
+	return fmt.Sprintf("%s.%s", s.service, s.method)
+}
+
+func (s *microCallStep) Name() string {
+	return s.String()
+}
+
+func (s *microCallStep) Hashcode() interface{} {
+	return s.String()
+}
+
+func (s *microCallStep) Execute(ctx context.Context, req interface{}, opts ...ExecuteOption) error {
+	options := NewExecuteOptions(opts...)
+	if options.Client == nil {
+		return fmt.Errorf("client not set")
+	}
+	rsp := &codec.Frame{}
+	copts := []client.CallOption{client.WithRetries(0)}
+	if options.Timeout > 0 {
+		copts = append(copts, client.WithRequestTimeout(options.Timeout), client.WithDialTimeout(options.Timeout))
+	}
+	err := options.Client.Call(ctx, options.Client.NewRequest(s.service, s.method, req), rsp)
+	return err
+}
+
+type microPublishStep struct {
+	opts  StepOptions
+	topic string
+}
+
+func (s *microPublishStep) ID() string {
+	return s.String()
+}
+
+func (s *microPublishStep) Options() StepOptions {
+	return s.opts
+}
+
+func (s *microPublishStep) Endpoint() string {
+	return s.topic
+}
+
+func (s *microPublishStep) Requires() []string {
+	return s.opts.Requires
+}
+
+func (s *microPublishStep) Require(steps ...Step) error {
+	for _, step := range steps {
+		s.opts.Requires = append(s.opts.Requires, step.String())
+	}
+	return nil
+}
+
+func (s *microPublishStep) String() string {
+	if s.opts.ID != "" {
+		return s.opts.ID
+	}
+	return fmt.Sprintf("%s", s.topic)
+}
+
+func (s *microPublishStep) Name() string {
+	return s.String()
+}
+
+func (s *microPublishStep) Hashcode() interface{} {
+	return s.String()
+}
+
+func (s *microPublishStep) Execute(ctx context.Context, req interface{}, opts ...ExecuteOption) error {
+	return nil
+}
+
+func NewCallStep(service string, method string, opts ...StepOption) Step {
+	options := NewStepOptions(opts...)
+	return &microCallStep{service: service, method: method, opts: options}
+}
+
+func NewPublishStep(topic string, opts ...StepOption) Step {
+	options := NewStepOptions(opts...)
+	return &microPublishStep{topic: topic, opts: options}
+}

flow/flow.go

@@ -1,17 +1,59 @@
 // Package flow is an interface used for saga pattern microservice workflow
 package flow
 
+import (
+	"context"
+	"errors"
+)
+
+var (
+	ErrStepNotExists = errors.New("step not exists")
+)
+
+// Step represents dedicated workflow step
 type Step interface {
+	// ID returns step id
+	ID() string
 	// Endpoint returns rpc endpoint service_name.service_method or broker topic
 	Endpoint() string
+	// Execute step run
+	Execute(ctx context.Context, req interface{}, opts ...ExecuteOption) error
+	// Requires returns dependent steps
+	Requires() []string
+	// Options returns step options
+	Options() StepOptions
+	// Require add required steps
+	Require(steps ...Step) error
+	// String
+	String() string
 }
 
+// Workflow contains all steps to execute
 type Workflow interface {
+	// ID returns id of the workflow
+	ID() string
+	// Steps returns steps slice where parallel steps returned on the same level
 	Steps() [][]Step
-	Stop() error
+	// Execute workflow with args, return execution id and error
+	Execute(ctx context.Context, req interface{}, opts ...ExecuteOption) (string, error)
+	// RemoveSteps remove steps from workflow
+	RemoveSteps(ctx context.Context, steps ...Step) error
+	// AppendSteps append steps to workflow
+	AppendSteps(ctx context.Context, steps ...Step) error
 }
 
+// Flow the base interface to interact with workflows
 type Flow interface {
-	Start(Workflow) error
-	Stop(Workflow)
+	// Options returns options
+	Options() Options
+	// Init initialize
+	Init(...Option) error
+	// WorkflowCreate creates new workflow with specific id and steps
+	WorkflowCreate(ctx context.Context, id string, steps ...Step) (Workflow, error)
+	// WorkflowSave saves workflow
+	WorkflowSave(ctx context.Context, w Workflow) error
+	// WorkflowLoad loads workflow with specific id
+	WorkflowLoad(ctx context.Context, id string) (Workflow, error)
+	// WorkflowList lists all workflows
+	WorkflowList(ctx context.Context) ([]Workflow, error)
 }

flow/options.go

@@ -0,0 +1,222 @@
+package flow
+
+import (
+	"context"
+	"time"
+
+	"github.com/unistack-org/micro/v3/client"
+	"github.com/unistack-org/micro/v3/logger"
+	"github.com/unistack-org/micro/v3/meter"
+	"github.com/unistack-org/micro/v3/store"
+	"github.com/unistack-org/micro/v3/tracer"
+)
+
+// Option func
+type Option func(*Options)
+
+// Options server struct
+type Options struct {
+	// Context holds the external options and can be used for flow shutdown
+	Context context.Context
+	// Client holds the client.Client
+	Client client.Client
+	// Tracer holds the tracer
+	Tracer tracer.Tracer
+	// Logger holds the logger
+	Logger logger.Logger
+	// Meter holds the meter
+	Meter meter.Meter
+	// Store used for intermediate results
+	Store store.Store
+}
+
+// NewOptions returns new options struct with default or passed values
+func NewOptions(opts ...Option) Options {
+	options := Options{
+		Context: context.Background(),
+		Logger:  logger.DefaultLogger,
+		Meter:   meter.DefaultMeter,
+		Tracer:  tracer.DefaultTracer,
+		Client:  client.DefaultClient,
+	}
+
+	for _, o := range opts {
+		o(&options)
+	}
+
+	return options
+}
+
+// Logger sets the logger option
+func Logger(l logger.Logger) Option {
+	return func(o *Options) {
+		o.Logger = l
+	}
+}
+
+// Meter sets the meter option
+func Meter(m meter.Meter) Option {
+	return func(o *Options) {
+		o.Meter = m
+	}
+}
+
+// Client to use for sync/async communication
+func Client(c client.Client) Option {
+	return func(o *Options) {
+		o.Client = c
+	}
+}
+
+// Context specifies a context for the service.
+// Can be used to signal shutdown of the flow
+// Can be used for extra option values.
+func Context(ctx context.Context) Option {
+	return func(o *Options) {
+		o.Context = ctx
+	}
+}
+
+// Tracer mechanism for distributed tracking
+func Tracer(t tracer.Tracer) Option {
+	return func(o *Options) {
+		o.Tracer = t
+	}
+}
+
+// Store used for intermediate results
+func Store(s store.Store) Option {
+	return func(o *Options) {
+		o.Store = s
+	}
+}
+
+// WorflowOption signature
+type WorkflowOption func(*WorkflowOptions)
+
+// WorkflowOptions holds workflow options
+type WorkflowOptions struct {
+	ID      string
+	Context context.Context
+}
+
+// WorkflowID set workflow id
+func WorkflowID(id string) WorkflowOption {
+	return func(o *WorkflowOptions) {
+		o.ID = id
+	}
+}
+
+type ExecuteOptions struct {
+	// Client holds the client.Client
+	Client client.Client
+	// Tracer holds the tracer
+	Tracer tracer.Tracer
+	// Logger holds the logger
+	Logger logger.Logger
+	// Meter holds the meter
+	Meter meter.Meter
+	// Store used for intermediate results
+	Store store.Store
+	// Context can be used to abort execution or pass additional opts
+	Context context.Context
+	// Start step
+	Start string
+	// Reverse execution
+	Reverse bool
+	// Timeout for execution
+	Timeout time.Duration
+}
+
+type ExecuteOption func(*ExecuteOptions)
+
+func ExecuteClient(c client.Client) ExecuteOption {
+	return func(o *ExecuteOptions) {
+		o.Client = c
+	}
+}
+
+func ExecuteTracer(t tracer.Tracer) ExecuteOption {
+	return func(o *ExecuteOptions) {
+		o.Tracer = t
+	}
+}
+
+func ExecuteLogger(l logger.Logger) ExecuteOption {
+	return func(o *ExecuteOptions) {
+		o.Logger = l
+	}
+}
+
+func ExecuteMeter(m meter.Meter) ExecuteOption {
+	return func(o *ExecuteOptions) {
+		o.Meter = m
+	}
+}
+
+func ExecuteStore(s store.Store) ExecuteOption {
+	return func(o *ExecuteOptions) {
+		o.Store = s
+	}
+}
+
+func ExecuteContext(ctx context.Context) ExecuteOption {
+	return func(o *ExecuteOptions) {
+		o.Context = ctx
+	}
+}
+
+func ExecuteReverse(b bool) ExecuteOption {
+	return func(o *ExecuteOptions) {
+		o.Reverse = b
+	}
+}
+
+func ExecuteTimeout(td time.Duration) ExecuteOption {
+	return func(o *ExecuteOptions) {
+		o.Timeout = td
+	}
+}
+
+func NewExecuteOptions(opts ...ExecuteOption) ExecuteOptions {
+	options := ExecuteOptions{}
+	for _, o := range opts {
+		o(&options)
+	}
+	return options
+}
+
+type StepOptions struct {
+	ID       string
+	Context  context.Context
+	Requires []string
+	Fallback string
+}
+
+type StepOption func(*StepOptions)
+
+func NewStepOptions(opts ...StepOption) StepOptions {
+	options := StepOptions{Context: context.Background()}
+	for _, o := range opts {
+		o(&options)
+	}
+	return options
+}
+
+func StepID(id string) StepOption {
+	return func(o *StepOptions) {
+		o.ID = id
+	}
+}
+
+func StepRequires(steps ...string) StepOption {
+	return func(o *StepOptions) {
+		o.Requires = steps
+	}
+}
+
+func StepFallback(step string) StepOption {
+	return func(o *StepOptions) {
+		o.Fallback = step
+	}
+}

util/reflect/struct.go

@@ -7,6 +7,7 @@ import (
 	"reflect"
 	"regexp"
 	"strings"
+	"time"
 )
 
 // ErrInvalidParam specifies invalid url query params
@@ -14,6 +15,13 @@ var ErrInvalidParam = errors.New("invalid url query param provided")
 
 var bracketSplitter = regexp.MustCompile(`\[|\]`)
 
+var timeKind = reflect.TypeOf(time.Time{}).Kind()
+
+type StructField struct {
+	Field reflect.StructField
+	Value reflect.Value
+}
+
 func StructFieldByTag(src interface{}, tkey string, tval string) (interface{}, error) {
 	sv := reflect.ValueOf(src)
 	if sv.Kind() == reflect.Ptr {
@@ -98,8 +106,8 @@ func StructFieldByName(src interface{}, tkey string) (interface{}, error) {
 }
 
 // StructFields returns slice of struct fields
-func StructFields(src interface{}) ([]reflect.StructField, error) {
-	var fields []reflect.StructField
+func StructFields(src interface{}) ([]StructField, error) {
+	var fields []StructField
 
 	sv := reflect.ValueOf(src)
 	if sv.Kind() == reflect.Ptr {
@@ -116,14 +124,17 @@ func StructFields(src interface{}) ([]reflect.StructField, error) {
 		if !val.CanSet() || len(fld.PkgPath) != 0 {
 			continue
 		}
-		if val.Kind() == reflect.Struct {
+		switch val.Kind() {
+		case timeKind:
+			fields = append(fields, StructField{Field: fld, Value: val})
+		case reflect.Struct:
 			infields, err := StructFields(val.Interface())
 			if err != nil {
 				return nil, err
 			}
 			fields = append(fields, infields...)
-		} else {
-			fields = append(fields, fld)
+		default:
+			fields = append(fields, StructField{Field: fld, Value: val})
 		}
 	}