From 7ecff8bb181d44fc8236151de8db5fc64b13fae8 Mon Sep 17 00:00:00 2001 From: Vasiliy Tolstov Date: Sun, 22 Dec 2024 19:06:37 +0300 Subject: [PATCH 1/4] util/id: add uuid v8 Signed-off-by: Vasiliy Tolstov --- config/default_test.go | 3 - go.mod | 6 +- go.sum | 5 ++ network/options.go | 2 +- network/tunnel/options.go | 2 +- router/options.go | 2 +- server/options.go | 2 +- util/id/LICENSE | 22 ----- util/id/id.go | 164 ++++++++++++++++++++++++-------------- util/id/id_test.go | 11 +++ util/ring/buffer.go | 2 +- 11 files changed, 128 insertions(+), 93 deletions(-) delete mode 100644 util/id/LICENSE create mode 100644 util/id/id_test.go diff --git a/config/default_test.go b/config/default_test.go index a94c9cb2..7e57d846 100644 --- a/config/default_test.go +++ b/config/default_test.go @@ -8,7 +8,6 @@ import ( "time" "go.unistack.org/micro/v3/config" - mid "go.unistack.org/micro/v3/util/id" mtime "go.unistack.org/micro/v3/util/time" ) @@ -115,8 +114,6 @@ func TestDefault(t *testing.T) { if conf.IDValue == "" { t.Fatalf("id value empty") - } else if len(conf.IDValue) != mid.DefaultSize { - t.Fatalf("id value invalid: %s", conf.IDValue) } _ = conf // t.Logf("%#+v\n", conf) diff --git a/go.mod b/go.mod index 5c46a792..36368918 100644 --- a/go.mod +++ b/go.mod @@ -1,14 +1,14 @@ module go.unistack.org/micro/v3 -go 1.22.0 - -toolchain go1.23.4 +go 1.23.4 require ( dario.cat/mergo v1.0.1 github.com/DATA-DOG/go-sqlmock v1.5.0 github.com/KimMachineGun/automemlimit v0.6.1 + github.com/ash3in/uuidv8 v1.0.1 github.com/google/uuid v1.6.0 + github.com/matoous/go-nanoid v1.5.1 github.com/patrickmn/go-cache v2.1.0+incompatible github.com/silas/dag v0.0.0-20220518035006-a7e85ada93c5 go.uber.org/automaxprocs v1.6.0 diff --git a/go.sum b/go.sum index 63e7d695..e3dc57a4 100644 --- a/go.sum +++ b/go.sum @@ -4,6 +4,8 @@ github.com/DATA-DOG/go-sqlmock v1.5.0 h1:Shsta01QNfFxHCfpW6YH2STWB0MudeXXEWMr20O github.com/DATA-DOG/go-sqlmock v1.5.0/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM= github.com/KimMachineGun/automemlimit v0.6.1 h1:ILa9j1onAAMadBsyyUJv5cack8Y1WT26yLj/V+ulKp8= github.com/KimMachineGun/automemlimit v0.6.1/go.mod h1:T7xYht7B8r6AG/AqFcUdc7fzd2bIdBKmepfP2S1svPY= +github.com/ash3in/uuidv8 v1.0.1 h1:dIq1XRkWT8lGA7N5s7WRTB4V3k49WTBLvILz7aCLp80= +github.com/ash3in/uuidv8 v1.0.1/go.mod h1:EoyUgCtxNBnrnpc9efw5rVN1cQ+LFGCoJiFuD6maOMw= github.com/cilium/ebpf v0.16.0 h1:+BiEnHL6Z7lXnlGUsXQPPAE7+kenAd4ES8MQ5min0Ok= github.com/cilium/ebpf v0.16.0/go.mod h1:L7u2Blt2jMM/vLAVgjxluxtBKlz3/GWjB0dMOEngfwE= github.com/containerd/cgroups/v3 v3.0.4 h1:2fs7l3P0Qxb1nKWuJNFiwhp2CqiKzho71DQkDrHJIo4= @@ -40,6 +42,8 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/matoous/go-nanoid v1.5.1 h1:aCjdvTyO9LLnTIi0fgdXhOPPvOHjpXN6Ik9DaNjIct4= +github.com/matoous/go-nanoid v1.5.1/go.mod h1:zyD2a71IubI24efhpvkJz+ZwfwagzgSO6UNiFsZKN7U= github.com/mdlayher/netlink v1.7.2 h1:/UtM3ofJap7Vl4QWCPDGXY8d3GIY2UGSDbK+QWmY8/g= github.com/mdlayher/netlink v1.7.2/go.mod h1:xraEF7uJbxLhc5fpHL4cPe221LI2bdttWlU+ZGLfQSw= github.com/mdlayher/socket v0.4.1 h1:eM9y2/jlbs1M615oshPQOHZzj6R6wMT7bX5NPiQvn2U= @@ -58,6 +62,7 @@ github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH github.com/prashantv/gostub v1.1.0 h1:BTyx3RfQjRHnUWaGF9oQos79AlQ5k8WNktv7VGvVH4g= github.com/prashantv/gostub v1.1.0/go.mod h1:A5zLQHz7ieHGG7is6LLXLz7I8+3LZzsrV0P1IAHhP5U= github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII= +github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o= github.com/silas/dag v0.0.0-20220518035006-a7e85ada93c5 h1:G/FZtUu7a6NTWl3KUHMV9jkLAh/Rvtf03NWMHaEDl+E= github.com/silas/dag v0.0.0-20220518035006-a7e85ada93c5/go.mod h1:7RTUFBdIRC9nZ7/3RyRNH1bdqIShrDejd1YbLwgPS+I= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= diff --git a/network/options.go b/network/options.go index d4c68c6a..0430bfe9 100644 --- a/network/options.go +++ b/network/options.go @@ -119,7 +119,7 @@ func Tracer(t tracer.Tracer) Option { // NewOptions returns network default options func NewOptions(opts ...Option) Options { options := Options{ - ID: id.Must(), + ID: id.MustNew(), Name: "go.micro", Address: ":0", Logger: logger.DefaultLogger, diff --git a/network/tunnel/options.go b/network/tunnel/options.go index e3f33e97..3ae13a0e 100644 --- a/network/tunnel/options.go +++ b/network/tunnel/options.go @@ -164,7 +164,7 @@ func DialWait(b bool) DialOption { // NewOptions returns router default options with filled values func NewOptions(opts ...Option) Options { options := Options{ - ID: id.Must(), + ID: id.MustNew(), Address: DefaultAddress, Token: DefaultToken, Logger: logger.DefaultLogger, diff --git a/router/options.go b/router/options.go index ab99475f..09e1045a 100644 --- a/router/options.go +++ b/router/options.go @@ -80,7 +80,7 @@ func Name(n string) Option { // NewOptions returns router default options func NewOptions(opts ...Option) Options { options := Options{ - ID: id.Must(), + ID: id.MustNew(), Network: DefaultNetwork, Register: register.DefaultRegister, Logger: logger.DefaultLogger, diff --git a/server/options.go b/server/options.go index a45bc43d..2ff25a75 100644 --- a/server/options.go +++ b/server/options.go @@ -100,7 +100,7 @@ func NewOptions(opts ...Option) Options { Address: DefaultAddress, Name: DefaultName, Version: DefaultVersion, - ID: id.Must(), + ID: id.MustNew(), Namespace: DefaultNamespace, GracefulTimeout: DefaultGracefulTimeout, } diff --git a/util/id/LICENSE b/util/id/LICENSE deleted file mode 100644 index fec90ede..00000000 --- a/util/id/LICENSE +++ /dev/null @@ -1,22 +0,0 @@ -The MIT License (MIT) - -Copyright (c) 2018-2021 Matous Dzivjak -Copyright (c) 2021 Unistack LLC - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. diff --git a/util/id/id.go b/util/id/id.go index f7d34563..77c3acb4 100644 --- a/util/id/id.go +++ b/util/id/id.go @@ -1,112 +1,156 @@ package id import ( - "context" "crypto/rand" + "encoding/binary" "errors" - "math" + "fmt" + "time" - "go.unistack.org/micro/v3/logger" + uuidv8 "github.com/ash3in/uuidv8" + nanoid "github.com/matoous/go-nanoid" ) -// DefaultAlphabet is the alphabet used for ID characters by default -var DefaultAlphabet = []rune("6789BCDFGHJKLMNPQRTWbcdfghjkmnpqrtwz") +var generatedNode [6]byte -// DefaultSize is the size used for ID by default -// To get uuid like collision specify 21 -var DefaultSize = 16 - -// getMask generates bit mask used to obtain bits from the random bytes that are used to get index of random character -// from the alphabet. Example: if the alphabet has 6 = (110)_2 characters it is sufficient to use mask 7 = (111)_2 -func getMask(alphabetSize int) int { - for i := 1; i <= 8; i++ { - mask := (2 << uint(i)) - 1 - if mask >= alphabetSize-1 { - return mask - } +func init() { + if _, err := rand.Read(generatedNode[:]); err != nil { + panic(err) } - return 0 +} + +type Type int + +var zeroNode [6]byte + +const ( + TypeUnspecified Type = iota + TypeNanoid + TypeUUIDv8 +) + +// DefaultNanoidAlphabet is the alphabet used for ID characters by default +var DefaultNanoidAlphabet = "6789BCDFGHJKLMNPQRTWbcdfghjkmnpqrtwz" + +// DefaultNanoidSize is the size used for ID by default +// To get uuid like collision specify 21 +var DefaultNanoidSize = 16 + +type Generator struct { + opts Options +} + +func (g *Generator) MustNew() string { + id, err := g.New() + if err != nil { + panic(err) + } + return id +} + +func (g *Generator) New() (string, error) { + switch g.opts.Type { + case TypeNanoid: + if len(g.opts.NanoidAlphabet) == 0 || len(g.opts.NanoidAlphabet) > 255 { + return "", errors.New("invalid option, NanoidAlphabet must not be empty and contain no more than 255 chars") + } + if g.opts.NanoidSize <= 0 { + return "", errors.New("invalid option, NanoidSize must be positive integer") + } + + return nanoid.Generate(g.opts.NanoidAlphabet, g.opts.NanoidSize) + case TypeUUIDv8: + timestamp := uint64(time.Now().UnixNano()) + clockSeq := make([]byte, 2) + if _, err := rand.Read(clockSeq); err != nil { + return "", fmt.Errorf("failed to generate random clock sequence: %w", err) + } + clockSeqValue := binary.BigEndian.Uint16(clockSeq) & 0x0FFF // Mask to 12 bits + return uuidv8.NewWithParams(timestamp, clockSeqValue, g.opts.UUIDNode[:], uuidv8.TimestampBits48) + } + return "", errors.New("invalid option, Type unspecified") } // New returns new id or error func New(opts ...Option) (string, error) { options := NewOptions(opts...) - if len(options.Alphabet) == 0 || len(options.Alphabet) > 255 { - return "", errors.New("alphabet must not be empty and contain no more than 255 chars") - } - if options.Size <= 0 { - return "", errors.New("size must be positive integer") - } - - chars := options.Alphabet - - mask := getMask(len(chars)) - // estimate how many random bytes we will need for the ID, we might actually need more but this is tradeoff - // between average case and worst case - ceilArg := 1.6 * float64(mask*options.Size) / float64(len(options.Alphabet)) - step := int(math.Ceil(ceilArg)) - - id := make([]rune, options.Size) - bytes := make([]byte, step) - for j := 0; ; { - _, err := rand.Read(bytes) - if err != nil { - return "", err + switch options.Type { + case TypeNanoid: + if len(options.NanoidAlphabet) == 0 || len(options.NanoidAlphabet) > 255 { + return "", errors.New("invalid option, NanoidAlphabet must not be empty and contain no more than 255 chars") } - for i := 0; i < step; i++ { - currByte := bytes[i] & byte(mask) - if currByte < byte(len(chars)) { - id[j] = chars[currByte] - j++ - if j == options.Size { - return string(id[:options.Size]), nil - } - } + if options.NanoidSize <= 0 { + return "", errors.New("invalid option, NanoidSize must be positive integer") } + + return nanoid.Generate(options.NanoidAlphabet, options.NanoidSize) + case TypeUUIDv8: + timestamp := uint64(time.Now().UnixNano()) + clockSeq := make([]byte, 2) + if _, err := rand.Read(clockSeq); err != nil { + return "", fmt.Errorf("failed to generate random clock sequence: %w", err) + } + clockSeqValue := binary.BigEndian.Uint16(clockSeq) & 0x0FFF // Mask to 12 bits + return uuidv8.NewWithParams(timestamp, clockSeqValue, options.UUIDNode[:], uuidv8.TimestampBits48) } + + return "", errors.New("invalid option, Type unspecified") } // Must is the same as New but fatals on error -func Must(opts ...Option) string { +func MustNew(opts ...Option) string { id, err := New(opts...) if err != nil { - logger.DefaultLogger.Fatal(context.TODO(), "Must call is failed", err) + panic(err) } return id } // Options contains id deneration options type Options struct { - Alphabet []rune - Size int + Type Type + NanoidAlphabet string + NanoidSize int + UUIDNode [6]byte } // Option func signature type Option func(*Options) -// Alphabet specifies alphabet to use -func Alphabet(alphabet string) Option { +// WithNanoidAlphabet specifies alphabet to use +func WithNanoidAlphabet(alphabet string) Option { return func(o *Options) { - o.Alphabet = []rune(alphabet) + o.NanoidAlphabet = alphabet } } -// Size specifies id size -func Size(size int) Option { +// WithNanoidSize specifies generated id size +func WithNanoidSize(size int) Option { return func(o *Options) { - o.Size = size + o.NanoidSize = size + } +} + +// WithUUIDNode specifies node component for UUIDv8 +func WithUUIDNode(node [6]byte) Option { + return func(o *Options) { + o.UUIDNode = node } } // NewOptions returns new Options struct filled by opts func NewOptions(opts ...Option) Options { options := Options{ - Alphabet: DefaultAlphabet, - Size: DefaultSize, + Type: TypeUUIDv8, + NanoidAlphabet: DefaultNanoidAlphabet, + NanoidSize: DefaultNanoidSize, + UUIDNode: generatedNode, } + for _, o := range opts { o(&options) } + return options } diff --git a/util/id/id_test.go b/util/id/id_test.go new file mode 100644 index 00000000..01f8196f --- /dev/null +++ b/util/id/id_test.go @@ -0,0 +1,11 @@ +package id + +import "testing" + +func TestUUIDv8(t *testing.T) { + id, err := New() + if err != nil { + t.Fatal(err) + } + t.Logf("xxx %s\n", id) +} diff --git a/util/ring/buffer.go b/util/ring/buffer.go index 680c31dc..ab931a83 100644 --- a/util/ring/buffer.go +++ b/util/ring/buffer.go @@ -113,7 +113,7 @@ func (b *Buffer) Stream() (<-chan *Entry, chan bool) { defer b.Unlock() entries := make(chan *Entry, 128) - id := id.Must() + id := id.MustNew() stop := make(chan bool) b.streams[id] = &Stream{ -- 2.45.2 From d01561365c049fd105cec934f462ecb0163d1a18 Mon Sep 17 00:00:00 2001 From: Vasiliy Tolstov Date: Sun, 22 Dec 2024 21:53:04 +0300 Subject: [PATCH 2/4] logger/slog: write stacktrace on error if enabled, and always on fatal Signed-off-by: Vasiliy Tolstov --- logger/options.go | 3 --- logger/slog/slog.go | 2 +- logger/slog/slog_test.go | 18 ++++++++++++++++++ 3 files changed, 19 insertions(+), 4 deletions(-) diff --git a/logger/options.go b/logger/options.go index 4bccf085..85932135 100644 --- a/logger/options.go +++ b/logger/options.go @@ -30,7 +30,6 @@ type Options struct { StacktraceKey string // Name holds the logger name Name string - // Out holds the output writer Out io.Writer // Context holds exernal options @@ -39,12 +38,10 @@ type Options struct { Meter meter.Meter // TimeFunc used to obtain current time TimeFunc func() time.Time - // Fields holds additional metadata Fields []interface{} // ContextAttrFuncs contains funcs that executed before log func on context ContextAttrFuncs []ContextAttrFunc - // callerSkipCount number of frmaes to skip CallerSkipCount int // The logging level the logger should log diff --git a/logger/slog/slog.go b/logger/slog/slog.go index ce8b9d0e..3965efb6 100644 --- a/logger/slog/slog.go +++ b/logger/slog/slog.go @@ -270,7 +270,7 @@ func (s *slogLogger) printLog(ctx context.Context, lvl logger.Level, msg string, } } - if s.opts.AddStacktrace && lvl == logger.ErrorLevel { + if (s.opts.AddStacktrace || lvl == logger.FatalLevel) || (s.opts.AddStacktrace && lvl == logger.ErrorLevel) { stackInfo := make([]byte, 1024*1024) if stackSize := runtime.Stack(stackInfo, false); stackSize > 0 { traceLines := reTrace.Split(string(stackInfo[:stackSize]), -1) diff --git a/logger/slog/slog_test.go b/logger/slog/slog_test.go index 3604dc7f..5032a392 100644 --- a/logger/slog/slog_test.go +++ b/logger/slog/slog_test.go @@ -15,6 +15,24 @@ import ( "go.unistack.org/micro/v3/metadata" ) +func TestStacktrace(t *testing.T) { + ctx := context.TODO() + buf := bytes.NewBuffer(nil) + l := NewLogger(logger.WithLevel(logger.ErrorLevel), logger.WithOutput(buf), + WithHandlerFunc(slog.NewTextHandler), + logger.WithAddStacktrace(true), + ) + if err := l.Init(logger.WithFields("key1", "val1")); err != nil { + t.Fatal(err) + } + + l.Error(ctx, "msg1", errors.New("err")) + + if !bytes.Contains(buf.Bytes(), []byte(`slog_test.go:29`)) { + t.Fatalf("logger error not works, buf contains: %s", buf.Bytes()) + } +} + func TestWithFields(t *testing.T) { ctx := context.TODO() buf := bytes.NewBuffer(nil) -- 2.45.2 From d9604f792263e5b815514159b318af825655df08 Mon Sep 17 00:00:00 2001 From: Vasiliy Tolstov Date: Sun, 22 Dec 2024 21:57:58 +0300 Subject: [PATCH 3/4] util/id: add uuid v8 Signed-off-by: Vasiliy Tolstov --- util/id/id.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/util/id/id.go b/util/id/id.go index 77c3acb4..ce6a69a5 100644 --- a/util/id/id.go +++ b/util/id/id.go @@ -21,8 +21,6 @@ func init() { type Type int -var zeroNode [6]byte - const ( TypeUnspecified Type = iota TypeNanoid -- 2.45.2 From 86d99f086ec64df34c5d3e8ddc023b6955a2e717 Mon Sep 17 00:00:00 2001 From: Vasiliy Tolstov Date: Sun, 22 Dec 2024 22:09:45 +0300 Subject: [PATCH 4/4] logger/slog: close writer, and sleep Signed-off-by: Vasiliy Tolstov --- logger/slog/slog.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/logger/slog/slog.go b/logger/slog/slog.go index 3965efb6..63cc20bf 100644 --- a/logger/slog/slog.go +++ b/logger/slog/slog.go @@ -2,6 +2,7 @@ package slog import ( "context" + "io" "log/slog" "os" "reflect" @@ -10,6 +11,7 @@ import ( "strconv" "sync" "sync/atomic" + "time" "go.unistack.org/micro/v3/logger" "go.unistack.org/micro/v3/semconv" @@ -224,6 +226,10 @@ func (s *slogLogger) Error(ctx context.Context, msg string, attrs ...interface{} func (s *slogLogger) Fatal(ctx context.Context, msg string, attrs ...interface{}) { s.printLog(ctx, logger.FatalLevel, msg, attrs...) + if closer, ok := s.opts.Out.(io.Closer); ok { + closer.Close() + } + time.Sleep(1 * time.Second) os.Exit(1) } -- 2.45.2