From b1d3992f357d4379abbf20c52ba314690ba53a9e Mon Sep 17 00:00:00 2001 From: Aliaksandr Valialkin Date: Mon, 8 Apr 2019 16:29:16 +0300 Subject: [PATCH] Initial commit --- LICENSE | 22 +++++ README.md | 26 ++++++ go.mod | 3 + go.sum | 4 + metrics.go | 225 ++++++++++++++++++++++++++++++++++++++++++++++ summary.go | 145 ++++++++++++++++++++++++++++++ validator.go | 81 +++++++++++++++++ validator_test.go | 60 +++++++++++++ 8 files changed, 566 insertions(+) create mode 100644 LICENSE create mode 100644 README.md create mode 100644 go.mod create mode 100644 go.sum create mode 100644 metrics.go create mode 100644 summary.go create mode 100644 validator.go create mode 100644 validator_test.go diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..539b7a4 --- /dev/null +++ b/LICENSE @@ -0,0 +1,22 @@ +The MIT License (MIT) + +Copyright (c) 2019 VictoriaMetrics + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + diff --git a/README.md b/README.md new file mode 100644 index 0000000..9fc6922 --- /dev/null +++ b/README.md @@ -0,0 +1,26 @@ +[![Build Status](https://travis-ci.org/VictoriaMetrics/metrics.svg)](https://travis-ci.org/VictoriaMetrics/metrics) +[![GoDoc](https://godoc.org/github.com/VictoriaMetrics/metrics?status.svg)](http://godoc.org/github.com/VictoriaMetrics/metrics) +[![Go Report](https://goreportcard.com/badge/github.com/VictoriaMetrics/metrics)](https://goreportcard.com/report/github.com/VictoriaMetrics/metrics) +[![codecov](https://codecov.io/gh/VictoriaMetrics/metrics/branch/master/graph/badge.svg)](https://codecov.io/gh/VictoriaMetrics/metrics) + +# metrics - lightweight alternative to `github.com/prometheus/client_golang/prometheus`. + + +### Features + +* Lightweight. Has minimal number of third-party dependencies and all these deps are small. + See [this article](https://medium.com/@valyala/stripping-dependency-bloat-in-victoriametrics-docker-image-983fb5912b0d) for details. +* Easy to use. See the [API docs](http://godoc.org/github.com/VictoriaMetrics/metrics). +* Fast. + + +### Limitations + +* It doesn't implement advanced functionality from [github.com/prometheus/client_golang/prometheus](https://godoc.org/github.com/prometheus/client_golang/prometheus). + + +### Users + +* `Metrics` has been extracted from [VictoriaMetrics](https://github.com/VictoriaMetrics/VictoriaMetrics) sources. + See [this article](https://medium.com/devopslinks/victoriametrics-creating-the-best-remote-storage-for-prometheus-5d92d66787ac) + for more info about `VictoriaMetrics`. diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..7a4629b --- /dev/null +++ b/go.mod @@ -0,0 +1,3 @@ +module github.com/VictoriaMetrics/metrics + +require github.com/valyala/histogram v1.0.1 diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..9dd64f0 --- /dev/null +++ b/go.sum @@ -0,0 +1,4 @@ +github.com/valyala/fastrand v1.0.0 h1:LUKT9aKer2dVQNUi3waewTbKV+7H17kvWFNKs2ObdkI= +github.com/valyala/fastrand v1.0.0/go.mod h1:HWqCzkrkg6QXT8V2EXWvXCoow7vLwOFN002oeRzjapQ= +github.com/valyala/histogram v1.0.1 h1:FzA7n2Tz/wKRMejgu3PV1vw3htAklTjjuoI6z3d4KDg= +github.com/valyala/histogram v1.0.1/go.mod h1:lQy0xA4wUz2+IUnf97SivorsJIp8FxsnRd6x25q7Mto= diff --git a/metrics.go b/metrics.go new file mode 100644 index 0000000..8d844f4 --- /dev/null +++ b/metrics.go @@ -0,0 +1,225 @@ +// Package metrics implements Prometheus-compatible metrics for applications. +// +// This package is similar to https://github.com/prometheus/client_golang , +// but is simpler to use and is more lightweight. +package metrics + +import ( + "fmt" + "io" + "runtime" + "sort" + "sync" + "sync/atomic" + "time" + + "github.com/valyala/histogram" +) + +type gauge struct { + f func() float64 +} + +func (g *gauge) marshalTo(prefix string, w io.Writer) { + v := g.f() + fmt.Fprintf(w, "%s %g\n", prefix, v) +} + +// NewGauge creates a gauge with the given name, which calls f +// to obtain gauge value. +// +// name must be valid Prometheus-compatible metric with possible labels. +// For instance, +// +// * foo +// * foo{bar="baz"} +// * foo{bar="baz",aaa="b"} +// +// f must be safe for concurrent calls. +func NewGauge(name string, f func() float64) { + g := &gauge{ + f: f, + } + registerMetric(name, g) +} + +// Counter is a counter. +// +// It may be used as a gauge if Dec and Set are called. +type Counter struct { + n uint64 +} + +// NewCounter creates and returns new counter with the given name. +// +// name must be valid Prometheus-compatible metric with possible lables. +// For instance, +// +// * foo +// * foo{bar="baz"} +// * foo{bar="baz",aaa="b"} +// +// The returned counter is safe to use from concurrent goroutines. +func NewCounter(name string) *Counter { + c := &Counter{} + registerMetric(name, c) + return c +} + +func registerMetric(name string, m metric) { + if err := validateMetric(name); err != nil { + // Do not use logger.Panicf here, since it may be uninitialized yet. + panic(fmt.Errorf("BUG: invalid metric name %q: %s", name, err)) + } + metricsMapLock.Lock() + ok := isRegisteredMetric(metricsMap, name) + if !ok { + nm := namedMetric{ + name: name, + metric: m, + } + metricsMap = append(metricsMap, nm) + } + metricsMapLock.Unlock() + if ok { + // Do not use logger.Panicf here, since it may be uninitialized yet. + panic(fmt.Errorf("BUG: metric with name %q is already registered", name)) + } +} + +// Inc increments c. +func (c *Counter) Inc() { + atomic.AddUint64(&c.n, 1) +} + +// Dec decrements c. +func (c *Counter) Dec() { + atomic.AddUint64(&c.n, ^uint64(0)) +} + +// Add adds n to c. +func (c *Counter) Add(n int) { + atomic.AddUint64(&c.n, uint64(n)) +} + +// Get returns the current value for c. +func (c *Counter) Get() uint64 { + return atomic.LoadUint64(&c.n) +} + +// Set sets c value to n. +func (c *Counter) Set(n uint64) { + atomic.StoreUint64(&c.n, n) +} + +// marshalTo marshals c with the given prefix to w. +func (c *Counter) marshalTo(prefix string, w io.Writer) { + v := c.Get() + fmt.Fprintf(w, "%s %d\n", prefix, v) +} + +var ( + metricsMapLock sync.Mutex + metricsMap []namedMetric +) + +type namedMetric struct { + name string + metric metric +} + +func isRegisteredMetric(mm []namedMetric, name string) bool { + for _, nm := range mm { + if nm.name == name { + return true + } + } + return false +} + +func sortMetrics(mm []namedMetric) { + lessFunc := func(i, j int) bool { + return mm[i].name < mm[j].name + } + if !sort.SliceIsSorted(mm, lessFunc) { + sort.Slice(mm, lessFunc) + } +} + +type metric interface { + marshalTo(prefix string, w io.Writer) +} + +// WritePrometheus writes all the registered metrics in Prometheus format to w. +// +// If exposeProcessMetrics is true, then various `go_*` metrics are exposed +// for the current process. +func WritePrometheus(w io.Writer, exposeProcessMetrics bool) { + // Export user-defined metrics. + metricsMapLock.Lock() + sortMetrics(metricsMap) + for _, nm := range metricsMap { + nm.metric.marshalTo(nm.name, w) + } + metricsMapLock.Unlock() + + if !exposeProcessMetrics { + return + } + + // Export memory stats. + var ms runtime.MemStats + runtime.ReadMemStats(&ms) + fmt.Fprintf(w, `go_memstats_alloc_bytes %d`+"\n", ms.Alloc) + fmt.Fprintf(w, `go_memstats_alloc_bytes_total %d`+"\n", ms.TotalAlloc) + fmt.Fprintf(w, `go_memstats_buck_hash_sys_bytes %d`+"\n", ms.BuckHashSys) + fmt.Fprintf(w, `go_memstats_frees_total %d`+"\n", ms.Frees) + fmt.Fprintf(w, `go_memstats_gc_cpu_fraction %f`+"\n", ms.GCCPUFraction) + fmt.Fprintf(w, `go_memstats_gc_sys_bytes %d`+"\n", ms.GCSys) + fmt.Fprintf(w, `go_memstats_heap_alloc_bytes %d`+"\n", ms.HeapAlloc) + fmt.Fprintf(w, `go_memstats_heap_idle_bytes %d`+"\n", ms.HeapIdle) + fmt.Fprintf(w, `go_memstats_heap_inuse_bytes %d`+"\n", ms.HeapInuse) + fmt.Fprintf(w, `go_memstats_heap_objects %d`+"\n", ms.HeapObjects) + fmt.Fprintf(w, `go_memstats_heap_released_bytes %d`+"\n", ms.HeapReleased) + fmt.Fprintf(w, `go_memstats_heap_sys_bytes %d`+"\n", ms.HeapSys) + fmt.Fprintf(w, `go_memstats_last_gc_time_seconds %f`+"\n", float64(ms.LastGC)/1e9) + fmt.Fprintf(w, `go_memstats_lookups_total %d`+"\n", ms.Lookups) + fmt.Fprintf(w, `go_memstats_mallocs_total %d`+"\n", ms.Mallocs) + fmt.Fprintf(w, `go_memstats_mcache_inuse_bytes %d`+"\n", ms.MCacheInuse) + fmt.Fprintf(w, `go_memstats_mcache_sys_bytes %d`+"\n", ms.MCacheSys) + fmt.Fprintf(w, `go_memstats_mspan_inuse_bytes %d`+"\n", ms.MSpanInuse) + fmt.Fprintf(w, `go_memstats_mspan_sys_bytes %d`+"\n", ms.MSpanSys) + fmt.Fprintf(w, `go_memstats_next_gc_bytes %d`+"\n", ms.NextGC) + fmt.Fprintf(w, `go_memstats_other_sys_bytes %d`+"\n", ms.OtherSys) + fmt.Fprintf(w, `go_memstats_stack_inuse_bytes %d`+"\n", ms.StackInuse) + fmt.Fprintf(w, `go_memstats_stack_sys_bytes %d`+"\n", ms.StackSys) + fmt.Fprintf(w, `go_memstats_sys_bytes %d`+"\n", ms.Sys) + + fmt.Fprintf(w, `go_cgo_calls_count %d`+"\n", runtime.NumCgoCall()) + fmt.Fprintf(w, `go_cpu_count %d`+"\n", runtime.NumCPU()) + + gcPauses := histogram.NewFast() + for _, pauseNs := range ms.PauseNs[:] { + gcPauses.Update(float64(pauseNs) / 1e9) + } + phis := []float64{0, 0.25, 0.5, 0.75, 1} + quantiles := make([]float64, 0, len(phis)) + for i, q := range gcPauses.Quantiles(quantiles[:0], phis) { + fmt.Fprintf(w, `go_gc_duration_seconds{quantile="%g"} %f`+"\n", phis[i], q) + } + fmt.Fprintf(w, `go_gc_duration_seconds_sum %f`+"\n", float64(ms.PauseTotalNs)/1e9) + fmt.Fprintf(w, `go_gc_duration_seconds_count %d`+"\n", ms.NumGC) + fmt.Fprintf(w, `go_gc_forced_count %d`+"\n", ms.NumForcedGC) + + fmt.Fprintf(w, `go_gomaxprocs %d`+"\n", runtime.GOMAXPROCS(0)) + fmt.Fprintf(w, `go_goroutines %d`+"\n", runtime.NumGoroutine()) + numThread, _ := runtime.ThreadCreateProfile(nil) + fmt.Fprintf(w, `go_threads %d`+"\n", numThread) + + // Export build details. + fmt.Fprintf(w, "go_info{version=%q} 1\n", runtime.Version()) + fmt.Fprintf(w, "go_info_ext{compiler=%q, GOARCH=%q, GOOS=%q, GOROOT=%q} 1\n", + runtime.Compiler, runtime.GOARCH, runtime.GOOS, runtime.GOROOT()) +} + +var startTime = time.Now() diff --git a/summary.go b/summary.go new file mode 100644 index 0000000..7f2e560 --- /dev/null +++ b/summary.go @@ -0,0 +1,145 @@ +package metrics + +import ( + "fmt" + "io" + "sync" + "time" + + "github.com/valyala/histogram" +) + +const defaultSummaryWindow = 5 * time.Minute + +var defaultSummaryQuantiles = []float64{0.5, 0.9, 0.97, 0.99, 1} + +// Summary implements summary. +type Summary struct { + mu sync.Mutex + + curr *histogram.Fast + next *histogram.Fast + + quantiles []float64 + quantileValues []float64 +} + +// NewSummary creates and returns new summary with the given name. +// +// name must be valid Prometheus-compatible metric with possible lables. +// For instance, +// +// * foo +// * foo{bar="baz"} +// * foo{bar="baz",aaa="b"} +// +// The returned summary is safe to use from concurrent goroutines. +func NewSummary(name string) *Summary { + return NewSummaryExt(name, defaultSummaryWindow, defaultSummaryQuantiles) +} + +// NewSummaryExt creates and returns new summary with the given name, +// window and quantiles. +// +// name must be valid Prometheus-compatible metric with possible lables. +// For instance, +// +// * foo +// * foo{bar="baz"} +// * foo{bar="baz",aaa="b"} +// +// The returned summary is safe to use from concurrent goroutines. +func NewSummaryExt(name string, window time.Duration, quantiles []float64) *Summary { + s := &Summary{ + curr: histogram.NewFast(), + next: histogram.NewFast(), + quantiles: quantiles, + quantileValues: make([]float64, len(quantiles)), + } + registerSummary(s, window) + registerMetric(fmt.Sprintf("\x00%s", name), s) + for i, q := range quantiles { + quantileValueName := addTag(name, fmt.Sprintf(`quantile="%g"`, q)) + qv := &quantileValue{ + s: s, + idx: i, + } + registerMetric(quantileValueName, qv) + } + return s +} + +// Update updates the summary. +func (s *Summary) Update(v float64) { + s.mu.Lock() + s.curr.Update(v) + s.next.Update(v) + s.mu.Unlock() +} + +// UpdateDuration updates request duration based on the given startTime. +func (s *Summary) UpdateDuration(startTime time.Time) { + d := time.Since(startTime).Seconds() + s.Update(d) +} + +func (s *Summary) marshalTo(prefix string, w io.Writer) { + // Just update s.quantileValues and don't write anything to w. + // s.quantileValues will be marshaled later via quantileValue.marshalTo. + s.updateQuantiles() +} + +func (s *Summary) updateQuantiles() { + s.mu.Lock() + s.quantileValues = s.curr.Quantiles(s.quantileValues[:0], s.quantiles) + s.mu.Unlock() +} + +type quantileValue struct { + s *Summary + idx int +} + +func (qv *quantileValue) marshalTo(prefix string, w io.Writer) { + qv.s.mu.Lock() + v := qv.s.quantileValues[qv.idx] + qv.s.mu.Unlock() + fmt.Fprintf(w, "%s %g\n", prefix, v) +} + +func addTag(name, tag string) string { + if len(name) == 0 || name[len(name)-1] != '}' { + return fmt.Sprintf("%s{%s}", name, tag) + } + return fmt.Sprintf("%s, %s}", name[:len(name)-1], tag) +} + +func registerSummary(s *Summary, window time.Duration) { + summariesLock.Lock() + summaries[window] = append(summaries[window], s) + if len(summaries[window]) == 1 { + go summariesSwapCron(window) + } + summariesLock.Unlock() +} + +func summariesSwapCron(window time.Duration) { + for { + time.Sleep(window / 2) + summariesLock.Lock() + for _, s := range summaries[window] { + s.mu.Lock() + tmp := s.curr + s.curr = s.next + s.next = tmp + s.next.Reset() + s.mu.Unlock() + } + summariesLock.Unlock() + } +} + +var ( + summaries = map[time.Duration][]*Summary{} + summariesLock sync.Mutex +) diff --git a/validator.go b/validator.go new file mode 100644 index 0000000..6b74d21 --- /dev/null +++ b/validator.go @@ -0,0 +1,81 @@ +package metrics + +import ( + "fmt" + "regexp" + "strings" +) + +func validateMetric(s string) error { + if len(s) == 0 { + return fmt.Errorf("metric cannot be empty") + } + if s[0] == 0 { + // Skip special case metrics. See Histogram for details. + return nil + } + n := strings.IndexByte(s, '{') + if n < 0 { + return validateIdent(s) + } + ident := s[:n] + s = s[n+1:] + if err := validateIdent(ident); err != nil { + return err + } + if len(s) == 0 || s[len(s)-1] != '}' { + return fmt.Errorf("missing closing curly brace at the end of %q", ident) + } + return validateTags(s[:len(s)-1]) +} + +func validateTags(s string) error { + if len(s) == 0 { + return nil + } + for { + n := strings.IndexByte(s, '=') + if n < 0 { + return fmt.Errorf("missing `=` after %q", s) + } + ident := s[:n] + s = s[n+1:] + if err := validateIdent(ident); err != nil { + return err + } + if len(s) == 0 || s[0] != '"' { + return fmt.Errorf("missing starting `\"` for %q value; tail=%q", ident, s) + } + s = s[1:] + again: + n = strings.IndexByte(s, '"') + if n < 0 { + return fmt.Errorf("missing trailing `\"` for %q value; tail=%q", ident, s) + } + m := n + for m > 0 && s[m-1] == '\\' { + m-- + } + if (n-m)%2 == 1 { + s = s[n+1:] + goto again + } + s = s[n+1:] + if len(s) == 0 { + return nil + } + if !strings.HasPrefix(s, ", ") { + return fmt.Errorf("missing `, ` after %q value; tail=%q", ident, s) + } + s = s[2:] + } +} + +func validateIdent(s string) error { + if !identRegexp.MatchString(s) { + return fmt.Errorf("invalid identifier %q", s) + } + return nil +} + +var identRegexp = regexp.MustCompile("^[a-zA-Z_:][a-zA-Z0-9_:]*$") diff --git a/validator_test.go b/validator_test.go new file mode 100644 index 0000000..dd3c338 --- /dev/null +++ b/validator_test.go @@ -0,0 +1,60 @@ +package metrics + +import ( + "testing" +) + +func TestValidateMetricSuccess(t *testing.T) { + f := func(s string) { + t.Helper() + if err := validateMetric(s); err != nil { + t.Fatalf("cannot validate %q: %s", s, err) + } + } + f("a") + f("_9:8") + f("a{}") + f(`a{foo="bar"}`) + f(`foo{bar="baz", x="y\"z"}`) + f(`foo{bar="b}az"}`) +} + +func TestValidateMetricError(t *testing.T) { + f := func(s string) { + t.Helper() + if err := validateMetric(s); err == nil { + t.Fatalf("expecting non-nil error when validating %q", s) + } + } + f("") + f("{}") + + // superflouos space + f("a ") + f(" a") + f(" a ") + f("a {}") + f("a{} ") + f("a{ }") + f(`a{foo ="bar"}`) + f(`a{ foo="bar"}`) + f(`a{foo= "bar"}`) + f(`a{foo="bar" }`) + f(`a{foo="bar",baz="a"}`) + f(`a{foo="bar" ,baz="a"}`) + + // invalid tags + f("a{foo}") + f("a{=}") + f(`a{=""}`) + f(`a{`) + f(`a}`) + f(`a{foo=}`) + f(`a{foo="`) + f(`a{foo="}`) + f(`a{foo="bar",}`) + f(`a{foo="bar", x`) + f(`a{foo="bar", x=`) + f(`a{foo="bar", x="`) + f(`a{foo="bar", x="}`) +}