commit b1d3992f357d4379abbf20c52ba314690ba53a9e Author: Aliaksandr Valialkin Date: Mon Apr 8 16:29:16 2019 +0300 Initial commit diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..539b7a4 --- /dev/null +++ b/LICENSE @@ -0,0 +1,22 @@ +The MIT License (MIT) + +Copyright (c) 2019 VictoriaMetrics + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + diff --git a/README.md b/README.md new file mode 100644 index 0000000..9fc6922 --- /dev/null +++ b/README.md @@ -0,0 +1,26 @@ +[![Build Status](https://travis-ci.org/VictoriaMetrics/metrics.svg)](https://travis-ci.org/VictoriaMetrics/metrics) +[![GoDoc](https://godoc.org/github.com/VictoriaMetrics/metrics?status.svg)](http://godoc.org/github.com/VictoriaMetrics/metrics) +[![Go Report](https://goreportcard.com/badge/github.com/VictoriaMetrics/metrics)](https://goreportcard.com/report/github.com/VictoriaMetrics/metrics) +[![codecov](https://codecov.io/gh/VictoriaMetrics/metrics/branch/master/graph/badge.svg)](https://codecov.io/gh/VictoriaMetrics/metrics) + +# metrics - lightweight alternative to `github.com/prometheus/client_golang/prometheus`. + + +### Features + +* Lightweight. Has minimal number of third-party dependencies and all these deps are small. + See [this article](https://medium.com/@valyala/stripping-dependency-bloat-in-victoriametrics-docker-image-983fb5912b0d) for details. +* Easy to use. See the [API docs](http://godoc.org/github.com/VictoriaMetrics/metrics). +* Fast. + + +### Limitations + +* It doesn't implement advanced functionality from [github.com/prometheus/client_golang/prometheus](https://godoc.org/github.com/prometheus/client_golang/prometheus). + + +### Users + +* `Metrics` has been extracted from [VictoriaMetrics](https://github.com/VictoriaMetrics/VictoriaMetrics) sources. + See [this article](https://medium.com/devopslinks/victoriametrics-creating-the-best-remote-storage-for-prometheus-5d92d66787ac) + for more info about `VictoriaMetrics`. diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..7a4629b --- /dev/null +++ b/go.mod @@ -0,0 +1,3 @@ +module github.com/VictoriaMetrics/metrics + +require github.com/valyala/histogram v1.0.1 diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..9dd64f0 --- /dev/null +++ b/go.sum @@ -0,0 +1,4 @@ +github.com/valyala/fastrand v1.0.0 h1:LUKT9aKer2dVQNUi3waewTbKV+7H17kvWFNKs2ObdkI= +github.com/valyala/fastrand v1.0.0/go.mod h1:HWqCzkrkg6QXT8V2EXWvXCoow7vLwOFN002oeRzjapQ= +github.com/valyala/histogram v1.0.1 h1:FzA7n2Tz/wKRMejgu3PV1vw3htAklTjjuoI6z3d4KDg= +github.com/valyala/histogram v1.0.1/go.mod h1:lQy0xA4wUz2+IUnf97SivorsJIp8FxsnRd6x25q7Mto= diff --git a/metrics.go b/metrics.go new file mode 100644 index 0000000..8d844f4 --- /dev/null +++ b/metrics.go @@ -0,0 +1,225 @@ +// Package metrics implements Prometheus-compatible metrics for applications. +// +// This package is similar to https://github.com/prometheus/client_golang , +// but is simpler to use and is more lightweight. +package metrics + +import ( + "fmt" + "io" + "runtime" + "sort" + "sync" + "sync/atomic" + "time" + + "github.com/valyala/histogram" +) + +type gauge struct { + f func() float64 +} + +func (g *gauge) marshalTo(prefix string, w io.Writer) { + v := g.f() + fmt.Fprintf(w, "%s %g\n", prefix, v) +} + +// NewGauge creates a gauge with the given name, which calls f +// to obtain gauge value. +// +// name must be valid Prometheus-compatible metric with possible labels. +// For instance, +// +// * foo +// * foo{bar="baz"} +// * foo{bar="baz",aaa="b"} +// +// f must be safe for concurrent calls. +func NewGauge(name string, f func() float64) { + g := &gauge{ + f: f, + } + registerMetric(name, g) +} + +// Counter is a counter. +// +// It may be used as a gauge if Dec and Set are called. +type Counter struct { + n uint64 +} + +// NewCounter creates and returns new counter with the given name. +// +// name must be valid Prometheus-compatible metric with possible lables. +// For instance, +// +// * foo +// * foo{bar="baz"} +// * foo{bar="baz",aaa="b"} +// +// The returned counter is safe to use from concurrent goroutines. +func NewCounter(name string) *Counter { + c := &Counter{} + registerMetric(name, c) + return c +} + +func registerMetric(name string, m metric) { + if err := validateMetric(name); err != nil { + // Do not use logger.Panicf here, since it may be uninitialized yet. + panic(fmt.Errorf("BUG: invalid metric name %q: %s", name, err)) + } + metricsMapLock.Lock() + ok := isRegisteredMetric(metricsMap, name) + if !ok { + nm := namedMetric{ + name: name, + metric: m, + } + metricsMap = append(metricsMap, nm) + } + metricsMapLock.Unlock() + if ok { + // Do not use logger.Panicf here, since it may be uninitialized yet. + panic(fmt.Errorf("BUG: metric with name %q is already registered", name)) + } +} + +// Inc increments c. +func (c *Counter) Inc() { + atomic.AddUint64(&c.n, 1) +} + +// Dec decrements c. +func (c *Counter) Dec() { + atomic.AddUint64(&c.n, ^uint64(0)) +} + +// Add adds n to c. +func (c *Counter) Add(n int) { + atomic.AddUint64(&c.n, uint64(n)) +} + +// Get returns the current value for c. +func (c *Counter) Get() uint64 { + return atomic.LoadUint64(&c.n) +} + +// Set sets c value to n. +func (c *Counter) Set(n uint64) { + atomic.StoreUint64(&c.n, n) +} + +// marshalTo marshals c with the given prefix to w. +func (c *Counter) marshalTo(prefix string, w io.Writer) { + v := c.Get() + fmt.Fprintf(w, "%s %d\n", prefix, v) +} + +var ( + metricsMapLock sync.Mutex + metricsMap []namedMetric +) + +type namedMetric struct { + name string + metric metric +} + +func isRegisteredMetric(mm []namedMetric, name string) bool { + for _, nm := range mm { + if nm.name == name { + return true + } + } + return false +} + +func sortMetrics(mm []namedMetric) { + lessFunc := func(i, j int) bool { + return mm[i].name < mm[j].name + } + if !sort.SliceIsSorted(mm, lessFunc) { + sort.Slice(mm, lessFunc) + } +} + +type metric interface { + marshalTo(prefix string, w io.Writer) +} + +// WritePrometheus writes all the registered metrics in Prometheus format to w. +// +// If exposeProcessMetrics is true, then various `go_*` metrics are exposed +// for the current process. +func WritePrometheus(w io.Writer, exposeProcessMetrics bool) { + // Export user-defined metrics. + metricsMapLock.Lock() + sortMetrics(metricsMap) + for _, nm := range metricsMap { + nm.metric.marshalTo(nm.name, w) + } + metricsMapLock.Unlock() + + if !exposeProcessMetrics { + return + } + + // Export memory stats. + var ms runtime.MemStats + runtime.ReadMemStats(&ms) + fmt.Fprintf(w, `go_memstats_alloc_bytes %d`+"\n", ms.Alloc) + fmt.Fprintf(w, `go_memstats_alloc_bytes_total %d`+"\n", ms.TotalAlloc) + fmt.Fprintf(w, `go_memstats_buck_hash_sys_bytes %d`+"\n", ms.BuckHashSys) + fmt.Fprintf(w, `go_memstats_frees_total %d`+"\n", ms.Frees) + fmt.Fprintf(w, `go_memstats_gc_cpu_fraction %f`+"\n", ms.GCCPUFraction) + fmt.Fprintf(w, `go_memstats_gc_sys_bytes %d`+"\n", ms.GCSys) + fmt.Fprintf(w, `go_memstats_heap_alloc_bytes %d`+"\n", ms.HeapAlloc) + fmt.Fprintf(w, `go_memstats_heap_idle_bytes %d`+"\n", ms.HeapIdle) + fmt.Fprintf(w, `go_memstats_heap_inuse_bytes %d`+"\n", ms.HeapInuse) + fmt.Fprintf(w, `go_memstats_heap_objects %d`+"\n", ms.HeapObjects) + fmt.Fprintf(w, `go_memstats_heap_released_bytes %d`+"\n", ms.HeapReleased) + fmt.Fprintf(w, `go_memstats_heap_sys_bytes %d`+"\n", ms.HeapSys) + fmt.Fprintf(w, `go_memstats_last_gc_time_seconds %f`+"\n", float64(ms.LastGC)/1e9) + fmt.Fprintf(w, `go_memstats_lookups_total %d`+"\n", ms.Lookups) + fmt.Fprintf(w, `go_memstats_mallocs_total %d`+"\n", ms.Mallocs) + fmt.Fprintf(w, `go_memstats_mcache_inuse_bytes %d`+"\n", ms.MCacheInuse) + fmt.Fprintf(w, `go_memstats_mcache_sys_bytes %d`+"\n", ms.MCacheSys) + fmt.Fprintf(w, `go_memstats_mspan_inuse_bytes %d`+"\n", ms.MSpanInuse) + fmt.Fprintf(w, `go_memstats_mspan_sys_bytes %d`+"\n", ms.MSpanSys) + fmt.Fprintf(w, `go_memstats_next_gc_bytes %d`+"\n", ms.NextGC) + fmt.Fprintf(w, `go_memstats_other_sys_bytes %d`+"\n", ms.OtherSys) + fmt.Fprintf(w, `go_memstats_stack_inuse_bytes %d`+"\n", ms.StackInuse) + fmt.Fprintf(w, `go_memstats_stack_sys_bytes %d`+"\n", ms.StackSys) + fmt.Fprintf(w, `go_memstats_sys_bytes %d`+"\n", ms.Sys) + + fmt.Fprintf(w, `go_cgo_calls_count %d`+"\n", runtime.NumCgoCall()) + fmt.Fprintf(w, `go_cpu_count %d`+"\n", runtime.NumCPU()) + + gcPauses := histogram.NewFast() + for _, pauseNs := range ms.PauseNs[:] { + gcPauses.Update(float64(pauseNs) / 1e9) + } + phis := []float64{0, 0.25, 0.5, 0.75, 1} + quantiles := make([]float64, 0, len(phis)) + for i, q := range gcPauses.Quantiles(quantiles[:0], phis) { + fmt.Fprintf(w, `go_gc_duration_seconds{quantile="%g"} %f`+"\n", phis[i], q) + } + fmt.Fprintf(w, `go_gc_duration_seconds_sum %f`+"\n", float64(ms.PauseTotalNs)/1e9) + fmt.Fprintf(w, `go_gc_duration_seconds_count %d`+"\n", ms.NumGC) + fmt.Fprintf(w, `go_gc_forced_count %d`+"\n", ms.NumForcedGC) + + fmt.Fprintf(w, `go_gomaxprocs %d`+"\n", runtime.GOMAXPROCS(0)) + fmt.Fprintf(w, `go_goroutines %d`+"\n", runtime.NumGoroutine()) + numThread, _ := runtime.ThreadCreateProfile(nil) + fmt.Fprintf(w, `go_threads %d`+"\n", numThread) + + // Export build details. + fmt.Fprintf(w, "go_info{version=%q} 1\n", runtime.Version()) + fmt.Fprintf(w, "go_info_ext{compiler=%q, GOARCH=%q, GOOS=%q, GOROOT=%q} 1\n", + runtime.Compiler, runtime.GOARCH, runtime.GOOS, runtime.GOROOT()) +} + +var startTime = time.Now() diff --git a/summary.go b/summary.go new file mode 100644 index 0000000..7f2e560 --- /dev/null +++ b/summary.go @@ -0,0 +1,145 @@ +package metrics + +import ( + "fmt" + "io" + "sync" + "time" + + "github.com/valyala/histogram" +) + +const defaultSummaryWindow = 5 * time.Minute + +var defaultSummaryQuantiles = []float64{0.5, 0.9, 0.97, 0.99, 1} + +// Summary implements summary. +type Summary struct { + mu sync.Mutex + + curr *histogram.Fast + next *histogram.Fast + + quantiles []float64 + quantileValues []float64 +} + +// NewSummary creates and returns new summary with the given name. +// +// name must be valid Prometheus-compatible metric with possible lables. +// For instance, +// +// * foo +// * foo{bar="baz"} +// * foo{bar="baz",aaa="b"} +// +// The returned summary is safe to use from concurrent goroutines. +func NewSummary(name string) *Summary { + return NewSummaryExt(name, defaultSummaryWindow, defaultSummaryQuantiles) +} + +// NewSummaryExt creates and returns new summary with the given name, +// window and quantiles. +// +// name must be valid Prometheus-compatible metric with possible lables. +// For instance, +// +// * foo +// * foo{bar="baz"} +// * foo{bar="baz",aaa="b"} +// +// The returned summary is safe to use from concurrent goroutines. +func NewSummaryExt(name string, window time.Duration, quantiles []float64) *Summary { + s := &Summary{ + curr: histogram.NewFast(), + next: histogram.NewFast(), + quantiles: quantiles, + quantileValues: make([]float64, len(quantiles)), + } + registerSummary(s, window) + registerMetric(fmt.Sprintf("\x00%s", name), s) + for i, q := range quantiles { + quantileValueName := addTag(name, fmt.Sprintf(`quantile="%g"`, q)) + qv := &quantileValue{ + s: s, + idx: i, + } + registerMetric(quantileValueName, qv) + } + return s +} + +// Update updates the summary. +func (s *Summary) Update(v float64) { + s.mu.Lock() + s.curr.Update(v) + s.next.Update(v) + s.mu.Unlock() +} + +// UpdateDuration updates request duration based on the given startTime. +func (s *Summary) UpdateDuration(startTime time.Time) { + d := time.Since(startTime).Seconds() + s.Update(d) +} + +func (s *Summary) marshalTo(prefix string, w io.Writer) { + // Just update s.quantileValues and don't write anything to w. + // s.quantileValues will be marshaled later via quantileValue.marshalTo. + s.updateQuantiles() +} + +func (s *Summary) updateQuantiles() { + s.mu.Lock() + s.quantileValues = s.curr.Quantiles(s.quantileValues[:0], s.quantiles) + s.mu.Unlock() +} + +type quantileValue struct { + s *Summary + idx int +} + +func (qv *quantileValue) marshalTo(prefix string, w io.Writer) { + qv.s.mu.Lock() + v := qv.s.quantileValues[qv.idx] + qv.s.mu.Unlock() + fmt.Fprintf(w, "%s %g\n", prefix, v) +} + +func addTag(name, tag string) string { + if len(name) == 0 || name[len(name)-1] != '}' { + return fmt.Sprintf("%s{%s}", name, tag) + } + return fmt.Sprintf("%s, %s}", name[:len(name)-1], tag) +} + +func registerSummary(s *Summary, window time.Duration) { + summariesLock.Lock() + summaries[window] = append(summaries[window], s) + if len(summaries[window]) == 1 { + go summariesSwapCron(window) + } + summariesLock.Unlock() +} + +func summariesSwapCron(window time.Duration) { + for { + time.Sleep(window / 2) + summariesLock.Lock() + for _, s := range summaries[window] { + s.mu.Lock() + tmp := s.curr + s.curr = s.next + s.next = tmp + s.next.Reset() + s.mu.Unlock() + } + summariesLock.Unlock() + } +} + +var ( + summaries = map[time.Duration][]*Summary{} + summariesLock sync.Mutex +) diff --git a/validator.go b/validator.go new file mode 100644 index 0000000..6b74d21 --- /dev/null +++ b/validator.go @@ -0,0 +1,81 @@ +package metrics + +import ( + "fmt" + "regexp" + "strings" +) + +func validateMetric(s string) error { + if len(s) == 0 { + return fmt.Errorf("metric cannot be empty") + } + if s[0] == 0 { + // Skip special case metrics. See Histogram for details. + return nil + } + n := strings.IndexByte(s, '{') + if n < 0 { + return validateIdent(s) + } + ident := s[:n] + s = s[n+1:] + if err := validateIdent(ident); err != nil { + return err + } + if len(s) == 0 || s[len(s)-1] != '}' { + return fmt.Errorf("missing closing curly brace at the end of %q", ident) + } + return validateTags(s[:len(s)-1]) +} + +func validateTags(s string) error { + if len(s) == 0 { + return nil + } + for { + n := strings.IndexByte(s, '=') + if n < 0 { + return fmt.Errorf("missing `=` after %q", s) + } + ident := s[:n] + s = s[n+1:] + if err := validateIdent(ident); err != nil { + return err + } + if len(s) == 0 || s[0] != '"' { + return fmt.Errorf("missing starting `\"` for %q value; tail=%q", ident, s) + } + s = s[1:] + again: + n = strings.IndexByte(s, '"') + if n < 0 { + return fmt.Errorf("missing trailing `\"` for %q value; tail=%q", ident, s) + } + m := n + for m > 0 && s[m-1] == '\\' { + m-- + } + if (n-m)%2 == 1 { + s = s[n+1:] + goto again + } + s = s[n+1:] + if len(s) == 0 { + return nil + } + if !strings.HasPrefix(s, ", ") { + return fmt.Errorf("missing `, ` after %q value; tail=%q", ident, s) + } + s = s[2:] + } +} + +func validateIdent(s string) error { + if !identRegexp.MatchString(s) { + return fmt.Errorf("invalid identifier %q", s) + } + return nil +} + +var identRegexp = regexp.MustCompile("^[a-zA-Z_:][a-zA-Z0-9_:]*$") diff --git a/validator_test.go b/validator_test.go new file mode 100644 index 0000000..dd3c338 --- /dev/null +++ b/validator_test.go @@ -0,0 +1,60 @@ +package metrics + +import ( + "testing" +) + +func TestValidateMetricSuccess(t *testing.T) { + f := func(s string) { + t.Helper() + if err := validateMetric(s); err != nil { + t.Fatalf("cannot validate %q: %s", s, err) + } + } + f("a") + f("_9:8") + f("a{}") + f(`a{foo="bar"}`) + f(`foo{bar="baz", x="y\"z"}`) + f(`foo{bar="b}az"}`) +} + +func TestValidateMetricError(t *testing.T) { + f := func(s string) { + t.Helper() + if err := validateMetric(s); err == nil { + t.Fatalf("expecting non-nil error when validating %q", s) + } + } + f("") + f("{}") + + // superflouos space + f("a ") + f(" a") + f(" a ") + f("a {}") + f("a{} ") + f("a{ }") + f(`a{foo ="bar"}`) + f(`a{ foo="bar"}`) + f(`a{foo= "bar"}`) + f(`a{foo="bar" }`) + f(`a{foo="bar",baz="a"}`) + f(`a{foo="bar" ,baz="a"}`) + + // invalid tags + f("a{foo}") + f("a{=}") + f(`a{=""}`) + f(`a{`) + f(`a}`) + f(`a{foo=}`) + f(`a{foo="`) + f(`a{foo="}`) + f(`a{foo="bar",}`) + f(`a{foo="bar", x`) + f(`a{foo="bar", x=`) + f(`a{foo="bar", x="`) + f(`a{foo="bar", x="}`) +}