From f9f61d29de69cb6c2c9252ac7cee924f1837a266 Mon Sep 17 00:00:00 2001 From: Prawn Date: Fri, 21 Aug 2020 20:57:10 +1200 Subject: [PATCH] Observability/metrics update (#1962) * Removing logging from the NOOP implementatino * Simplifying the percentiles option * Simple logging implementation Co-authored-by: chris --- metrics/logging/reporter.go | 53 +++++++++++++++++++++++++++++ metrics/logging/reporter_test.go | 51 +++++++++++++++++++++++++++ metrics/noop/reporter.go | 3 -- metrics/options.go | 26 +++++++------- metrics/options_test.go | 12 +++++-- metrics/prometheus/metric_family.go | 11 +++++- metrics/prometheus/reporter.go | 6 ++++ 7 files changed, 142 insertions(+), 20 deletions(-) create mode 100644 metrics/logging/reporter.go create mode 100644 metrics/logging/reporter_test.go diff --git a/metrics/logging/reporter.go b/metrics/logging/reporter.go new file mode 100644 index 00000000..8ce513d9 --- /dev/null +++ b/metrics/logging/reporter.go @@ -0,0 +1,53 @@ +package logging + +import ( + "time" + + log "github.com/micro/go-micro/v3/logger" + "github.com/micro/go-micro/v3/metrics" +) + +// Reporter is an implementation of metrics.Reporter: +type Reporter struct { + logger log.Logger + options metrics.Options +} + +// New returns a configured noop reporter: +func New(opts ...metrics.Option) *Reporter { + options := metrics.NewOptions(opts...) + logger := log.NewLogger(log.WithFields(convertTags(options.DefaultTags))) + logger.Log(log.InfoLevel, "Metrics/Logging - metrics will be logged (at TRACE level)") + + return &Reporter{ + logger: logger, + options: metrics.NewOptions(opts...), + } +} + +// Count implements the metrics.Reporter interface Count method: +func (r *Reporter) Count(metricName string, value int64, tags metrics.Tags) error { + r.logger.Logf(log.TraceLevel, "Count metric: %s", tags) + return nil +} + +// Gauge implements the metrics.Reporter interface Gauge method: +func (r *Reporter) Gauge(metricName string, value float64, tags metrics.Tags) error { + r.logger.Logf(log.TraceLevel, "Gauge metric: %s", tags) + return nil +} + +// Timing implements the metrics.Reporter interface Timing method: +func (r *Reporter) Timing(metricName string, value time.Duration, tags metrics.Tags) error { + r.logger.Logf(log.TraceLevel, "Timing metric: %s", tags) + return nil +} + +// convertTags turns Tags into prometheus labels: +func convertTags(tags metrics.Tags) map[string]interface{} { + labels := make(map[string]interface{}) + for key, value := range tags { + labels[key] = value + } + return labels +} diff --git a/metrics/logging/reporter_test.go b/metrics/logging/reporter_test.go new file mode 100644 index 00000000..70bf6776 --- /dev/null +++ b/metrics/logging/reporter_test.go @@ -0,0 +1,51 @@ +package logging + +import ( + "bytes" + "testing" + "time" + + log "github.com/micro/go-micro/v3/logger" + "github.com/micro/go-micro/v3/metrics" + + "github.com/stretchr/testify/assert" +) + +func TestLoggingReporter(t *testing.T) { + + // Make a Reporter: + reporter := New(metrics.Path("/prometheus"), metrics.DefaultTags(map[string]string{"service": "prometheus-test"})) + assert.NotNil(t, reporter) + assert.Equal(t, "prometheus-test", reporter.options.DefaultTags["service"]) + assert.Equal(t, ":9000", reporter.options.Address) + assert.Equal(t, "/prometheus", reporter.options.Path) + + // Make a log buffer and create a new logger to use it: + logBuffer := new(bytes.Buffer) + reporter.logger = log.NewLogger(log.WithLevel(log.TraceLevel), log.WithOutput(logBuffer), log.WithFields(convertTags(reporter.options.DefaultTags))) + + // Check that our implementation is valid: + assert.Implements(t, new(metrics.Reporter), reporter) + + // Test tag conversion: + tags := metrics.Tags{ + "tag1": "false", + "tag2": "true", + } + convertedTags := convertTags(tags) + assert.Equal(t, "false", convertedTags["tag1"]) + assert.Equal(t, "true", convertedTags["tag2"]) + + // Test submitting metrics through the interface methods: + assert.NoError(t, reporter.Count("test.counter.1", 6, tags)) + assert.NoError(t, reporter.Count("test.counter.2", 19, tags)) + assert.NoError(t, reporter.Count("test.counter.1", 5, tags)) + assert.NoError(t, reporter.Gauge("test.gauge.1", 99, tags)) + assert.NoError(t, reporter.Gauge("test.gauge.2", 55, tags)) + assert.NoError(t, reporter.Gauge("test.gauge.1", 98, tags)) + assert.NoError(t, reporter.Timing("test.timing.1", time.Second, tags)) + assert.NoError(t, reporter.Timing("test.timing.2", time.Minute, tags)) + + // Test reading back the metrics from the logbuffer (doesn't seem to work because the output still goes to StdOut): + // assert.Contains(t, logBuffer.String(), "level=debug service=prometheus-test Count metric: map[tag1:false tag2:true]") +} diff --git a/metrics/noop/reporter.go b/metrics/noop/reporter.go index 4cf36884..42aea0f4 100644 --- a/metrics/noop/reporter.go +++ b/metrics/noop/reporter.go @@ -3,7 +3,6 @@ package noop import ( "time" - log "github.com/micro/go-micro/v3/logger" "github.com/micro/go-micro/v3/metrics" ) @@ -14,8 +13,6 @@ type Reporter struct { // New returns a configured noop reporter: func New(opts ...metrics.Option) *Reporter { - log.Info("Metrics/NoOp - not doing anything") - return &Reporter{ options: metrics.NewOptions(opts...), } diff --git a/metrics/options.go b/metrics/options.go index e760ef1a..bba9c995 100644 --- a/metrics/options.go +++ b/metrics/options.go @@ -5,8 +5,8 @@ var ( defaultPrometheusListenAddress = ":9000" // This is the endpoint where the Prometheus metrics will be made available ("/metrics" is the default with Prometheus): defaultPath = "/metrics" - // timingObjectives is the default spread of stats we maintain for timings / histograms: - defaultTimingObjectives = map[float64]float64{0.0: 0, 0.5: 0.05, 0.75: 0.04, 0.90: 0.03, 0.95: 0.02, 0.98: 0.001, 1: 0} + // defaultPercentiles is the default spread of percentiles/quantiles we maintain for timings / histogram metrics: + defaultPercentiles = []float64{0, 0.5, 0.75, 0.90, 0.95, 0.98, 0.99, 1} ) // Option powers the configuration for metrics implementations: @@ -14,19 +14,19 @@ type Option func(*Options) // Options for metrics implementations: type Options struct { - Address string - Path string - DefaultTags Tags - TimingObjectives map[float64]float64 + Address string + DefaultTags Tags + Path string + Percentiles []float64 } // NewOptions prepares a set of options: func NewOptions(opt ...Option) Options { opts := Options{ - Address: defaultPrometheusListenAddress, - DefaultTags: make(Tags), - Path: defaultPath, - TimingObjectives: defaultTimingObjectives, + Address: defaultPrometheusListenAddress, + DefaultTags: make(Tags), + Path: defaultPath, + Percentiles: defaultPercentiles, } for _, o := range opt { @@ -57,9 +57,9 @@ func DefaultTags(value Tags) Option { } } -// TimingObjectives defines the desired spread of statistics for histogram / timing metrics: -func TimingObjectives(value map[float64]float64) Option { +// Percentiles defines the desired spread of statistics for histogram / timing metrics: +func Percentiles(value []float64) Option { return func(o *Options) { - o.TimingObjectives = value + o.Percentiles = value } } diff --git a/metrics/options_test.go b/metrics/options_test.go index c367f8d0..421aad99 100644 --- a/metrics/options_test.go +++ b/metrics/options_test.go @@ -9,10 +9,16 @@ import ( func TestOptions(t *testing.T) { // Make some new options: - options := NewOptions(Path("/prometheus"), DefaultTags(map[string]string{"service": "prometheus-test"})) + options := NewOptions( + Address(":9999"), + DefaultTags(map[string]string{"service": "prometheus-test"}), + Path("/prometheus"), + Percentiles([]float64{0.11, 0.22, 0.33}), + ) // Check that the defaults and overrides were accepted: - assert.Equal(t, ":9000", options.Address) - assert.Equal(t, "/prometheus", options.Path) + assert.Equal(t, ":9999", options.Address) assert.Equal(t, "prometheus-test", options.DefaultTags["service"]) + assert.Equal(t, "/prometheus", options.Path) + assert.Equal(t, []float64{0.11, 0.22, 0.33}, options.Percentiles) } diff --git a/metrics/prometheus/metric_family.go b/metrics/prometheus/metric_family.go index fd9602cb..e8f0b1f1 100644 --- a/metrics/prometheus/metric_family.go +++ b/metrics/prometheus/metric_family.go @@ -19,13 +19,22 @@ type metricFamily struct { // newMetricFamily returns a new metricFamily (useful in case we want to change the structure later): func (r *Reporter) newMetricFamily() metricFamily { + + // Take quantile thresholds from our pre-defined list: + timingObjectives := make(map[float64]float64) + for _, percentile := range r.options.Percentiles { + if quantileThreshold, ok := quantileThresholds[percentile]; ok { + timingObjectives[percentile] = quantileThreshold + } + } + return metricFamily{ counters: make(map[string]*prometheus.CounterVec), gauges: make(map[string]*prometheus.GaugeVec), timings: make(map[string]*prometheus.SummaryVec), defaultLabels: r.convertTags(r.options.DefaultTags), prometheusRegistry: r.prometheusRegistry, - timingObjectives: r.options.TimingObjectives, + timingObjectives: timingObjectives, } } diff --git a/metrics/prometheus/reporter.go b/metrics/prometheus/reporter.go index 79614ab7..eb2c4f8d 100644 --- a/metrics/prometheus/reporter.go +++ b/metrics/prometheus/reporter.go @@ -10,6 +10,12 @@ import ( "github.com/prometheus/client_golang/prometheus/promhttp" ) +var ( + // quantileThresholds maps quantiles / percentiles to error thresholds (required by the Prometheus client). + // Must be from our pre-defined set [0.0, 0.5, 0.75, 0.90, 0.95, 0.98, 0.99, 1]: + quantileThresholds = map[float64]float64{0.0: 0, 0.5: 0.05, 0.75: 0.04, 0.90: 0.03, 0.95: 0.02, 0.98: 0.001, 1: 0} +) + // Reporter is an implementation of metrics.Reporter: type Reporter struct { options metrics.Options