Merge pull request #54 from unistack-org/optimization

use newMetric helper in all places

.github/workflows/build.yml

@@ -12,7 +12,7 @@ jobs:
     - name: setup
       uses: actions/setup-go@v2
       with:
-        go-version: 1.16
+        go-version: 1.17
     - name: checkout
       uses: actions/checkout@v3
     - name: cache

.github/workflows/codeql-analysis.yml

@@ -47,7 +47,7 @@ jobs:
     - name: setup
       uses: actions/setup-go@v2
       with:
-        go-version: 1.16
+        go-version: 1.17
     # Initializes the CodeQL tools for scanning.
     - name: init
       uses: github/codeql-action/init@v1

.github/workflows/pr.yml

@@ -12,7 +12,7 @@ jobs:
     - name: setup
       uses: actions/setup-go@v2
       with:
-        go-version: 1.16
+        go-version: 1.17
     - name: checkout
       uses: actions/checkout@v3
     - name: cache

prometheus.go

@@ -2,6 +2,7 @@ package prometheus
 
 import (
 	"fmt"
+	"hash/fnv"
 	"io"
 	"sync"
 	"time"
@@ -13,26 +14,58 @@ import (
 	"go.unistack.org/micro/v3/meter"
 )
 
+var _ meter.Meter = &prometheusMeter{}
+
 type prometheusMeter struct {
 	opts         meter.Options
 	set          prometheus.Registerer
-	counter      map[string]prometheusCounter
-	floatCounter map[string]prometheusFloatCounter
-	gauge        map[string]prometheusGauge
-	histogram    map[string]prometheusHistogram
-	summary      map[string]prometheusSummary
+	counter      map[string]*counters
+	floatCounter map[string]*floatCounters
+	gauge        map[string]*gauges
+	histogram    map[string]*histograms
+	summary      map[string]*summaries
 	sync.Mutex
 }
 
-func NewMeter(opts ...meter.Option) meter.Meter {
+type counters struct {
+	cs map[uint64]*prometheusCounter
+}
+
+type gauges struct {
+	cs map[uint64]*prometheusGauge
+}
+
+type histograms struct {
+	cs map[uint64]*prometheusHistogram
+}
+
+type summaries struct {
+	cs map[uint64]*prometheusSummary
+}
+
+type floatCounters struct {
+	cs map[uint64]*prometheusFloatCounter
+}
+
+func newFloat64(v float64) *float64 {
+	nv := v
+	return &nv
+}
+
+func newString(v string) *string {
+	nv := v
+	return &nv
+}
+
+func NewMeter(opts ...meter.Option) *prometheusMeter {
 	return &prometheusMeter{
-		set:          prometheus.DefaultRegisterer,
+		set:          prometheus.NewRegistry(), // prometheus.DefaultRegisterer,
 		opts:         meter.NewOptions(opts...),
-		counter:      make(map[string]prometheusCounter),
-		floatCounter: make(map[string]prometheusFloatCounter),
-		gauge:        make(map[string]prometheusGauge),
-		histogram:    make(map[string]prometheusHistogram),
-		summary:      make(map[string]prometheusSummary),
+		counter:      make(map[string]*counters),
+		floatCounter: make(map[string]*floatCounters),
+		gauge:        make(map[string]*gauges),
+		histogram:    make(map[string]*histograms),
+		summary:      make(map[string]*summaries),
 	}
 }
 
@@ -54,9 +87,8 @@ func (m *prometheusMeter) buildMetric(name string, labels ...string) string {
 		return meter.BuildName(name, nlabels...)
 	}
 
-	for idx := 0; idx < nl; idx++ {
+	for idx := 0; idx < nl; idx += 2 {
 		nlabels[idx] = m.opts.LabelPrefix + nlabels[idx]
-		idx++
 	}
 	return meter.BuildName(name, nlabels...)
 }
@@ -69,18 +101,16 @@ func (m *prometheusMeter) buildName(name string) string {
 }
 
 func (m *prometheusMeter) buildLabels(labels ...string) []string {
-	nl := len(m.opts.Labels) + len(labels)
+	nl := len(labels)
 	if nl == 0 {
 		return nil
 	}
 
 	nlabels := make([]string, 0, nl)
-	nlabels = append(nlabels, m.opts.Labels...)
-	nlabels = append(nlabels, labels...)
 
-	for idx := 0; idx < nl; idx++ {
-		nlabels[idx] = m.opts.LabelPrefix + nlabels[idx]
-		idx++
+	for idx := 0; idx < nl; idx += 2 {
+		nlabels = append(nlabels, m.opts.LabelPrefix+labels[idx])
+		nlabels = append(nlabels, labels[idx+1])
 	}
 	return nlabels
 }
@@ -89,191 +119,156 @@ func (m *prometheusMeter) Name() string {
 	return m.opts.Name
 }
 
-func (m *prometheusMeter) mapLabels(src ...string) map[string]string {
-	src = m.buildLabels(src...)
-	mp := make(map[string]string, len(src)/2)
-	for idx := 0; idx < len(src); idx++ {
-		mp[src[idx]] = src[idx+1]
-		idx++
-	}
-	return mp
-}
-
-func (m *prometheusMeter) metricEqual(src []string, dst []string) bool {
-	if len(src) != len(dst)/2 {
-		return false
-	}
-	dst = m.buildLabels(dst...)
-	mp := make(map[string]struct{}, len(src))
-	for idx := range src {
-		mp[src[idx]] = struct{}{}
-	}
-	for idx := 0; idx < len(dst); idx++ {
-		if _, ok := mp[dst[idx]]; !ok {
-			return false
-		}
-		idx++
-	}
-	return true
-}
-
-func (m *prometheusMeter) labelNames(src map[string]string, dst []string) []string {
-	dst = m.buildLabels(dst...)
-	nlabels := make([]string, 0, len(src))
-	for idx := 0; idx < len(dst); idx++ {
-		if src == nil {
-			nlabels = append(nlabels, dst[idx])
-		} else if _, ok := src[dst[idx]]; ok {
-			nlabels = append(nlabels, dst[idx])
-		} else {
-			nlabels = append(nlabels, dst[idx])
-		}
-		idx++
-	}
-	return nlabels
-}
-
 func (m *prometheusMeter) Counter(name string, labels ...string) meter.Counter {
 	m.Lock()
 	defer m.Unlock()
 	nm := m.buildName(name)
-	c, ok := m.counter[nm]
-	var lnames []string
+	labels = m.buildLabels(append(m.opts.Labels, labels...)...)
+	cd, ok := m.counter[nm]
+	h := newHash(labels)
 	if !ok {
-		lnames = m.labelNames(nil, labels)
-		fmt.Printf("!ok lnames: %v\n", lnames)
-		nc := prometheus.NewGaugeVec(prometheus.GaugeOpts{Name: nm}, lnames)
-		c = prometheusCounter{c: nc, lnames: lnames}
-		m.counter[nm] = c
-	} else if !m.metricEqual(c.lnames, labels) {
-		fmt.Printf("ok && !m.metricEqual lnames: %v labels: %v\n", c.lnames, labels)
-		lnames = m.labelNames(c.labels, labels)
-		m.set.Unregister(c.c)
-		nc := prometheus.NewGaugeVec(prometheus.GaugeOpts{Name: nm}, lnames)
-		c = prometheusCounter{c: nc, lnames: lnames}
-		m.counter[nm] = c
-		m.set.MustRegister(c.c)
-	} else {
-		lnames = c.lnames
+		cd = &counters{cs: make(map[uint64]*prometheusCounter)}
+		c := &prometheusCounter{c: prometheus.NewGauge(prometheus.GaugeOpts{Name: nm}), labels: labels}
+		cd.cs[h] = c
+		m.counter[nm] = cd
+		return c
 	}
-	fmt.Printf("lnames %v\n", lnames)
-	return prometheusCounter{c: c.c, lnames: lnames, labels: m.mapLabels(labels...)}
+	c, ok := cd.cs[h]
+	if !ok {
+		c = &prometheusCounter{c: prometheus.NewGauge(prometheus.GaugeOpts{Name: nm}), labels: labels}
+		cd.cs[h] = c
+		m.counter[nm] = cd
+	}
+	return c
 }
 
 func (m *prometheusMeter) FloatCounter(name string, labels ...string) meter.FloatCounter {
 	m.Lock()
 	defer m.Unlock()
-
 	nm := m.buildName(name)
-	c, ok := m.floatCounter[nm]
+	labels = m.buildLabels(append(m.opts.Labels, labels...)...)
+	cd, ok := m.floatCounter[nm]
+	h := newHash(labels)
 	if !ok {
-		nc := prometheus.NewGaugeVec(prometheus.GaugeOpts{Name: nm}, m.labelNames(c.labels, labels))
-		c = prometheusFloatCounter{c: nc}
-		m.floatCounter[nm] = c
-	} else if !m.metricEqual(c.lnames, labels) {
-		m.set.Unregister(c.c)
-		nc := prometheus.NewGaugeVec(prometheus.GaugeOpts{Name: nm}, m.labelNames(c.labels, labels))
-		c = prometheusFloatCounter{c: nc}
-		m.floatCounter[nm] = c
-		m.set.MustRegister(c.c)
+		cd = &floatCounters{cs: make(map[uint64]*prometheusFloatCounter)}
+		c := &prometheusFloatCounter{c: prometheus.NewGauge(prometheus.GaugeOpts{Name: nm}), labels: labels}
+		cd.cs[h] = c
+		m.floatCounter[nm] = cd
+		return c
 	}
-
-	return prometheusFloatCounter{c: c.c, labels: m.mapLabels(labels...)}
+	c, ok := cd.cs[h]
+	if !ok {
+		c = &prometheusFloatCounter{c: prometheus.NewGauge(prometheus.GaugeOpts{Name: nm}), labels: labels}
+		cd.cs[h] = c
+		m.floatCounter[nm] = cd
+	}
+	return c
 }
 
 func (m *prometheusMeter) Gauge(name string, fn func() float64, labels ...string) meter.Gauge {
 	m.Lock()
 	defer m.Unlock()
-
 	nm := m.buildName(name)
-	c, ok := m.gauge[nm]
+	labels = m.buildLabels(append(m.opts.Labels, labels...)...)
+	cd, ok := m.gauge[nm]
+	h := newHash(labels)
 	if !ok {
-		nc := prometheus.NewGaugeVec(prometheus.GaugeOpts{Name: nm}, m.labelNames(c.labels, labels))
-		c = prometheusGauge{c: nc}
-		m.gauge[nm] = c
-	} else if !m.metricEqual(c.lnames, labels) {
-		m.set.Unregister(c.c)
-		nc := prometheus.NewGaugeVec(prometheus.GaugeOpts{Name: nm}, m.labelNames(c.labels, labels))
-		c = prometheusGauge{c: nc}
-		m.gauge[nm] = c
-		m.set.MustRegister(c.c)
+		cd = &gauges{cs: make(map[uint64]*prometheusGauge)}
+		c := &prometheusGauge{c: prometheus.NewGauge(prometheus.GaugeOpts{Name: nm}), labels: labels}
+		cd.cs[h] = c
+		m.gauge[nm] = cd
+		return c
 	}
-
-	return prometheusGauge{c: c.c, labels: m.mapLabels(labels...)}
+	c, ok := cd.cs[h]
+	if !ok {
+		c = &prometheusGauge{c: prometheus.NewGauge(prometheus.GaugeOpts{Name: nm}), labels: labels}
+		cd.cs[h] = c
+		m.gauge[nm] = cd
+	}
+	return c
 }
 
 func (m *prometheusMeter) Histogram(name string, labels ...string) meter.Histogram {
 	m.Lock()
 	defer m.Unlock()
-
 	nm := m.buildName(name)
-	c, ok := m.histogram[nm]
+	labels = m.buildLabels(append(m.opts.Labels, labels...)...)
+	cd, ok := m.histogram[nm]
+	h := newHash(labels)
 	if !ok {
-		nc := prometheus.NewHistogramVec(prometheus.HistogramOpts{Name: nm}, m.labelNames(c.labels, labels))
-		c = prometheusHistogram{c: nc}
-		m.histogram[nm] = c
-	} else if !m.metricEqual(c.lnames, labels) {
-		m.set.Unregister(c.c)
-		nc := prometheus.NewHistogramVec(prometheus.HistogramOpts{Name: nm}, m.labelNames(c.labels, labels))
-		c = prometheusHistogram{c: nc}
-		m.histogram[nm] = c
-		m.set.MustRegister(c.c)
+		cd = &histograms{cs: make(map[uint64]*prometheusHistogram)}
+		c := &prometheusHistogram{c: prometheus.NewHistogram(prometheus.HistogramOpts{Name: nm}), labels: labels}
+		cd.cs[h] = c
+		m.histogram[nm] = cd
+		return c
 	}
-
-	return prometheusHistogram{c: c.c, labels: m.mapLabels(labels...)}
+	c, ok := cd.cs[h]
+	if !ok {
+		c = &prometheusHistogram{c: prometheus.NewHistogram(prometheus.HistogramOpts{Name: nm}), labels: labels}
+		cd.cs[h] = c
+		m.histogram[nm] = cd
+	}
+	return c
 }
 
 func (m *prometheusMeter) Summary(name string, labels ...string) meter.Summary {
 	m.Lock()
 	defer m.Unlock()
-
 	nm := m.buildName(name)
-	c, ok := m.summary[nm]
+	labels = m.buildLabels(append(m.opts.Labels, labels...)...)
+	cd, ok := m.summary[nm]
+	h := newHash(labels)
 	if !ok {
-		nc := prometheus.NewSummaryVec(prometheus.SummaryOpts{Name: nm}, m.labelNames(c.labels, labels))
-		c = prometheusSummary{c: nc}
-		m.summary[nm] = c
-	} else if !m.metricEqual(c.lnames, labels) {
-		m.set.Unregister(c.c)
-		nc := prometheus.NewSummaryVec(prometheus.SummaryOpts{Name: nm}, m.labelNames(c.labels, labels))
-		c = prometheusSummary{c: nc}
-		m.summary[nm] = c
-		m.set.MustRegister(c.c)
+		cd = &summaries{cs: make(map[uint64]*prometheusSummary)}
+		c := &prometheusSummary{c: prometheus.NewSummary(prometheus.SummaryOpts{Name: nm}), labels: labels}
+		cd.cs[h] = c
+		m.summary[nm] = cd
+		return c
 	}
-
-	return prometheusSummary{c: c.c, labels: m.mapLabels(labels...)}
+	c, ok := cd.cs[h]
+	if !ok {
+		c = &prometheusSummary{c: prometheus.NewSummary(prometheus.SummaryOpts{Name: nm}), labels: labels}
+		cd.cs[h] = c
+		m.summary[nm] = cd
+	}
+	return c
 }
 
 func (m *prometheusMeter) SummaryExt(name string, window time.Duration, quantiles []float64, labels ...string) meter.Summary {
 	m.Lock()
 	defer m.Unlock()
-
 	nm := m.buildName(name)
-	c, ok := m.summary[nm]
+	labels = m.buildLabels(append(m.opts.Labels, labels...)...)
+	cd, ok := m.summary[nm]
+	h := newHash(labels)
 	if !ok {
-		nc := prometheus.NewSummaryVec(prometheus.SummaryOpts{
+		cd = &summaries{cs: make(map[uint64]*prometheusSummary)}
+		c := &prometheusSummary{c: prometheus.NewSummary(prometheus.SummaryOpts{
 			Name:       nm,
 			MaxAge:     window,
 			Objectives: map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001},
-		}, m.labelNames(c.labels, labels))
-		c = prometheusSummary{c: nc}
-		m.summary[nm] = c
-	} else if !m.metricEqual(c.lnames, labels) {
-		m.set.Unregister(c.c)
-		nc := prometheus.NewSummaryVec(prometheus.SummaryOpts{Name: nm}, m.labelNames(c.labels, labels))
-		c = prometheusSummary{c: nc}
-		m.summary[nm] = c
-		m.set.MustRegister(c.c)
+		}), labels: labels}
+		cd.cs[h] = c
+		m.summary[nm] = cd
+		return c
 	}
-
-	return prometheusSummary{c: c.c, labels: m.mapLabels(labels...)}
+	c, ok := cd.cs[h]
+	if !ok {
+		c = &prometheusSummary{c: prometheus.NewSummary(prometheus.SummaryOpts{
+			Name:       nm,
+			MaxAge:     window,
+			Objectives: map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001},
+		}), labels: labels}
+		cd.cs[h] = c
+		m.summary[nm] = cd
+	}
+	return c
 }
 
 func (m *prometheusMeter) Init(opts ...meter.Option) error {
 	for _, o := range opts {
 		o(&m.opts)
 	}
-
 	return nil
 }
 
@@ -299,6 +294,82 @@ func (m *prometheusMeter) Write(w io.Writer, opts ...meter.Option) error {
 	}
 
 	enc := expfmt.NewEncoder(w, expfmt.FmtText)
+
+	for name, metrics := range m.counter {
+		mf := &dto.MetricFamily{
+			Name:   newString(name),
+			Type:   dto.MetricType_GAUGE.Enum(),
+			Metric: make([]*dto.Metric, 0, len(metrics.cs)),
+		}
+		for _, c := range metrics.cs {
+			m := &dto.Metric{}
+			_ = c.c.Write(m)
+			fillMetric(m, c.labels)
+			mf.Metric = append(mf.Metric, m)
+		}
+		mfs = append(mfs, mf)
+	}
+
+	for name, metrics := range m.gauge {
+		mf := &dto.MetricFamily{
+			Name:   newString(name),
+			Type:   dto.MetricType_GAUGE.Enum(),
+			Metric: make([]*dto.Metric, 0, len(metrics.cs)),
+		}
+		for _, c := range metrics.cs {
+			m := &dto.Metric{}
+			_ = c.c.Write(m)
+			fillMetric(m, c.labels)
+			mf.Metric = append(mf.Metric, m)
+		}
+		mfs = append(mfs, mf)
+	}
+
+	for name, metrics := range m.floatCounter {
+		mf := &dto.MetricFamily{
+			Name:   newString(name),
+			Type:   dto.MetricType_GAUGE.Enum(),
+			Metric: make([]*dto.Metric, 0, len(metrics.cs)),
+		}
+		for _, c := range metrics.cs {
+			m := &dto.Metric{}
+			_ = c.c.Write(m)
+			fillMetric(m, c.labels)
+			mf.Metric = append(mf.Metric, m)
+		}
+		mfs = append(mfs, mf)
+	}
+
+	for name, metrics := range m.histogram {
+		mf := &dto.MetricFamily{
+			Name:   newString(name),
+			Type:   dto.MetricType_HISTOGRAM.Enum(),
+			Metric: make([]*dto.Metric, 0, len(metrics.cs)),
+		}
+		for _, c := range metrics.cs {
+			m := &dto.Metric{}
+			_ = c.c.Write(m)
+			fillMetric(m, c.labels)
+			mf.Metric = append(mf.Metric, m)
+		}
+		mfs = append(mfs, mf)
+	}
+
+	for name, metrics := range m.summary {
+		mf := &dto.MetricFamily{
+			Name:   newString(name),
+			Type:   dto.MetricType_SUMMARY.Enum(),
+			Metric: make([]*dto.Metric, 0, len(metrics.cs)),
+		}
+		for _, c := range metrics.cs {
+			m := &dto.Metric{}
+			_ = c.c.Write(m)
+			fillMetric(m, c.labels)
+			mf.Metric = append(mf.Metric, m)
+		}
+		mfs = append(mfs, mf)
+	}
+
 	for _, mf := range mfs {
 		_ = enc.Encode(mf)
 	}
@@ -344,103 +415,117 @@ func (m *prometheusMeter) Set(opts ...meter.Option) meter.Meter {
 }
 
 type prometheusCounter struct {
-	c      *prometheus.GaugeVec
-	lnames []string
-	labels prometheus.Labels
+	c      prometheus.Gauge
+	labels []string
 }
 
-func (c prometheusCounter) Add(n int) {
-	nc, _ := c.c.GetMetricWith(c.labels)
-	nc.Add(float64(n))
+func (c *prometheusCounter) Add(n int) {
+	c.c.Add(float64(n))
 }
 
-func (c prometheusCounter) Dec() {
-	c.c.With(c.labels).Dec()
+func (c *prometheusCounter) Dec() {
+	c.c.Dec()
 }
 
-func (c prometheusCounter) Inc() {
-	c.c.With(c.labels).Inc()
+func (c *prometheusCounter) Inc() {
+	c.c.Inc()
 }
 
-func (c prometheusCounter) Get() uint64 {
+func (c *prometheusCounter) Get() uint64 {
 	m := &dto.Metric{}
-	if err := c.c.With(c.labels).Write(m); err != nil {
+	if err := c.c.Write(m); err != nil {
 		return 0
 	}
 	return uint64(m.GetGauge().GetValue())
 }
 
-func (c prometheusCounter) Set(n uint64) {
-	c.c.With(c.labels).Set(float64(n))
+func (c *prometheusCounter) Set(n uint64) {
+	c.c.Set(float64(n))
 }
 
 type prometheusFloatCounter struct {
-	c      *prometheus.GaugeVec
-	lnames []string
-	labels prometheus.Labels
+	c      prometheus.Gauge
+	labels []string
 }
 
 func (c prometheusFloatCounter) Add(n float64) {
-	c.c.With(c.labels).Add(n)
+	c.c.Add(n)
 }
 
 func (c prometheusFloatCounter) Get() float64 {
 	m := &dto.Metric{}
-	if err := c.c.With(c.labels).Write(m); err != nil {
+	if err := c.c.Write(m); err != nil {
 		return 0
 	}
 	return m.GetGauge().GetValue()
 }
 
 func (c prometheusFloatCounter) Set(n float64) {
-	c.c.With(c.labels).Set(n)
+	c.c.Set(n)
 }
 
 func (c prometheusFloatCounter) Sub(n float64) {
-	c.c.With(c.labels).Add(-n)
+	c.c.Add(-n)
 }
 
 type prometheusGauge struct {
-	c      *prometheus.GaugeVec
-	lnames []string
-	labels prometheus.Labels
+	c      prometheus.Gauge
+	labels []string
 }
 
 func (c prometheusGauge) Get() float64 {
 	m := &dto.Metric{}
-	if err := c.c.With(c.labels).Write(m); err != nil {
+	if err := c.c.Write(m); err != nil {
 		return 0
 	}
 	return float64(m.GetGauge().GetValue())
 }
 
 type prometheusHistogram struct {
-	c      *prometheus.HistogramVec
-	lnames []string
-	labels prometheus.Labels
+	c      prometheus.Histogram
+	labels []string
 }
 
 func (c prometheusHistogram) Reset() {
 }
 
 func (c prometheusHistogram) Update(n float64) {
-	c.c.With(c.labels).Observe(n)
+	c.c.Observe(n)
 }
 
 func (c prometheusHistogram) UpdateDuration(n time.Time) {
-	c.c.With(c.labels).Observe(time.Since(n).Seconds())
+	c.c.Observe(time.Since(n).Seconds())
 }
 
 type prometheusSummary struct {
-	c      *prometheus.SummaryVec
-	lnames []string
-	labels prometheus.Labels
+	c      prometheus.Summary
+	labels []string
 }
 
 func (c prometheusSummary) Update(n float64) {
-	c.c.With(c.labels).Observe(n)
+	c.c.Observe(n)
 }
 
 func (c prometheusSummary) UpdateDuration(n time.Time) {
-	c.c.With(c.labels).Observe(time.Since(n).Seconds())
+	c.c.Observe(time.Since(n).Seconds())
+}
+
+func newHash(labels []string) uint64 {
+	labels = meter.BuildLabels(labels...)
+	h := fnv.New64a()
+	for _, l := range labels {
+		h.Write([]byte(l))
+	}
+	return h.Sum64()
+}
+
+func fillMetric(m *dto.Metric, labels []string) *dto.Metric {
+	m.Label = make([]*dto.LabelPair, 0, len(labels)/2)
+	for idx := 0; idx < len(labels); idx += 2 {
+		m.Label = append(m.Label, &dto.LabelPair{
+			Name:  newString(labels[idx]),
+			Value: newString(labels[idx+1]),
+		})
+	}
+	return m
 }

prometheus_test.go

@@ -13,9 +13,8 @@ import (
 
 func TestBuildName(t *testing.T) {
 	m := NewMeter()
-	im := m.(*prometheusMeter)
 	check := `micro_foo{micro_aaa="b",micro_bar="baz",micro_ccc="d"}`
-	name := im.buildMetric("foo", "bar", "baz", "aaa", "b", "ccc", "d")
+	name := m.buildMetric("foo", "bar", "baz", "aaa", "b", "ccc", "d")
 	if name != check {
 		t.Fatalf("metric name error: %s != %s", name, check)
 	}