package metrics import ( "bytes" "fmt" "io" "log" "net/http" "time" "compress/gzip" ) // InitPushProcessMetrics sets up periodic push for 'process_*' metrics to the given pushURL with the given interval. // // extraLabels may contain comma-separated list of `label="value"` labels, which will be added // to all the metrics before pushing them to pushURL. // // The metrics are pushed to pushURL in Prometheus text exposition format. // See https://github.com/prometheus/docs/blob/main/content/docs/instrumenting/exposition_formats.md#text-based-format // // It is recommended pushing metrics to /api/v1/import/prometheus endpoint according to // https://docs.victoriametrics.com/#how-to-import-data-in-prometheus-exposition-format // // It is OK calling InitPushProcessMetrics multiple times with different pushURL - // in this case metrics are pushed to all the provided pushURL urls. func InitPushProcessMetrics(pushURL string, interval time.Duration, extraLabels string) error { writeMetrics := func(w io.Writer) { WriteProcessMetrics(w) } return InitPushExt(pushURL, interval, extraLabels, writeMetrics) } // InitPush sets up periodic push for globally registered metrics to the given pushURL with the given interval. // // extraLabels may contain comma-separated list of `label="value"` labels, which will be added // to all the metrics before pushing them to pushURL. // // If pushProcessMetrics is set to true, then 'process_*' metrics are also pushed to pushURL. // // The metrics are pushed to pushURL in Prometheus text exposition format. // See https://github.com/prometheus/docs/blob/main/content/docs/instrumenting/exposition_formats.md#text-based-format // // It is recommended pushing metrics to /api/v1/import/prometheus endpoint according to // https://docs.victoriametrics.com/#how-to-import-data-in-prometheus-exposition-format // // It is OK calling InitPush multiple times with different pushURL - // in this case metrics are pushed to all the provided pushURL urls. func InitPush(pushURL string, interval time.Duration, extraLabels string, pushProcessMetrics bool) error { writeMetrics := func(w io.Writer) { WritePrometheus(w, pushProcessMetrics) } return InitPushExt(pushURL, interval, extraLabels, writeMetrics) } // InitPush sets up periodic push for metrics from s to the given pushURL with the given interval. // // extraLabels may contain comma-separated list of `label="value"` labels, which will be added // to all the metrics before pushing them to pushURL. // // / The metrics are pushed to pushURL in Prometheus text exposition format. // See https://github.com/prometheus/docs/blob/main/content/docs/instrumenting/exposition_formats.md#text-based-format // // It is recommended pushing metrics to /api/v1/import/prometheus endpoint according to // https://docs.victoriametrics.com/#how-to-import-data-in-prometheus-exposition-format // // It is OK calling InitPush multiple times with different pushURL - // in this case metrics are pushed to all the provided pushURL urls. func (s *Set) InitPush(pushURL string, interval time.Duration, extraLabels string) error { writeMetrics := func(w io.Writer) { s.WritePrometheus(w) } return InitPushExt(pushURL, interval, extraLabels, writeMetrics) } // InitPushExt sets up periodic push for metrics obtained by calling writeMetrics with the given interval. // // extraLabels may contain comma-separated list of `label="value"` labels, which will be added // to all the metrics before pushing them to pushURL. // // The writeMetrics callback must write metrics to w in Prometheus text exposition format without timestamps and trailing comments. // See https://github.com/prometheus/docs/blob/main/content/docs/instrumenting/exposition_formats.md#text-based-format // // It is recommended pushing metrics to /api/v1/import/prometheus endpoint according to // https://docs.victoriametrics.com/#how-to-import-data-in-prometheus-exposition-format // // It is OK calling InitPushExt multiple times with different pushURL - // in this case metrics are pushed to all the provided pushURL urls. // // It is OK calling InitPushExt multiple times with different writeMetrics - // in this case all the metrics generated by writeMetrics callbacks are written to pushURL. func InitPushExt(pushURL string, interval time.Duration, extraLabels string, writeMetrics func(w io.Writer)) error { cfg := PushConfig{ URL: pushURL, Interval: interval, ExtraLabels: extraLabels, WriteMetricsFn: writeMetrics, } return InitPushWithConfig(cfg) } // InitPushWithConfig sets up periodic push for metrics based on params from passed PushConfig. // // It is recommended pushing metrics to /api/v1/import/prometheus endpoint according to // https://docs.victoriametrics.com/#how-to-import-data-in-prometheus-exposition-format // // It is OK calling InitPushWithConfig multiple times with different PushConfig.URL - // in this case metrics are pushed to all the provided PushConfig.URL urls. // // It is OK calling InitPushWithConfig multiple times with different PushConfig.WriteMetricsFn - // in this case all the metrics generated by PushConfig.WriteMetricsFn callbacks are written to PushConfig.URL. // // If func InitPushWithConfig(cfg PushConfig) error { if err := cfg.Validate(); err != nil { return fmt.Errorf("failed to validate PushConfig: %w", err) } writeMetrics := cfg.WriteMetricsFn if writeMetrics == nil { writeMetrics = func(w io.Writer) { WritePrometheus(w, true) } } pushURL := cfg.pushURL pushURLRedacted := pushURL.Redacted() interval := cfg.Interval c := &http.Client{ Timeout: interval, } pushesTotal := pushMetrics.GetOrCreateCounter(fmt.Sprintf(`metrics_push_total{url=%q}`, pushURLRedacted)) pushErrorsTotal := pushMetrics.GetOrCreateCounter(fmt.Sprintf(`metrics_push_errors_total{url=%q}`, pushURLRedacted)) bytesPushedTotal := pushMetrics.GetOrCreateCounter(fmt.Sprintf(`metrics_push_bytes_pushed_total{url=%q}`, pushURLRedacted)) pushDuration := pushMetrics.GetOrCreateHistogram(fmt.Sprintf(`metrics_push_duration_seconds{url=%q}`, pushURLRedacted)) pushBlockSize := pushMetrics.GetOrCreateHistogram(fmt.Sprintf(`metrics_push_block_size_bytes{url=%q}`, pushURLRedacted)) pushMetrics.GetOrCreateFloatCounter(fmt.Sprintf(`metrics_push_interval_seconds{url=%q}`, pushURLRedacted)).Set(interval.Seconds()) extraLabels := cfg.ExtraLabels go func() { ticker := time.NewTicker(interval) var bb bytes.Buffer var tmpBuf []byte zw := gzip.NewWriter(&bb) for range ticker.C { bb.Reset() writeMetrics(&bb) if len(extraLabels) > 0 { tmpBuf = addExtraLabels(tmpBuf[:0], bb.Bytes(), extraLabels) bb.Reset() if _, err := bb.Write(tmpBuf); err != nil { panic(fmt.Errorf("BUG: cannot write %d bytes to bytes.Buffer: %s", len(tmpBuf), err)) } } tmpBuf = append(tmpBuf[:0], bb.Bytes()...) bb.Reset() zw.Reset(&bb) if _, err := zw.Write(tmpBuf); err != nil { panic(fmt.Errorf("BUG: cannot write %d bytes to gzip writer: %s", len(tmpBuf), err)) } if err := zw.Close(); err != nil { panic(fmt.Errorf("BUG: cannot flush metrics to gzip writer: %s", err)) } pushesTotal.Inc() blockLen := bb.Len() bytesPushedTotal.Add(blockLen) pushBlockSize.Update(float64(blockLen)) req, err := http.NewRequest("GET", pushURL.String(), &bb) if err != nil { panic(fmt.Errorf("BUG: metrics.push: cannot initialize request for metrics push to %q: %w", pushURLRedacted, err)) } req.Header.Set("Content-Type", "text/plain") req.Header.Set("Content-Encoding", "gzip") for k, vs := range cfg.Headers { for _, v := range vs { req.Header.Add(k, v) } } startTime := time.Now() resp, err := c.Do(req) pushDuration.UpdateDuration(startTime) if err != nil { log.Printf("ERROR: metrics.push: cannot push metrics to %q: %s", pushURLRedacted, err) pushErrorsTotal.Inc() continue } if resp.StatusCode/100 != 2 { body, _ := io.ReadAll(resp.Body) _ = resp.Body.Close() log.Printf("ERROR: metrics.push: unexpected status code in response from %q: %d; expecting 2xx; response body: %q", pushURLRedacted, resp.StatusCode, body) pushErrorsTotal.Inc() continue } _ = resp.Body.Close() } }() return nil } var pushMetrics = NewSet() func writePushMetrics(w io.Writer) { pushMetrics.WritePrometheus(w) } func addExtraLabels(dst, src []byte, extraLabels string) []byte { for len(src) > 0 { var line []byte n := bytes.IndexByte(src, '\n') if n >= 0 { line = src[:n] src = src[n+1:] } else { line = src src = nil } line = bytes.TrimSpace(line) if len(line) == 0 { // Skip empy lines continue } if bytes.HasPrefix(line, bashBytes) { // Copy comments as is dst = append(dst, line...) dst = append(dst, '\n') continue } n = bytes.IndexByte(line, '{') if n >= 0 { dst = append(dst, line[:n+1]...) dst = append(dst, extraLabels...) dst = append(dst, ',') dst = append(dst, line[n+1:]...) } else { n = bytes.LastIndexByte(line, ' ') if n < 0 { panic(fmt.Errorf("BUG: missing whitespace between metric name and metric value in Prometheus text exposition line %q", line)) } dst = append(dst, line[:n]...) dst = append(dst, '{') dst = append(dst, extraLabels...) dst = append(dst, '}') dst = append(dst, line[n:]...) } dst = append(dst, '\n') } return dst } var bashBytes = []byte("#")