Add process_* metrics similar to the metrics exposed by https://github.com/prometheus/client_golang
Updates https://github.com/VictoriaMetrics/VictoriaMetrics/issues/92
This commit is contained in:
Aliaksandr Valialkin
parent
2c308dd067
commit
5a9cdd0bef
64
go_metrics.go
Normal file
64
go_metrics.go
Normal file
@ -0,0 +1,64 @@
|
||||
package metrics
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io"
|
||||
"runtime"
|
||||
|
||||
"github.com/valyala/histogram"
|
||||
)
|
||||
|
||||
func writeGoMetrics(w io.Writer) {
|
||||
var ms runtime.MemStats
|
||||
runtime.ReadMemStats(&ms)
|
||||
fmt.Fprintf(w, "go_memstats_alloc_bytes %d\n", ms.Alloc)
|
||||
fmt.Fprintf(w, "go_memstats_alloc_bytes_total %d\n", ms.TotalAlloc)
|
||||
fmt.Fprintf(w, "go_memstats_buck_hash_sys_bytes %d\n", ms.BuckHashSys)
|
||||
fmt.Fprintf(w, "go_memstats_frees_total %d\n", ms.Frees)
|
||||
fmt.Fprintf(w, "go_memstats_gc_cpu_fraction %g\n", ms.GCCPUFraction)
|
||||
fmt.Fprintf(w, "go_memstats_gc_sys_bytes %d\n", ms.GCSys)
|
||||
fmt.Fprintf(w, "go_memstats_heap_alloc_bytes %d\n", ms.HeapAlloc)
|
||||
fmt.Fprintf(w, "go_memstats_heap_idle_bytes %d\n", ms.HeapIdle)
|
||||
fmt.Fprintf(w, "go_memstats_heap_inuse_bytes %d\n", ms.HeapInuse)
|
||||
fmt.Fprintf(w, "go_memstats_heap_objects %d\n", ms.HeapObjects)
|
||||
fmt.Fprintf(w, "go_memstats_heap_released_bytes %d\n", ms.HeapReleased)
|
||||
fmt.Fprintf(w, "go_memstats_heap_sys_bytes %d\n", ms.HeapSys)
|
||||
fmt.Fprintf(w, "go_memstats_last_gc_time_seconds %g\n", float64(ms.LastGC)/1e9)
|
||||
fmt.Fprintf(w, "go_memstats_lookups_total %d\n", ms.Lookups)
|
||||
fmt.Fprintf(w, "go_memstats_mallocs_total %d\n", ms.Mallocs)
|
||||
fmt.Fprintf(w, "go_memstats_mcache_inuse_bytes %d\n", ms.MCacheInuse)
|
||||
fmt.Fprintf(w, "go_memstats_mcache_sys_bytes %d\n", ms.MCacheSys)
|
||||
fmt.Fprintf(w, "go_memstats_mspan_inuse_bytes %d\n", ms.MSpanInuse)
|
||||
fmt.Fprintf(w, "go_memstats_mspan_sys_bytes %d\n", ms.MSpanSys)
|
||||
fmt.Fprintf(w, "go_memstats_next_gc_bytes %d\n", ms.NextGC)
|
||||
fmt.Fprintf(w, "go_memstats_other_sys_bytes %d\n", ms.OtherSys)
|
||||
fmt.Fprintf(w, "go_memstats_stack_inuse_bytes %d\n", ms.StackInuse)
|
||||
fmt.Fprintf(w, "go_memstats_stack_sys_bytes %d\n", ms.StackSys)
|
||||
fmt.Fprintf(w, "go_memstats_sys_bytes %d\n", ms.Sys)
|
||||
|
||||
fmt.Fprintf(w, "go_cgo_calls_count %d\n", runtime.NumCgoCall())
|
||||
fmt.Fprintf(w, "go_cpu_count %d\n", runtime.NumCPU())
|
||||
|
||||
gcPauses := histogram.NewFast()
|
||||
for _, pauseNs := range ms.PauseNs[:] {
|
||||
gcPauses.Update(float64(pauseNs) / 1e9)
|
||||
}
|
||||
phis := []float64{0, 0.25, 0.5, 0.75, 1}
|
||||
quantiles := make([]float64, 0, len(phis))
|
||||
for i, q := range gcPauses.Quantiles(quantiles[:0], phis) {
|
||||
fmt.Fprintf(w, `go_gc_duration_seconds{quantile="%g"} %g`+"\n", phis[i], q)
|
||||
}
|
||||
fmt.Fprintf(w, `go_gc_duration_seconds_sum %g`+"\n", float64(ms.PauseTotalNs)/1e9)
|
||||
fmt.Fprintf(w, `go_gc_duration_seconds_count %d`+"\n", ms.NumGC)
|
||||
fmt.Fprintf(w, `go_gc_forced_count %d`+"\n", ms.NumForcedGC)
|
||||
|
||||
fmt.Fprintf(w, `go_gomaxprocs %d`+"\n", runtime.GOMAXPROCS(0))
|
||||
fmt.Fprintf(w, `go_goroutines %d`+"\n", runtime.NumGoroutine())
|
||||
numThread, _ := runtime.ThreadCreateProfile(nil)
|
||||
fmt.Fprintf(w, `go_threads %d`+"\n", numThread)
|
||||
|
||||
// Export build details.
|
||||
fmt.Fprintf(w, "go_info{version=%q} 1\n", runtime.Version())
|
||||
fmt.Fprintf(w, "go_info_ext{compiler=%q, GOARCH=%q, GOOS=%q, GOROOT=%q} 1\n",
|
||||
runtime.Compiler, runtime.GOARCH, runtime.GOOS, runtime.GOROOT())
|
||||
}
|
||||
64
metrics.go
64
metrics.go
@ -13,11 +13,7 @@
|
||||
package metrics
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io"
|
||||
"runtime"
|
||||
|
||||
"github.com/valyala/histogram"
|
||||
)
|
||||
|
||||
type namedMetric struct {
|
||||
@ -33,8 +29,8 @@ var defaultSet = NewSet()
|
||||
|
||||
// WritePrometheus writes all the registered metrics in Prometheus format to w.
|
||||
//
|
||||
// If exposeProcessMetrics is true, then various `go_*` metrics are exposed
|
||||
// for the current process.
|
||||
// If exposeProcessMetrics is true, then various `go_*` and `process_*` metrics
|
||||
// are exposed for the current process.
|
||||
//
|
||||
// The WritePrometheus func is usually called inside "/metrics" handler:
|
||||
//
|
||||
@ -45,61 +41,7 @@ var defaultSet = NewSet()
|
||||
func WritePrometheus(w io.Writer, exposeProcessMetrics bool) {
|
||||
defaultSet.WritePrometheus(w)
|
||||
if exposeProcessMetrics {
|
||||
writeGoMetrics(w)
|
||||
writeProcessMetrics(w)
|
||||
}
|
||||
}
|
||||
|
||||
func writeProcessMetrics(w io.Writer) {
|
||||
var ms runtime.MemStats
|
||||
runtime.ReadMemStats(&ms)
|
||||
fmt.Fprintf(w, `go_memstats_alloc_bytes %d`+"\n", ms.Alloc)
|
||||
fmt.Fprintf(w, `go_memstats_alloc_bytes_total %d`+"\n", ms.TotalAlloc)
|
||||
fmt.Fprintf(w, `go_memstats_buck_hash_sys_bytes %d`+"\n", ms.BuckHashSys)
|
||||
fmt.Fprintf(w, `go_memstats_frees_total %d`+"\n", ms.Frees)
|
||||
fmt.Fprintf(w, `go_memstats_gc_cpu_fraction %f`+"\n", ms.GCCPUFraction)
|
||||
fmt.Fprintf(w, `go_memstats_gc_sys_bytes %d`+"\n", ms.GCSys)
|
||||
fmt.Fprintf(w, `go_memstats_heap_alloc_bytes %d`+"\n", ms.HeapAlloc)
|
||||
fmt.Fprintf(w, `go_memstats_heap_idle_bytes %d`+"\n", ms.HeapIdle)
|
||||
fmt.Fprintf(w, `go_memstats_heap_inuse_bytes %d`+"\n", ms.HeapInuse)
|
||||
fmt.Fprintf(w, `go_memstats_heap_objects %d`+"\n", ms.HeapObjects)
|
||||
fmt.Fprintf(w, `go_memstats_heap_released_bytes %d`+"\n", ms.HeapReleased)
|
||||
fmt.Fprintf(w, `go_memstats_heap_sys_bytes %d`+"\n", ms.HeapSys)
|
||||
fmt.Fprintf(w, `go_memstats_last_gc_time_seconds %f`+"\n", float64(ms.LastGC)/1e9)
|
||||
fmt.Fprintf(w, `go_memstats_lookups_total %d`+"\n", ms.Lookups)
|
||||
fmt.Fprintf(w, `go_memstats_mallocs_total %d`+"\n", ms.Mallocs)
|
||||
fmt.Fprintf(w, `go_memstats_mcache_inuse_bytes %d`+"\n", ms.MCacheInuse)
|
||||
fmt.Fprintf(w, `go_memstats_mcache_sys_bytes %d`+"\n", ms.MCacheSys)
|
||||
fmt.Fprintf(w, `go_memstats_mspan_inuse_bytes %d`+"\n", ms.MSpanInuse)
|
||||
fmt.Fprintf(w, `go_memstats_mspan_sys_bytes %d`+"\n", ms.MSpanSys)
|
||||
fmt.Fprintf(w, `go_memstats_next_gc_bytes %d`+"\n", ms.NextGC)
|
||||
fmt.Fprintf(w, `go_memstats_other_sys_bytes %d`+"\n", ms.OtherSys)
|
||||
fmt.Fprintf(w, `go_memstats_stack_inuse_bytes %d`+"\n", ms.StackInuse)
|
||||
fmt.Fprintf(w, `go_memstats_stack_sys_bytes %d`+"\n", ms.StackSys)
|
||||
fmt.Fprintf(w, `go_memstats_sys_bytes %d`+"\n", ms.Sys)
|
||||
|
||||
fmt.Fprintf(w, `go_cgo_calls_count %d`+"\n", runtime.NumCgoCall())
|
||||
fmt.Fprintf(w, `go_cpu_count %d`+"\n", runtime.NumCPU())
|
||||
|
||||
gcPauses := histogram.NewFast()
|
||||
for _, pauseNs := range ms.PauseNs[:] {
|
||||
gcPauses.Update(float64(pauseNs) / 1e9)
|
||||
}
|
||||
phis := []float64{0, 0.25, 0.5, 0.75, 1}
|
||||
quantiles := make([]float64, 0, len(phis))
|
||||
for i, q := range gcPauses.Quantiles(quantiles[:0], phis) {
|
||||
fmt.Fprintf(w, `go_gc_duration_seconds{quantile="%g"} %f`+"\n", phis[i], q)
|
||||
}
|
||||
fmt.Fprintf(w, `go_gc_duration_seconds_sum %f`+"\n", float64(ms.PauseTotalNs)/1e9)
|
||||
fmt.Fprintf(w, `go_gc_duration_seconds_count %d`+"\n", ms.NumGC)
|
||||
fmt.Fprintf(w, `go_gc_forced_count %d`+"\n", ms.NumForcedGC)
|
||||
|
||||
fmt.Fprintf(w, `go_gomaxprocs %d`+"\n", runtime.GOMAXPROCS(0))
|
||||
fmt.Fprintf(w, `go_goroutines %d`+"\n", runtime.NumGoroutine())
|
||||
numThread, _ := runtime.ThreadCreateProfile(nil)
|
||||
fmt.Fprintf(w, `go_threads %d`+"\n", numThread)
|
||||
|
||||
// Export build details.
|
||||
fmt.Fprintf(w, "go_info{version=%q} 1\n", runtime.Version())
|
||||
fmt.Fprintf(w, "go_info_ext{compiler=%q, GOARCH=%q, GOOS=%q, GOROOT=%q} 1\n",
|
||||
runtime.Compiler, runtime.GOARCH, runtime.GOOS, runtime.GOROOT())
|
||||
}
|
||||
|
||||
79
process_metrics_linux.go
Normal file
79
process_metrics_linux.go
Normal file
@ -0,0 +1,79 @@
|
||||
package metrics
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"log"
|
||||
"time"
|
||||
)
|
||||
|
||||
const statFilepath = "/proc/self/stat"
|
||||
|
||||
// See https://github.com/prometheus/procfs/blob/a4ac0826abceb44c40fc71daed2b301db498b93e/proc_stat.go#L40 .
|
||||
const userHZ = 100
|
||||
|
||||
// See http://man7.org/linux/man-pages/man5/proc.5.html
|
||||
type procStat struct {
|
||||
State byte
|
||||
Ppid int
|
||||
Pgrp int
|
||||
Session int
|
||||
TtyNr int
|
||||
Tpgid int
|
||||
Flags uint
|
||||
Minflt uint
|
||||
Cminflt uint
|
||||
Majflt uint
|
||||
Cmajflt uint
|
||||
Utime uint
|
||||
Stime uint
|
||||
Cutime int
|
||||
Cstime int
|
||||
Priority int
|
||||
Nice int
|
||||
NumThreads int
|
||||
ItrealValue int
|
||||
Starttime uint64
|
||||
Vsize uint
|
||||
Rss int
|
||||
}
|
||||
|
||||
func writeProcessMetrics(w io.Writer) {
|
||||
data, err := ioutil.ReadFile(statFilepath)
|
||||
if err != nil {
|
||||
log.Printf("ERROR: cannot open %s: %s", statFilepath, err)
|
||||
return
|
||||
}
|
||||
// Search for the end of command.
|
||||
n := bytes.LastIndex(data, []byte(") "))
|
||||
if n < 0 {
|
||||
log.Printf("ERROR: cannot find command in parentheses in %q read from %s", data, statFilepath)
|
||||
return
|
||||
}
|
||||
data = data[n+2:]
|
||||
|
||||
var p procStat
|
||||
bb := bytes.NewBuffer(data)
|
||||
_, err = fmt.Fscanf(bb, "%c %d %d %d %d %d %d %d %d %d %d %d %d %d %d %d %d %d %d %d %d %d",
|
||||
&p.State, &p.Ppid, &p.Pgrp, &p.Session, &p.TtyNr, &p.Tpgid, &p.Flags, &p.Minflt, &p.Cminflt, &p.Majflt, &p.Cmajflt,
|
||||
&p.Utime, &p.Stime, &p.Cutime, &p.Cstime, &p.Priority, &p.Nice, &p.NumThreads, &p.ItrealValue, &p.Starttime, &p.Vsize, &p.Rss)
|
||||
if err != nil {
|
||||
log.Printf("ERROR: cannot parse %q read from %s: %s", data, statFilepath, err)
|
||||
return
|
||||
}
|
||||
|
||||
// It is expensive obtaining `process_open_fds` when big number of file descriptors is opened,
|
||||
// don't do it here.
|
||||
|
||||
fmt.Fprintf(w, "process_cpu_seconds_total %g\n", float64(p.Utime+p.Stime)/userHZ)
|
||||
fmt.Fprintf(w, "process_major_pagefaults_total %d\n", p.Majflt)
|
||||
fmt.Fprintf(w, "process_minor_pagefaults_total %d\n", p.Minflt)
|
||||
fmt.Fprintf(w, "process_num_threads %d\n", p.NumThreads)
|
||||
fmt.Fprintf(w, "process_resident_memory_bytes %d\n", p.Rss*4096)
|
||||
fmt.Fprintf(w, "process_start_time_seconds %d\n", startTimeSeconds)
|
||||
fmt.Fprintf(w, "process_virtual_memory_bytes %d\n", p.Vsize)
|
||||
}
|
||||
|
||||
var startTimeSeconds = time.Now().Unix()
|
||||
11
process_metrics_other.go
Normal file
11
process_metrics_other.go
Normal file
@ -0,0 +1,11 @@
|
||||
// +build !linux
|
||||
|
||||
package metrics
|
||||
|
||||
import (
|
||||
"io"
|
||||
)
|
||||
|
||||
func writeProcessMetrics(w io.Writer) {
|
||||
// TODO: implement it
|
||||
}
|
||||
Loading…
Reference in New Issue
Block a user