Add process_* metrics similar to the metrics exposed by https://github.com/prometheus/client_golang

Updates https://github.com/VictoriaMetrics/VictoriaMetrics/issues/92

go_metrics.go

@@ -0,0 +1,64 @@
+package metrics
+
+import (
+	"fmt"
+	"io"
+	"runtime"
+
+	"github.com/valyala/histogram"
+)
+
+func writeGoMetrics(w io.Writer) {
+	var ms runtime.MemStats
+	runtime.ReadMemStats(&ms)
+	fmt.Fprintf(w, "go_memstats_alloc_bytes %d\n", ms.Alloc)
+	fmt.Fprintf(w, "go_memstats_alloc_bytes_total %d\n", ms.TotalAlloc)
+	fmt.Fprintf(w, "go_memstats_buck_hash_sys_bytes %d\n", ms.BuckHashSys)
+	fmt.Fprintf(w, "go_memstats_frees_total %d\n", ms.Frees)
+	fmt.Fprintf(w, "go_memstats_gc_cpu_fraction %g\n", ms.GCCPUFraction)
+	fmt.Fprintf(w, "go_memstats_gc_sys_bytes %d\n", ms.GCSys)
+	fmt.Fprintf(w, "go_memstats_heap_alloc_bytes %d\n", ms.HeapAlloc)
+	fmt.Fprintf(w, "go_memstats_heap_idle_bytes %d\n", ms.HeapIdle)
+	fmt.Fprintf(w, "go_memstats_heap_inuse_bytes %d\n", ms.HeapInuse)
+	fmt.Fprintf(w, "go_memstats_heap_objects %d\n", ms.HeapObjects)
+	fmt.Fprintf(w, "go_memstats_heap_released_bytes %d\n", ms.HeapReleased)
+	fmt.Fprintf(w, "go_memstats_heap_sys_bytes %d\n", ms.HeapSys)
+	fmt.Fprintf(w, "go_memstats_last_gc_time_seconds %g\n", float64(ms.LastGC)/1e9)
+	fmt.Fprintf(w, "go_memstats_lookups_total %d\n", ms.Lookups)
+	fmt.Fprintf(w, "go_memstats_mallocs_total %d\n", ms.Mallocs)
+	fmt.Fprintf(w, "go_memstats_mcache_inuse_bytes %d\n", ms.MCacheInuse)
+	fmt.Fprintf(w, "go_memstats_mcache_sys_bytes %d\n", ms.MCacheSys)
+	fmt.Fprintf(w, "go_memstats_mspan_inuse_bytes %d\n", ms.MSpanInuse)
+	fmt.Fprintf(w, "go_memstats_mspan_sys_bytes %d\n", ms.MSpanSys)
+	fmt.Fprintf(w, "go_memstats_next_gc_bytes %d\n", ms.NextGC)
+	fmt.Fprintf(w, "go_memstats_other_sys_bytes %d\n", ms.OtherSys)
+	fmt.Fprintf(w, "go_memstats_stack_inuse_bytes %d\n", ms.StackInuse)
+	fmt.Fprintf(w, "go_memstats_stack_sys_bytes %d\n", ms.StackSys)
+	fmt.Fprintf(w, "go_memstats_sys_bytes %d\n", ms.Sys)
+
+	fmt.Fprintf(w, "go_cgo_calls_count %d\n", runtime.NumCgoCall())
+	fmt.Fprintf(w, "go_cpu_count %d\n", runtime.NumCPU())
+
+	gcPauses := histogram.NewFast()
+	for _, pauseNs := range ms.PauseNs[:] {
+		gcPauses.Update(float64(pauseNs) / 1e9)
+	}
+	phis := []float64{0, 0.25, 0.5, 0.75, 1}
+	quantiles := make([]float64, 0, len(phis))
+	for i, q := range gcPauses.Quantiles(quantiles[:0], phis) {
+		fmt.Fprintf(w, `go_gc_duration_seconds{quantile="%g"} %g`+"\n", phis[i], q)
+	}
+	fmt.Fprintf(w, `go_gc_duration_seconds_sum %g`+"\n", float64(ms.PauseTotalNs)/1e9)
+	fmt.Fprintf(w, `go_gc_duration_seconds_count %d`+"\n", ms.NumGC)
+	fmt.Fprintf(w, `go_gc_forced_count %d`+"\n", ms.NumForcedGC)
+
+	fmt.Fprintf(w, `go_gomaxprocs %d`+"\n", runtime.GOMAXPROCS(0))
+	fmt.Fprintf(w, `go_goroutines %d`+"\n", runtime.NumGoroutine())
+	numThread, _ := runtime.ThreadCreateProfile(nil)
+	fmt.Fprintf(w, `go_threads %d`+"\n", numThread)
+
+	// Export build details.
+	fmt.Fprintf(w, "go_info{version=%q} 1\n", runtime.Version())
+	fmt.Fprintf(w, "go_info_ext{compiler=%q, GOARCH=%q, GOOS=%q, GOROOT=%q} 1\n",
+		runtime.Compiler, runtime.GOARCH, runtime.GOOS, runtime.GOROOT())
+}

metrics.go

@@ -13,11 +13,7 @@
 package metrics
 
 import (
-	"fmt"
 	"io"
-	"runtime"
-
-	"github.com/valyala/histogram"
 )
 
 type namedMetric struct {
@@ -33,8 +29,8 @@ var defaultSet = NewSet()
 
 // WritePrometheus writes all the registered metrics in Prometheus format to w.
 //
-// If exposeProcessMetrics is true, then various `go_*` metrics are exposed
+// If exposeProcessMetrics is true, then various `go_*` and `process_*` metrics
-// for the current process.
+// are exposed for the current process.
 //
 // The WritePrometheus func is usually called inside "/metrics" handler:
 //
@@ -45,61 +41,7 @@ var defaultSet = NewSet()
 func WritePrometheus(w io.Writer, exposeProcessMetrics bool) {
 	defaultSet.WritePrometheus(w)
 	if exposeProcessMetrics {
+		writeGoMetrics(w)
 		writeProcessMetrics(w)
 	}
 }
-
-func writeProcessMetrics(w io.Writer) {
-	var ms runtime.MemStats
-	runtime.ReadMemStats(&ms)
-	fmt.Fprintf(w, `go_memstats_alloc_bytes %d`+"\n", ms.Alloc)
-	fmt.Fprintf(w, `go_memstats_alloc_bytes_total %d`+"\n", ms.TotalAlloc)
-	fmt.Fprintf(w, `go_memstats_buck_hash_sys_bytes %d`+"\n", ms.BuckHashSys)
-	fmt.Fprintf(w, `go_memstats_frees_total %d`+"\n", ms.Frees)
-	fmt.Fprintf(w, `go_memstats_gc_cpu_fraction %f`+"\n", ms.GCCPUFraction)
-	fmt.Fprintf(w, `go_memstats_gc_sys_bytes %d`+"\n", ms.GCSys)
-	fmt.Fprintf(w, `go_memstats_heap_alloc_bytes %d`+"\n", ms.HeapAlloc)
-	fmt.Fprintf(w, `go_memstats_heap_idle_bytes %d`+"\n", ms.HeapIdle)
-	fmt.Fprintf(w, `go_memstats_heap_inuse_bytes %d`+"\n", ms.HeapInuse)
-	fmt.Fprintf(w, `go_memstats_heap_objects %d`+"\n", ms.HeapObjects)
-	fmt.Fprintf(w, `go_memstats_heap_released_bytes %d`+"\n", ms.HeapReleased)
-	fmt.Fprintf(w, `go_memstats_heap_sys_bytes %d`+"\n", ms.HeapSys)
-	fmt.Fprintf(w, `go_memstats_last_gc_time_seconds %f`+"\n", float64(ms.LastGC)/1e9)
-	fmt.Fprintf(w, `go_memstats_lookups_total %d`+"\n", ms.Lookups)
-	fmt.Fprintf(w, `go_memstats_mallocs_total %d`+"\n", ms.Mallocs)
-	fmt.Fprintf(w, `go_memstats_mcache_inuse_bytes %d`+"\n", ms.MCacheInuse)
-	fmt.Fprintf(w, `go_memstats_mcache_sys_bytes %d`+"\n", ms.MCacheSys)
-	fmt.Fprintf(w, `go_memstats_mspan_inuse_bytes %d`+"\n", ms.MSpanInuse)
-	fmt.Fprintf(w, `go_memstats_mspan_sys_bytes %d`+"\n", ms.MSpanSys)
-	fmt.Fprintf(w, `go_memstats_next_gc_bytes %d`+"\n", ms.NextGC)
-	fmt.Fprintf(w, `go_memstats_other_sys_bytes %d`+"\n", ms.OtherSys)
-	fmt.Fprintf(w, `go_memstats_stack_inuse_bytes %d`+"\n", ms.StackInuse)
-	fmt.Fprintf(w, `go_memstats_stack_sys_bytes %d`+"\n", ms.StackSys)
-	fmt.Fprintf(w, `go_memstats_sys_bytes %d`+"\n", ms.Sys)
-
-	fmt.Fprintf(w, `go_cgo_calls_count %d`+"\n", runtime.NumCgoCall())
-	fmt.Fprintf(w, `go_cpu_count %d`+"\n", runtime.NumCPU())
-
-	gcPauses := histogram.NewFast()
-	for _, pauseNs := range ms.PauseNs[:] {
-		gcPauses.Update(float64(pauseNs) / 1e9)
-	}
-	phis := []float64{0, 0.25, 0.5, 0.75, 1}
-	quantiles := make([]float64, 0, len(phis))
-	for i, q := range gcPauses.Quantiles(quantiles[:0], phis) {
-		fmt.Fprintf(w, `go_gc_duration_seconds{quantile="%g"} %f`+"\n", phis[i], q)
-	}
-	fmt.Fprintf(w, `go_gc_duration_seconds_sum %f`+"\n", float64(ms.PauseTotalNs)/1e9)
-	fmt.Fprintf(w, `go_gc_duration_seconds_count %d`+"\n", ms.NumGC)
-	fmt.Fprintf(w, `go_gc_forced_count %d`+"\n", ms.NumForcedGC)
-
-	fmt.Fprintf(w, `go_gomaxprocs %d`+"\n", runtime.GOMAXPROCS(0))
-	fmt.Fprintf(w, `go_goroutines %d`+"\n", runtime.NumGoroutine())
-	numThread, _ := runtime.ThreadCreateProfile(nil)
-	fmt.Fprintf(w, `go_threads %d`+"\n", numThread)
-
-	// Export build details.
-	fmt.Fprintf(w, "go_info{version=%q} 1\n", runtime.Version())
-	fmt.Fprintf(w, "go_info_ext{compiler=%q, GOARCH=%q, GOOS=%q, GOROOT=%q} 1\n",
-		runtime.Compiler, runtime.GOARCH, runtime.GOOS, runtime.GOROOT())
-}

process_metrics_linux.go

@@ -0,0 +1,79 @@
+package metrics
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"log"
+	"time"
+)
+
+const statFilepath = "/proc/self/stat"
+
+// See https://github.com/prometheus/procfs/blob/a4ac0826abceb44c40fc71daed2b301db498b93e/proc_stat.go#L40 .
+const userHZ = 100
+
+// See http://man7.org/linux/man-pages/man5/proc.5.html
+type procStat struct {
+	State       byte
+	Ppid        int
+	Pgrp        int
+	Session     int
+	TtyNr       int
+	Tpgid       int
+	Flags       uint
+	Minflt      uint
+	Cminflt     uint
+	Majflt      uint
+	Cmajflt     uint
+	Utime       uint
+	Stime       uint
+	Cutime      int
+	Cstime      int
+	Priority    int
+	Nice        int
+	NumThreads  int
+	ItrealValue int
+	Starttime   uint64
+	Vsize       uint
+	Rss         int
+}
+
+func writeProcessMetrics(w io.Writer) {
+	data, err := ioutil.ReadFile(statFilepath)
+	if err != nil {
+		log.Printf("ERROR: cannot open %s: %s", statFilepath, err)
+		return
+	}
+	// Search for the end of command.
+	n := bytes.LastIndex(data, []byte(") "))
+	if n < 0 {
+		log.Printf("ERROR: cannot find command in parentheses in %q read from %s", data, statFilepath)
+		return
+	}
+	data = data[n+2:]
+
+	var p procStat
+	bb := bytes.NewBuffer(data)
+	_, err = fmt.Fscanf(bb, "%c %d %d %d %d %d %d %d %d %d %d %d %d %d %d %d %d %d %d %d %d %d",
+		&p.State, &p.Ppid, &p.Pgrp, &p.Session, &p.TtyNr, &p.Tpgid, &p.Flags, &p.Minflt, &p.Cminflt, &p.Majflt, &p.Cmajflt,
+		&p.Utime, &p.Stime, &p.Cutime, &p.Cstime, &p.Priority, &p.Nice, &p.NumThreads, &p.ItrealValue, &p.Starttime, &p.Vsize, &p.Rss)
+	if err != nil {
+		log.Printf("ERROR: cannot parse %q read from %s: %s", data, statFilepath, err)
+		return
+	}
+
+	// It is expensive obtaining `process_open_fds` when big number of file descriptors is opened,
+	// don't do it here.
+
+	fmt.Fprintf(w, "process_cpu_seconds_total %g\n", float64(p.Utime+p.Stime)/userHZ)
+	fmt.Fprintf(w, "process_major_pagefaults_total %d\n", p.Majflt)
+	fmt.Fprintf(w, "process_minor_pagefaults_total %d\n", p.Minflt)
+	fmt.Fprintf(w, "process_num_threads %d\n", p.NumThreads)
+	fmt.Fprintf(w, "process_resident_memory_bytes %d\n", p.Rss*4096)
+	fmt.Fprintf(w, "process_start_time_seconds %d\n", startTimeSeconds)
+	fmt.Fprintf(w, "process_virtual_memory_bytes %d\n", p.Vsize)
+}
+
+var startTimeSeconds = time.Now().Unix()

process_metrics_other.go

@@ -0,0 +1,11 @@
+// +build !linux
+
+package metrics
+
+import (
+	"io"
+)
+
+func writeProcessMetrics(w io.Writer) {
+	// TODO: implement it
+}