diff --git a/go_metrics.go b/go_metrics.go new file mode 100644 index 0000000..f8b6067 --- /dev/null +++ b/go_metrics.go @@ -0,0 +1,64 @@ +package metrics + +import ( + "fmt" + "io" + "runtime" + + "github.com/valyala/histogram" +) + +func writeGoMetrics(w io.Writer) { + var ms runtime.MemStats + runtime.ReadMemStats(&ms) + fmt.Fprintf(w, "go_memstats_alloc_bytes %d\n", ms.Alloc) + fmt.Fprintf(w, "go_memstats_alloc_bytes_total %d\n", ms.TotalAlloc) + fmt.Fprintf(w, "go_memstats_buck_hash_sys_bytes %d\n", ms.BuckHashSys) + fmt.Fprintf(w, "go_memstats_frees_total %d\n", ms.Frees) + fmt.Fprintf(w, "go_memstats_gc_cpu_fraction %g\n", ms.GCCPUFraction) + fmt.Fprintf(w, "go_memstats_gc_sys_bytes %d\n", ms.GCSys) + fmt.Fprintf(w, "go_memstats_heap_alloc_bytes %d\n", ms.HeapAlloc) + fmt.Fprintf(w, "go_memstats_heap_idle_bytes %d\n", ms.HeapIdle) + fmt.Fprintf(w, "go_memstats_heap_inuse_bytes %d\n", ms.HeapInuse) + fmt.Fprintf(w, "go_memstats_heap_objects %d\n", ms.HeapObjects) + fmt.Fprintf(w, "go_memstats_heap_released_bytes %d\n", ms.HeapReleased) + fmt.Fprintf(w, "go_memstats_heap_sys_bytes %d\n", ms.HeapSys) + fmt.Fprintf(w, "go_memstats_last_gc_time_seconds %g\n", float64(ms.LastGC)/1e9) + fmt.Fprintf(w, "go_memstats_lookups_total %d\n", ms.Lookups) + fmt.Fprintf(w, "go_memstats_mallocs_total %d\n", ms.Mallocs) + fmt.Fprintf(w, "go_memstats_mcache_inuse_bytes %d\n", ms.MCacheInuse) + fmt.Fprintf(w, "go_memstats_mcache_sys_bytes %d\n", ms.MCacheSys) + fmt.Fprintf(w, "go_memstats_mspan_inuse_bytes %d\n", ms.MSpanInuse) + fmt.Fprintf(w, "go_memstats_mspan_sys_bytes %d\n", ms.MSpanSys) + fmt.Fprintf(w, "go_memstats_next_gc_bytes %d\n", ms.NextGC) + fmt.Fprintf(w, "go_memstats_other_sys_bytes %d\n", ms.OtherSys) + fmt.Fprintf(w, "go_memstats_stack_inuse_bytes %d\n", ms.StackInuse) + fmt.Fprintf(w, "go_memstats_stack_sys_bytes %d\n", ms.StackSys) + fmt.Fprintf(w, "go_memstats_sys_bytes %d\n", ms.Sys) + + fmt.Fprintf(w, "go_cgo_calls_count %d\n", runtime.NumCgoCall()) + fmt.Fprintf(w, "go_cpu_count %d\n", runtime.NumCPU()) + + gcPauses := histogram.NewFast() + for _, pauseNs := range ms.PauseNs[:] { + gcPauses.Update(float64(pauseNs) / 1e9) + } + phis := []float64{0, 0.25, 0.5, 0.75, 1} + quantiles := make([]float64, 0, len(phis)) + for i, q := range gcPauses.Quantiles(quantiles[:0], phis) { + fmt.Fprintf(w, `go_gc_duration_seconds{quantile="%g"} %g`+"\n", phis[i], q) + } + fmt.Fprintf(w, `go_gc_duration_seconds_sum %g`+"\n", float64(ms.PauseTotalNs)/1e9) + fmt.Fprintf(w, `go_gc_duration_seconds_count %d`+"\n", ms.NumGC) + fmt.Fprintf(w, `go_gc_forced_count %d`+"\n", ms.NumForcedGC) + + fmt.Fprintf(w, `go_gomaxprocs %d`+"\n", runtime.GOMAXPROCS(0)) + fmt.Fprintf(w, `go_goroutines %d`+"\n", runtime.NumGoroutine()) + numThread, _ := runtime.ThreadCreateProfile(nil) + fmt.Fprintf(w, `go_threads %d`+"\n", numThread) + + // Export build details. + fmt.Fprintf(w, "go_info{version=%q} 1\n", runtime.Version()) + fmt.Fprintf(w, "go_info_ext{compiler=%q, GOARCH=%q, GOOS=%q, GOROOT=%q} 1\n", + runtime.Compiler, runtime.GOARCH, runtime.GOOS, runtime.GOROOT()) +} diff --git a/metrics.go b/metrics.go index c28a08a..8fc8f6e 100644 --- a/metrics.go +++ b/metrics.go @@ -13,11 +13,7 @@ package metrics import ( - "fmt" "io" - "runtime" - - "github.com/valyala/histogram" ) type namedMetric struct { @@ -33,8 +29,8 @@ var defaultSet = NewSet() // WritePrometheus writes all the registered metrics in Prometheus format to w. // -// If exposeProcessMetrics is true, then various `go_*` metrics are exposed -// for the current process. +// If exposeProcessMetrics is true, then various `go_*` and `process_*` metrics +// are exposed for the current process. // // The WritePrometheus func is usually called inside "/metrics" handler: // @@ -45,61 +41,7 @@ var defaultSet = NewSet() func WritePrometheus(w io.Writer, exposeProcessMetrics bool) { defaultSet.WritePrometheus(w) if exposeProcessMetrics { + writeGoMetrics(w) writeProcessMetrics(w) } } - -func writeProcessMetrics(w io.Writer) { - var ms runtime.MemStats - runtime.ReadMemStats(&ms) - fmt.Fprintf(w, `go_memstats_alloc_bytes %d`+"\n", ms.Alloc) - fmt.Fprintf(w, `go_memstats_alloc_bytes_total %d`+"\n", ms.TotalAlloc) - fmt.Fprintf(w, `go_memstats_buck_hash_sys_bytes %d`+"\n", ms.BuckHashSys) - fmt.Fprintf(w, `go_memstats_frees_total %d`+"\n", ms.Frees) - fmt.Fprintf(w, `go_memstats_gc_cpu_fraction %f`+"\n", ms.GCCPUFraction) - fmt.Fprintf(w, `go_memstats_gc_sys_bytes %d`+"\n", ms.GCSys) - fmt.Fprintf(w, `go_memstats_heap_alloc_bytes %d`+"\n", ms.HeapAlloc) - fmt.Fprintf(w, `go_memstats_heap_idle_bytes %d`+"\n", ms.HeapIdle) - fmt.Fprintf(w, `go_memstats_heap_inuse_bytes %d`+"\n", ms.HeapInuse) - fmt.Fprintf(w, `go_memstats_heap_objects %d`+"\n", ms.HeapObjects) - fmt.Fprintf(w, `go_memstats_heap_released_bytes %d`+"\n", ms.HeapReleased) - fmt.Fprintf(w, `go_memstats_heap_sys_bytes %d`+"\n", ms.HeapSys) - fmt.Fprintf(w, `go_memstats_last_gc_time_seconds %f`+"\n", float64(ms.LastGC)/1e9) - fmt.Fprintf(w, `go_memstats_lookups_total %d`+"\n", ms.Lookups) - fmt.Fprintf(w, `go_memstats_mallocs_total %d`+"\n", ms.Mallocs) - fmt.Fprintf(w, `go_memstats_mcache_inuse_bytes %d`+"\n", ms.MCacheInuse) - fmt.Fprintf(w, `go_memstats_mcache_sys_bytes %d`+"\n", ms.MCacheSys) - fmt.Fprintf(w, `go_memstats_mspan_inuse_bytes %d`+"\n", ms.MSpanInuse) - fmt.Fprintf(w, `go_memstats_mspan_sys_bytes %d`+"\n", ms.MSpanSys) - fmt.Fprintf(w, `go_memstats_next_gc_bytes %d`+"\n", ms.NextGC) - fmt.Fprintf(w, `go_memstats_other_sys_bytes %d`+"\n", ms.OtherSys) - fmt.Fprintf(w, `go_memstats_stack_inuse_bytes %d`+"\n", ms.StackInuse) - fmt.Fprintf(w, `go_memstats_stack_sys_bytes %d`+"\n", ms.StackSys) - fmt.Fprintf(w, `go_memstats_sys_bytes %d`+"\n", ms.Sys) - - fmt.Fprintf(w, `go_cgo_calls_count %d`+"\n", runtime.NumCgoCall()) - fmt.Fprintf(w, `go_cpu_count %d`+"\n", runtime.NumCPU()) - - gcPauses := histogram.NewFast() - for _, pauseNs := range ms.PauseNs[:] { - gcPauses.Update(float64(pauseNs) / 1e9) - } - phis := []float64{0, 0.25, 0.5, 0.75, 1} - quantiles := make([]float64, 0, len(phis)) - for i, q := range gcPauses.Quantiles(quantiles[:0], phis) { - fmt.Fprintf(w, `go_gc_duration_seconds{quantile="%g"} %f`+"\n", phis[i], q) - } - fmt.Fprintf(w, `go_gc_duration_seconds_sum %f`+"\n", float64(ms.PauseTotalNs)/1e9) - fmt.Fprintf(w, `go_gc_duration_seconds_count %d`+"\n", ms.NumGC) - fmt.Fprintf(w, `go_gc_forced_count %d`+"\n", ms.NumForcedGC) - - fmt.Fprintf(w, `go_gomaxprocs %d`+"\n", runtime.GOMAXPROCS(0)) - fmt.Fprintf(w, `go_goroutines %d`+"\n", runtime.NumGoroutine()) - numThread, _ := runtime.ThreadCreateProfile(nil) - fmt.Fprintf(w, `go_threads %d`+"\n", numThread) - - // Export build details. - fmt.Fprintf(w, "go_info{version=%q} 1\n", runtime.Version()) - fmt.Fprintf(w, "go_info_ext{compiler=%q, GOARCH=%q, GOOS=%q, GOROOT=%q} 1\n", - runtime.Compiler, runtime.GOARCH, runtime.GOOS, runtime.GOROOT()) -} diff --git a/process_metrics_linux.go b/process_metrics_linux.go new file mode 100644 index 0000000..3f4744a --- /dev/null +++ b/process_metrics_linux.go @@ -0,0 +1,79 @@ +package metrics + +import ( + "bytes" + "fmt" + "io" + "io/ioutil" + "log" + "time" +) + +const statFilepath = "/proc/self/stat" + +// See https://github.com/prometheus/procfs/blob/a4ac0826abceb44c40fc71daed2b301db498b93e/proc_stat.go#L40 . +const userHZ = 100 + +// See http://man7.org/linux/man-pages/man5/proc.5.html +type procStat struct { + State byte + Ppid int + Pgrp int + Session int + TtyNr int + Tpgid int + Flags uint + Minflt uint + Cminflt uint + Majflt uint + Cmajflt uint + Utime uint + Stime uint + Cutime int + Cstime int + Priority int + Nice int + NumThreads int + ItrealValue int + Starttime uint64 + Vsize uint + Rss int +} + +func writeProcessMetrics(w io.Writer) { + data, err := ioutil.ReadFile(statFilepath) + if err != nil { + log.Printf("ERROR: cannot open %s: %s", statFilepath, err) + return + } + // Search for the end of command. + n := bytes.LastIndex(data, []byte(") ")) + if n < 0 { + log.Printf("ERROR: cannot find command in parentheses in %q read from %s", data, statFilepath) + return + } + data = data[n+2:] + + var p procStat + bb := bytes.NewBuffer(data) + _, err = fmt.Fscanf(bb, "%c %d %d %d %d %d %d %d %d %d %d %d %d %d %d %d %d %d %d %d %d %d", + &p.State, &p.Ppid, &p.Pgrp, &p.Session, &p.TtyNr, &p.Tpgid, &p.Flags, &p.Minflt, &p.Cminflt, &p.Majflt, &p.Cmajflt, + &p.Utime, &p.Stime, &p.Cutime, &p.Cstime, &p.Priority, &p.Nice, &p.NumThreads, &p.ItrealValue, &p.Starttime, &p.Vsize, &p.Rss) + if err != nil { + log.Printf("ERROR: cannot parse %q read from %s: %s", data, statFilepath, err) + return + } + + // It is expensive obtaining `process_open_fds` when big number of file descriptors is opened, + // don't do it here. + + fmt.Fprintf(w, "process_cpu_seconds_total %g\n", float64(p.Utime+p.Stime)/userHZ) + fmt.Fprintf(w, "process_major_pagefaults_total %d\n", p.Majflt) + fmt.Fprintf(w, "process_minor_pagefaults_total %d\n", p.Minflt) + fmt.Fprintf(w, "process_num_threads %d\n", p.NumThreads) + fmt.Fprintf(w, "process_resident_memory_bytes %d\n", p.Rss*4096) + fmt.Fprintf(w, "process_start_time_seconds %d\n", startTimeSeconds) + fmt.Fprintf(w, "process_virtual_memory_bytes %d\n", p.Vsize) +} + +var startTimeSeconds = time.Now().Unix() diff --git a/process_metrics_other.go b/process_metrics_other.go new file mode 100644 index 0000000..6874de3 --- /dev/null +++ b/process_metrics_other.go @@ -0,0 +1,11 @@ +// +build !linux + +package metrics + +import ( + "io" +) + +func writeProcessMetrics(w io.Writer) { + // TODO: implement it +}