From 5a49bb8e88e070e43cbffaa68776259f11f6c053 Mon Sep 17 00:00:00 2001 From: Nikolay Date: Thu, 18 Mar 2021 00:07:19 +0300 Subject: [PATCH] adds extended memory stats (#21) --- process_metrics_linux.go | 65 ++++++++++++++++++- process_metrics_linux_test.go | 15 +++++ testdata/status | 115 ++++++++++++++++++++++++++++++++++ testdata/status_bad | 115 ++++++++++++++++++++++++++++++++++ 4 files changed, 309 insertions(+), 1 deletion(-) create mode 100644 testdata/status create mode 100644 testdata/status_bad diff --git a/process_metrics_linux.go b/process_metrics_linux.go index eae9b24..a35d293 100644 --- a/process_metrics_linux.go +++ b/process_metrics_linux.go @@ -90,7 +90,7 @@ func writeProcessMetrics(w io.Writer) { fmt.Fprintf(w, "process_resident_memory_pagecache_bytes %d\n", rssPageCache) fmt.Fprintf(w, "process_start_time_seconds %d\n", startTimeSeconds) fmt.Fprintf(w, "process_virtual_memory_bytes %d\n", p.Vsize) - + writeProcessMemMetrics(w) writeIOMetrics(w) } @@ -208,6 +208,69 @@ func getMaxFilesLimit(path string) (uint64, error) { return 0, fmt.Errorf("cannot find max open files limit") } +// https://man7.org/linux/man-pages/man5/procfs.5.html +type memStats struct { + vmPeak uint64 + rssPeak uint64 + rssAnon uint64 + rssFile uint64 + rssShmem uint64 +} + +func writeProcessMemMetrics(w io.Writer) { + ms, err := getMemStats("/proc/self/status") + if err != nil { + log.Printf("ERROR: cannot determine memory status: %s", err) + return + } + fmt.Fprintf(w, "process_virtual_memory_peak_bytes %d\n", ms.vmPeak) + fmt.Fprintf(w, "process_resident_memory_peak_bytes %d\n", ms.rssPeak) + fmt.Fprintf(w, "process_resident_memory_anon_bytes %d\n", ms.rssAnon) + fmt.Fprintf(w, "process_resident_memory_file_bytes %d\n", ms.rssFile) + fmt.Fprintf(w, "process_resident_memory_shared_bytes %d\n", ms.rssShmem) + +} + +func getMemStats(path string) (*memStats, error) { + data, err := ioutil.ReadFile(path) + if err != nil { + return nil, err + } + var ms memStats + lines := strings.Split(string(data), "\n") + for _, s := range lines { + if !strings.HasPrefix(s, "Vm") && !strings.HasPrefix(s, "Rss") { + continue + } + // Extract key value. + line := strings.Fields(s) + if len(line) != 3 { + return nil, fmt.Errorf("cannot extract soft limit from %q", s) + } + memStatName := line[0] + memStatValue := line[1] + value, err := strconv.ParseUint(memStatValue, 10, 64) + if err != nil { + return nil, fmt.Errorf("cannot parse soft limit from %q: %s", s, err) + } + if line[2] == "kB" { + value = value * 1024 + } + switch memStatName { + case "VmPeak:": + ms.vmPeak = value + case "VmHWM:": + ms.rssPeak = value + case "RssAnon:": + ms.rssAnon = value + case "RssFile:": + ms.rssFile = value + case "RssShmem:": + ms.rssShmem = value + } + } + return &ms, nil +} // getRSSStats returns RSS bytes for page cache and anonymous memory. func getRSSStats() (uint64, uint64, error) { filepath := "/proc/self/smaps" diff --git a/process_metrics_linux_test.go b/process_metrics_linux_test.go index f32bbe1..ce5f49e 100644 --- a/process_metrics_linux_test.go +++ b/process_metrics_linux_test.go @@ -249,3 +249,18 @@ func TestGetOpenFDsCount(t *testing.T) { f(0, "testdata/fd/0", true) f(0, "testdata/limits", true) } + +func TestGetMemStats(t *testing.T) { + f := func(want memStats, path string, wantErr bool) { + t.Helper() + got, err := getMemStats(path) + if (err != nil && !wantErr) || (err == nil && wantErr) { + t.Fatalf("unexpected error: %v", err) + } + if got != nil && *got != want { + t.Fatalf("unexpected result: %d, want: %d at getMemStats", *got, want) + } + } + f(memStats{vmPeak: 2130489344, rssPeak: 200679424, rssAnon: 121602048, rssFile: 11362304}, "testdata/status", false) + f(memStats{}, "testdata/status_bad", true) +} diff --git a/testdata/status b/testdata/status new file mode 100644 index 0000000..d67a15e --- /dev/null +++ b/testdata/status @@ -0,0 +1,115 @@ +Name: victoria-metric +Umask: 0022 +State: S (sleeping) +Tgid: 1 +Ngid: 0 +Pid: 1 +PPid: 0 +TracerPid: 0 +Uid: 0 0 0 0 +Gid: 0 0 0 0 +FDSize: 256 +Groups: 1 2 3 4 6 10 11 20 26 27 +NStgid: 1 +NSpid: 1 +NSpgid: 1 +NSsid: 1 +VmPeak: 2080548 kB +VmSize: 2080464 kB +VmLck: 0 kB +VmPin: 0 kB +VmHWM: 195976 kB +VmRSS: 105212 kB +RssAnon: 94092 kB +RssFile: 11120 kB +RssShmem: 0 kB +VmData: 632076 kB +VmStk: 132 kB +VmExe: 7004 kB +VmLib: 8 kB +VmPTE: 940 kB +VmSwap: 0 kB +HugetlbPages: 0 kB +CoreDumping: 0 +THP_enabled: 1 +Threads: 14 +SigQ: 1/127458 +SigPnd: 0000000000000000 +ShdPnd: 0000000000000000 +SigBlk: fffffffc3bfa3a00 +SigIgn: 0000000000000000 +SigCgt: fffffffdffc1feff +CapInh: 00000000a80425fb +CapPrm: 00000000a80425fb +CapEff: 00000000a80425fb +CapBnd: 00000000a80425fb +CapAmb: 0000000000000000 +NoNewPrivs: 0 +Seccomp: 0 +Speculation_Store_Bypass: thread vulnerable +Cpus_allowed: ff +Cpus_allowed_list: 0-7 +Mems_allowed: 00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000001 +Mems_allowed_list: 0 +voluntary_ctxt_switches: 82 +nonvoluntary_ctxt_switches: 21 +/ # cat /proc/1/stat +stat statm status +/ # cat /proc/1/statm +520122 27057 2780 1751 0 158052 0 +/ # cat /proc/1/status +Name: victoria-metric +Umask: 0022 +State: S (sleeping) +Tgid: 1 +Ngid: 0 +Pid: 1 +PPid: 0 +TracerPid: 0 +Uid: 0 0 0 0 +Gid: 0 0 0 0 +FDSize: 256 +Groups: 1 2 3 4 6 10 11 20 26 27 +NStgid: 1 +NSpid: 1 +NSpgid: 1 +NSsid: 1 +VmPeak: 2080556 kB +VmSize: 2080520 kB +VmLck: 0 kB +VmPin: 0 kB +VmHWM: 195976 kB +VmRSS: 129848 kB +RssAnon: 118752 kB +RssFile: 11096 kB +RssShmem: 0 kB +VmData: 633020 kB +VmStk: 132 kB +VmExe: 7004 kB +VmLib: 8 kB +VmPTE: 984 kB +VmSwap: 0 kB +HugetlbPages: 0 kB +CoreDumping: 0 +THP_enabled: 1 +Threads: 14 +SigQ: 1/127458 +SigPnd: 0000000000000000 +ShdPnd: 0000000000000000 +SigBlk: fffffffc3bfa3a00 +SigIgn: 0000000000000000 +SigCgt: fffffffdffc1feff +CapInh: 00000000a80425fb +CapPrm: 00000000a80425fb +CapEff: 00000000a80425fb +CapBnd: 00000000a80425fb +CapAmb: 0000000000000000 +NoNewPrivs: 0 +Seccomp: 0 +Speculation_Store_Bypass: thread vulnerable +Cpus_allowed: ff +Cpus_allowed_list: 0-7 +Mems_allowed: 00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000001 +Mems_allowed_list: 0 +voluntary_ctxt_switches: 82 +nonvoluntary_ctxt_switches: 21 \ No newline at end of file diff --git a/testdata/status_bad b/testdata/status_bad new file mode 100644 index 0000000..24cf8cb --- /dev/null +++ b/testdata/status_bad @@ -0,0 +1,115 @@ +Name: victoria-metric +Umask: 0022 +State: S (sleeping) +Tgid: 1 +Ngid: 0 +Pid: 1 +PPid: 0 +TracerPid: 0 +Uid: 0 0 0 0 +Gid: 0 0 0 0 +FDSize: 256 +Groups: 1 2 3 4 6 10 11 20 26 27 +NStgid: 1 +NSpid: 1 +NSpgid: 1 +NSsid: 1 +VmPeak: 2080548 kB +VmSize: 2080464 kB +VmLck: 0 kB +VmPin: 0 kB +VmHWM: 195976 kB +VmRSS: 105212 kB +RssAnon: 94092 kB +RssFile: 11120 kB +RssShmem: 0 kB +VmData: 632076 kB +VmStk: 132 kB +VmExe: 7004 kB +VmLib: 8 kB +VmPTE: 940 kB +VmSwap: 0 kB +HugetlbPages: 0 kB +CoreDumping: 0 +THP_enabled: 1 +Threads: 14 +SigQ: 1/127458 +SigPnd: 0000000000000000 +ShdPnd: 0000000000000000 +SigBlk: fffffffc3bfa3a00 +SigIgn: 0000000000000000 +SigCgt: fffffffdffc1feff +CapInh: 00000000a80425fb +CapPrm: 00000000a80425fb +CapEff: 00000000a80425fb +CapBnd: 00000000a80425fb +CapAmb: 0000000000000000 +NoNewPrivs: 0 +Seccomp: 0 +Speculation_Store_Bypass: thread vulnerable +Cpus_allowed: ff +Cpus_allowed_list: 0-7 +Mems_allowed: 00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000001 +Mems_allowed_list: 0 +voluntary_ctxt_switches: 82 +nonvoluntary_ctxt_switches: 21 +/ # cat /proc/1/stat +stat statm status +/ # cat /proc/1/statm +520122 27057 2780 1751 0 158052 0 +/ # cat /proc/1/status +Name: victoria-metric +Umask: 0022 +State: S (sleeping) +Tgid: 1 +Ngid: 0 +Pid: 1 +PPid: 0 +TracerPid: 0 +Uid: 0 0 0 0 +Gid: 0 0 0 0 +FDSize: 256 +Groups: 1 2 3 4 6 10 11 20 26 27 +NStgid: 1 +NSpid: 1 +NSpgid: 1 +NSsid: 1 +VmPeak: 2080556 kB +VmSize: 2080520 kB as +VmLck: 0 kB +VmPin: 0 kB +VmHWM: 195976 kB +VmRSS: 129848 kB +RssAnon: 118752 kB +RssFile: 11096 kB +RssShmem: 0 kB +VmData: 633020 kB +VmStk: 132 kB +VmExe: 7004 kB +VmLib: 8 kB +VmPTE: 984 kB +VmSwap: 0 kB +HugetlbPages: 0 kB +CoreDumping: 0 +THP_enabled: 1 +Threads: 14 +SigQ: 1/127458 +SigPnd: 0000000000000000 +ShdPnd: 0000000000000000 +SigBlk: fffffffc3bfa3a00 fsa +SigIgn: 0000000000000000 +SigCgt: fffffffdffc1feff +CapInh: 00000000a80425fb +CapPrm: 00000000a80425fb +CapEff: 00000000a80425fb +CapBnd: 00000000a80425fb +CapAmb: 0000000000000000 +NoNewPrivs: 0 +Seccomp: 0 +Speculation_Store_Bypass: thread vulnerable +Cpus_allowed: ff +Cpus_allowed_list: 0-7 +Mems_allowed: 00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000001 +Mems_allowed_list: 0 +voluntary_ctxt_switches: 82 +nonvoluntary_ctxt_switches: 21 \ No newline at end of file