From 59d3baaf18946272d031d62a2b4da09a7a6ddade Mon Sep 17 00:00:00 2001 From: "dmytro.yurchenko" Date: Fri, 29 Apr 2022 18:00:55 +0200 Subject: [PATCH] Export last CPU id used by a process. The use case is to check that process on host system respects cpusets (https://www.kernel.org/doc/html/latest/admin-guide/cgroup-v1/cpusets.html). For example, when running benchmarks, it is imperative for repeatability of measurements that benchmarked process is isolated and none other process uses same CPU, or even better, CPU socket. -- **Linux:** It can be done by analyzing /proc//stat values. Field 39 is last processor used as per https://man7.org/linux/man-pages/man5/proc.5.html. -- **Darwin:** I use macOS Monterey 12.3.1, MacBook Pro 2020, Intel(R) Core(TM) i7-1068NG7 CPU @ 2.30GHz. I made a research in attempt to find a reliable way to identify last CPU id used by a process. However, it seems that MacOS doesn't provide necessary knobs by design. (1) As per https://developer.apple.com/library/archive/releasenotes/Performance/RN-AffinityAPI/ > OS X does not export interfaces that identify processors or control thread placement Neither top and ps utilities report last CPU id for processes. However it is possible to specify CPU affinity hints for a thread as describe in API above. (2) I found that in htop it is possible to enable PROCESSOR field, but it always displays 0. I analyzed source code of htop (https://github.com/htop-dev/htop/tree/main/darwin) and found that instead of spawning `ps` processes like gopsutil does, it relies on MacOS system calls, which is **much faster** and can be potential improvement for gopsutil (works like in https://blog.guillaume-gomez.fr/articles/2021-09-06+sysinfo%3A+how+to+extract+systems%27+information). I checked results from syscalls in detail, and found that they don't include information about last CPU id either: https://github.com/apple/darwin-xnu/blob/main/bsd/sys/proc.h#L102 https://github.com/apple/darwin-xnu/blob/main/bsd/sys/sysctl.h#L975 https://opensource.apple.com/source/adv_cmds/adv_cmds-158/ps/ps.c (3) The next thing I found that someone created DTrace script to report last CPU id per thread: https://github.com/elazarl/cpu_affinity However, it doesn't work any longer, because dtrace that is included in MacOS doesn't support `sched:::on-cpu` and `sched::off-cpu` (see https://docs.oracle.com/cd/E19253-01/817-6223/chp-sched-5/index.html for info on probes). When I run `sudo dtrace -l`, it has plenty of probes, but not the above mentioned. Dtrace in MacOS is quite limited, and apparently situation since http://dtrace.org/blogs/ahl/2008/01/18/mac-os-x-and-the-missing-probes/ didn't improve. (4) I found that you can call `sudo cpuwalk.d 1` (DTrace utility shipped with MacOS - source code is available on https://opensource.apple.com/source/dtrace/dtrace-147.20.2/DTTk/Cpu/cpuwalk.d.auto.html) to get CPU assignments for each process. It works, but integrating it into gopsutil will require quite some hacking, which I simply don't have time to do now. -- **Windows:** Same story as with Darwin. None of default tools display which CPU was used for running the process. It is possible to specify CPU affinity hints for a thread via `SetThreadIdealProcessorEx()` call (https://docs.microsoft.com/en-us/windows/win32/api/processthreadsapi/nf-processthreadsapi-setthreadidealprocessorex). There is also API call GetCurrentProcessorNumber() that returns the processor for the current thread, but it is callable only from inside the thread itself: https://docs.microsoft.com/en-gb/windows/win32/api/processthreadsapi/nf-processthreadsapi-getcurrentprocessornumber?redirectedfrom=MSDN I tried Microsoft pstools (https://docs.microsoft.com/en-gb/sysinternals/downloads/pstools), but though they provide a lot of interesting information, they don't report CPU id either. According to https://superuser.com/questions/867127/determine-which-cpu-a-process-is-running-on?lq=1, the requested feature is not easily available on Windows. However there is also mention that it is possible to get used CPU ids from xperf traces. I am not sure though if the overhead is worth it. --- process/process.go | 1 + process/process_linux.go | 32 +++++++++++++++++++------------- 2 files changed, 20 insertions(+), 13 deletions(-) diff --git a/process/process.go b/process/process.go index cc88ce7d4c..b87ac0bb88 100644 --- a/process/process.go +++ b/process/process.go @@ -43,6 +43,7 @@ type FilledProcess struct { Nice int32 CreateTime int64 OpenFdCount int32 + LastCpu int32 // status Name string diff --git a/process/process_linux.go b/process/process_linux.go index 330288686f..a4d11de973 100644 --- a/process/process_linux.go +++ b/process/process_linux.go @@ -226,7 +226,7 @@ func NewProcess(pid int32) (*Process, error) { // Ppid returns Parent Process ID of the process. func (p *Process) Ppid() (int32, error) { - ppid, _, _, _, _, err := p.fillFromStat() + ppid, _, _, _, _, _, err := p.fillFromStat() if err != nil { return -1, err } @@ -262,7 +262,7 @@ func (p *Process) CmdlineSlice() ([]string, error) { // CreateTime returns created time of the process in seconds since the epoch, in UTC. func (p *Process) CreateTime() (int64, error) { - _, _, _, createTime, _, err := p.fillFromStat() + _, _, _, createTime, _, _, err := p.fillFromStat() if err != nil { return 0, err } @@ -329,7 +329,7 @@ func (p *Process) Terminal() (string, error) { // Nice returns a nice value (priority). // Notice: gopsutil can not set nice value. func (p *Process) Nice() (int32, error) { - _, _, _, _, nice, err := p.fillFromStat() + _, _, _, _, nice, _, err := p.fillFromStat() if err != nil { return 0, err } @@ -385,7 +385,7 @@ func (p *Process) Threads() (map[string]string, error) { // Times returns CPU times of the process. func (p *Process) Times() (*cpu.TimesStat, error) { - _, _, cpuTimes, _, _, err := p.fillFromStat() + _, _, cpuTimes, _, _, _, err := p.fillFromStat() if err != nil { return nil, err } @@ -859,12 +859,12 @@ func (p *Process) fillTermFromStat() (string, error) { return terminal, err } -func (p *Process) fillFromStat() (int32, int32, *cpu.TimesStat, int64, int32, error) { +func (p *Process) fillFromStat() (int32, int32, *cpu.TimesStat, int64, int32, int32, error) { pid := p.Pid statPath := common.HostProc(strconv.Itoa(int(pid)), "stat") contents, err := ioutil.ReadFile(statPath) if err != nil { - return 0, 0, nil, 0, 0, err + return 0, 0, nil, 0, 0, 0, err } fields := strings.Fields(string(contents)) timestamp := time.Now().Unix() @@ -876,20 +876,20 @@ func (p *Process) fillFromStat() (int32, int32, *cpu.TimesStat, int64, int32, er ppid, err := strconv.ParseInt(fields[i+2], 10, 32) if err != nil { - return 0, 0, nil, 0, 0, err + return 0, 0, nil, 0, 0, 0, err } pgrp, err := strconv.ParseInt(fields[i+3], 10, 32) if err != nil { - return 0, 0, nil, 0, 0, err + return 0, 0, nil, 0, 0, 0, err } utime, err := strconv.ParseFloat(fields[i+12], 64) if err != nil { - return 0, 0, nil, 0, 0, err + return 0, 0, nil, 0, 0, 0, err } stime, err := strconv.ParseFloat(fields[i+13], 64) if err != nil { - return 0, 0, nil, 0, 0, err + return 0, 0, nil, 0, 0, 0, err } cpuTimes := &cpu.TimesStat{ @@ -906,7 +906,7 @@ func (p *Process) fillFromStat() (int32, int32, *cpu.TimesStat, int64, int32, er bootTime := CachedBootTime t, err := strconv.ParseUint(fields[i+20], 10, 64) if err != nil { - return 0, 0, nil, 0, 0, err + return 0, 0, nil, 0, 0, 0, err } ctime := (t / uint64(ClockTicks)) + uint64(bootTime) createTime := int64(ctime * 1000) @@ -916,7 +916,12 @@ func (p *Process) fillFromStat() (int32, int32, *cpu.TimesStat, int64, int32, er snice, _ := syscall.Getpriority(PrioProcess, int(pid)) nice := int32(snice) // FIXME: is this true? - return int32(ppid), int32(pgrp), cpuTimes, createTime, nice, nil + processor, err := strconv.ParseInt(fields[i+37], 10, 32) + if err != nil { + return 0, 0, nil, 0, 0, 0, err + } + + return int32(ppid), int32(pgrp), cpuTimes, createTime, nice, int32(processor), nil } // Pids returns a slice of process ID list which are running now. @@ -985,7 +990,7 @@ func AllProcesses() (map[int32]*FilledProcess, error) { log.Debugf("Unable to access /proc/%d/io: %s", pid, err) ioStat = &IOCountersStat{} } - ppid, _, t1, createTime, nice, err := p.fillFromStat() + ppid, _, t1, createTime, nice, processor, err := p.fillFromStat() if err != nil { log.Debugf("Unable to fill from /proc/%d/stat: %s", pid, err) t1 = &cpu.TimesStat{} @@ -1029,6 +1034,7 @@ func AllProcesses() (map[int32]*FilledProcess, error) { Nice: nice, CreateTime: createTime, OpenFdCount: openFdCount, + LastCpu: processor, // status Name: p.name,