From a267f31e28bf3af4490d62c57ba9b83bfe2829de Mon Sep 17 00:00:00 2001 From: Florian Pritz Date: Fri, 6 Sep 2013 21:45:05 +0200 Subject: update ps_mem.py Signed-off-by: Florian Pritz --- ps_mem.py | 424 +++++++++++++++++++++++++++++++++++++++++--------------------- 1 file changed, 279 insertions(+), 145 deletions(-) (limited to 'ps_mem.py') diff --git a/ps_mem.py b/ps_mem.py index 7692c01..b9165b1 100644 --- a/ps_mem.py +++ b/ps_mem.py @@ -1,4 +1,4 @@ -#!/usr/bin/python2 +#!/usr/bin/env python # Try to determine how much RAM is currently being used per program. # Note per _program_, not per process. So for example this script @@ -7,8 +7,9 @@ # The shared RAM is problematic to calculate, and this script automatically # selects the most accurate method available for your kernel. -# Author: P@draigBrady.com -# Source: http://www.pixelbeat.org/scripts/ps_mem.py +# Licence: LGPLv2 +# Author: P@draigBrady.com +# Source: http://www.pixelbeat.org/scripts/ps_mem.py # V1.0 06 Jul 2005 Initial release # V1.1 11 Aug 2006 root permission required for accuracy @@ -35,7 +36,7 @@ # Patch from patrice.bouchand.fedora@gmail.com # V1.9 20 Feb 2008 Fix invalid values reported when PSS is available. # Reported by Andrey Borzenkov -# V2.4 06 Mar 2011 +# V3.1 10 May 2013 # http://github.com/pixelb/scripts/commits/master/scripts/ps_mem.py # Notes: @@ -72,7 +73,12 @@ # FreeBSD is supported if linprocfs is mounted at /compat/linux/proc/ # FreeBSD 8.0 supports up to a level of Linux 2.6.16 -import sys, os, errno, string +import getopt +import time +import errno +import os +import sys + try: # md5 module is deprecated on python 2.6 # so try the newer hashlib first @@ -82,103 +88,174 @@ except ImportError: import md5 md5_new = md5.new + # The following exits cleanly on Ctrl-C or EPIPE # while treating other exceptions as before. def std_exceptions(etype, value, tb): - sys.excepthook=sys.__excepthook__ + sys.excepthook = sys.__excepthook__ if issubclass(etype, KeyboardInterrupt): pass elif issubclass(etype, IOError) and value.errno == errno.EPIPE: pass else: sys.__excepthook__(etype, value, tb) -sys.excepthook=std_exceptions +sys.excepthook = std_exceptions + +# +# Define some global variables +# + +PAGESIZE = os.sysconf("SC_PAGE_SIZE") / 1024 #KiB +our_pid = os.getpid() + +have_pss = 0 + +class Proc: + def __init__(self): + uname = os.uname() + if uname[0] == "FreeBSD": + self.proc = '/compat/linux/proc' + else: + self.proc = '/proc' + + def path(self, *args): + return os.path.join(self.proc, *(str(a) for a in args)) + + def open(self, *args): + try: + return open(self.path(*args)) + except (IOError, OSError): + val = sys.exc_info()[1] + if (val.errno == errno.ENOENT or # kernel thread or process gone + val.errno == errno.EPERM): + raise LookupError -if os.geteuid() != 0: - sys.stderr.write("Sorry, root permission required.\n"); - if __name__ == '__main__': - sys.stderr.close() - sys.exit(1) +proc = Proc() -uname=os.uname() -if uname[0]=="FreeBSD": - proc="/compat/linux/proc/" -else: - proc="/proc/" -split_args=False -if len(sys.argv)==2 and sys.argv[1] == "--split-args": - split_args = True +# +# Functions +# + +def parse_options(): + try: + long_options = ['split-args', 'help'] + opts, args = getopt.getopt(sys.argv[1:], "shp:w:", long_options) + except getopt.GetoptError: + sys.stderr.write(help()) + sys.exit(3) + + # ps_mem.py options + split_args = False + pids_to_show = None + watch = None + + for o, a in opts: + if o in ('-s', '--split-args'): + split_args = True + if o in ('-h', '--help'): + sys.stdout.write(help()) + sys.exit(0) + if o in ('-p',): + try: + pids_to_show = [int(x) for x in a.split(',')] + except: + sys.stderr.write(help()) + sys.exit(3) + if o in ('-w',): + try: + watch = int(a) + except: + sys.stderr.write(help()) + sys.exit(3) -PAGESIZE=os.sysconf("SC_PAGE_SIZE")/1024 #KiB -our_pid=os.getpid() + return (split_args, pids_to_show, watch) + +def help(): + help_msg = 'ps_mem.py - Show process memory usage\n'\ + '\n'\ + '-h Show this help\n'\ + '-w Measure and show process memory every N seconds\n'\ + '-p [,pid2,...pidN] Only show memory usage PIDs in the specified list\n' \ + '-s, --split-args Show and separate by, all command line arguments\n' + + return help_msg #(major,minor,release) def kernel_ver(): - kv=open(proc+"sys/kernel/osrelease", "rt").readline().split(".")[:2] + kv = proc.open('sys/kernel/osrelease').readline().split(".")[:3] + last = len(kv) + if last == 2: + kv.append('0') + last -= 1 for char in "-_": - kv[1]=kv[1].split(char)[0] - return (int(kv[0]), int(kv[1])) - -try: - kv=kernel_ver() -except (IOError, OSError), value: - if value.errno == errno.ENOENT: - sys.stderr.write( - "Couldn't access /proc\n" - "Only GNU/Linux and FreeBSD (with linprocfs) are supported\n") - sys.exit(2) - else: - raise + kv[last] = kv[last].split(char)[0] + try: + int(kv[last]) + except: + kv[last] = 0 + return (int(kv[0]), int(kv[1]), int(kv[2])) -have_pss=0 #return Private,Shared #Note shared is always a subset of rss (trs is not always) def getMemStats(pid): global have_pss mem_id = pid #unique - Private_lines=[] - Shared_lines=[] - Pss_lines=[] - Rss=int(open(proc+str(pid)+"/statm", "rt").readline().split()[1])*PAGESIZE - if os.path.exists(proc+str(pid)+"/smaps"): #stat + Private_lines = [] + Shared_lines = [] + Pss_lines = [] + Rss = (int(proc.open(pid, 'statm').readline().split()[1]) + * PAGESIZE) + if os.path.exists(proc.path(pid, 'smaps')): #stat digester = md5_new() - for line in open(proc+str(pid)+"/smaps", "rb").readlines(): #open + for line in proc.open(pid, 'smaps').readlines(): #open # Note we checksum smaps as maps is usually but # not always different for separate processes. - digester.update(line) - line = line.decode("ascii") + digester.update(line.encode('latin1')) if line.startswith("Shared"): Shared_lines.append(line) elif line.startswith("Private"): Private_lines.append(line) elif line.startswith("Pss"): - have_pss=1 + have_pss = 1 Pss_lines.append(line) mem_id = digester.hexdigest() - Shared=sum([int(line.split()[1]) for line in Shared_lines]) - Private=sum([int(line.split()[1]) for line in Private_lines]) + Shared = sum([int(line.split()[1]) for line in Shared_lines]) + Private = sum([int(line.split()[1]) for line in Private_lines]) #Note Shared + Private = Rss above #The Rss in smaps includes video card mem etc. if have_pss: - pss_adjust=0.5 #add 0.5KiB as this average error due to trunctation - Pss=sum([float(line.split()[1])+pss_adjust for line in Pss_lines]) + pss_adjust = 0.5 # add 0.5KiB as this avg error due to trunctation + Pss = sum([float(line.split()[1])+pss_adjust for line in Pss_lines]) Shared = Pss - Private - elif (2,6,1) <= kv <= (2,6,9): - Shared=0 #lots of overestimation, but what can we do? + elif (2,6,1) <= kernel_ver() <= (2,6,9): + Shared = 0 #lots of overestimation, but what can we do? Private = Rss else: - Shared=int(open(proc+str(pid)+"/statm", "rt").readline().split()[2]) - Shared*=PAGESIZE + Shared = int(proc.open(pid, 'statm').readline().split()[2]) + Shared *= PAGESIZE Private = Rss - Shared return (Private, Shared, mem_id) -def getCmdName(pid): - cmdline = open(proc+"%d/cmdline" % pid, "rt").read().split("\0") + +def getCmdName(pid, split_args): + cmdline = proc.open(pid, 'cmdline').read().split("\0") if cmdline[-1] == '' and len(cmdline) > 1: cmdline = cmdline[:-1] - path = os.path.realpath(proc+"%d/exe" % pid) #exception for kernel threads + + path = proc.path(pid, 'exe') + try: + path = os.readlink(path) + # Some symlink targets were seen to contain NULs on RHEL 5 at least + # https://github.com/pixelb/scripts/pull/10, so take string up to NUL + path = path.split('\0')[0] + except OSError: + val = sys.exc_info()[1] + if (val.errno == errno.ENOENT or # either kernel thread or process gone + val.errno == errno.EPERM): + raise LookupError + if split_args: return " ".join(cmdline) if path.endswith(" (deleted)"): @@ -194,101 +271,31 @@ def getCmdName(pid): else: path += " [deleted]" exe = os.path.basename(path) - cmd = open(proc+"%d/status" % pid, "rt").readline()[6:-1] + cmd = proc.open(pid, 'status').readline()[6:-1] if exe.startswith(cmd): - cmd=exe #show non truncated version + cmd = exe #show non truncated version #Note because we show the non truncated name #one can have separated programs as follows: #584.0 KiB + 1.0 MiB = 1.6 MiB mozilla-thunder (exe -> bash) # 56.0 MiB + 22.2 MiB = 78.2 MiB mozilla-thunderbird-bin return cmd -cmds={} -shareds={} -mem_ids={} -count={} -for pid in os.listdir(proc): - if not pid.isdigit(): - continue - pid = int(pid) - if pid == our_pid: - continue - try: - cmd = getCmdName(pid) - except: - #permission denied or - #kernel threads don't have exe links or - #process gone - continue - try: - private, shared, mem_id = getMemStats(pid) - except: - continue #process gone - if shareds.get(cmd): - if have_pss: #add shared portion of PSS together - shareds[cmd]+=shared - elif shareds[cmd] < shared: #just take largest shared val - shareds[cmd]=shared - else: - shareds[cmd]=shared - cmds[cmd]=cmds.setdefault(cmd,0)+private - if cmd in count: - count[cmd] += 1 - else: - count[cmd] = 1 - mem_ids.setdefault(cmd,{}).update({mem_id:None}) - -#Add shared mem for each program -total=0 -for cmd in cmds: - cmd_count = count[cmd] - if len(mem_ids[cmd]) == 1 and cmd_count > 1: - # Assume this program is using CLONE_VM without CLONE_THREAD - # so only account for one of the processes - cmds[cmd] /= cmd_count - if have_pss: - shareds[cmd] /= cmd_count - cmds[cmd]=cmds[cmd]+shareds[cmd] - total+=cmds[cmd] #valid if PSS available - -if sys.version_info >= (2, 6): - sort_list = sorted(cmds.items(), key=lambda x:x[1]) -else: - sort_list = cmds.items() - sort_list.sort(lambda x,y:cmp(x[1],y[1])) -# list wrapping is redundant on =pyk3 however -sort_list=list(filter(lambda x:x[1],sort_list)) #get rid of zero sized processes #The following matches "du -h" output #see also human.py def human(num, power="Ki"): - powers=["Ki","Mi","Gi","Ti"] + powers = ["Ki", "Mi", "Gi", "Ti"] while num >= 1000: #4 digits num /= 1024.0 - power=powers[powers.index(power)+1] - return "%.1f %s" % (num,power) + power = powers[powers.index(power)+1] + return "%.1f %s" % (num, power) + def cmd_with_count(cmd, count): - if count>1: - return "%s (%u)" % (cmd, count) + if count > 1: + return "%s (%u)" % (cmd, count) else: - return cmd - -if __name__ == '__main__': - sys.stdout.write(" Private + Shared = RAM used\tProgram \n\n") - for cmd in sort_list: - sys.stdout.write("%8sB + %8sB = %8sB\t%s\n" % - (human(cmd[1]-shareds[cmd[0]]), - human(shareds[cmd[0]]), human(cmd[1]), - cmd_with_count(cmd[0], count[cmd[0]]))) - if have_pss: - sys.stdout.write("%s\n%s%8sB\n%s\n" % - ("-" * 33, " " * 24, human(total), "=" * 33)) - sys.stdout.write("\n Private + Shared = RAM used\tProgram \n\n") - # We must close explicitly, so that any EPIPE exception - # is handled by our excepthook, rather than the default - # one which is reenabled after this script finishes. - sys.stdout.close() + return cmd #Warn of possible inaccuracies #2 = accurate & can total @@ -297,42 +304,169 @@ if __name__ == '__main__': #-1= all shared mem not reported def shared_val_accuracy(): """http://wiki.apache.org/spamassassin/TopSharedMemoryBug""" + kv = kernel_ver() if kv[:2] == (2,4): - if open(proc+"meminfo", "rt").read().find("Inact_") == -1: + if proc.open('meminfo').read().find("Inact_") == -1: return 1 return 0 elif kv[:2] == (2,6): - pid = str(os.getpid()) - if os.path.exists(proc+pid+"/smaps"): - if open(proc+pid+"/smaps", "rt").read().find("Pss:")!=-1: + pid = os.getpid() + if os.path.exists(proc.path(pid, 'smaps')): + if proc.open(pid, 'smaps').read().find("Pss:")!=-1: return 2 else: return 1 if (2,6,1) <= kv <= (2,6,9): return -1 return 0 + elif kv[0] > 2: + return 2 else: return 1 -if __name__ == '__main__': - vm_accuracy = shared_val_accuracy() - if vm_accuracy == -1: +def show_shared_val_accuracy( possible_inacc ): + if possible_inacc == -1: sys.stderr.write( "Warning: Shared memory is not reported by this system.\n" ) sys.stderr.write( "Values reported will be too large, and totals are not reported\n" ) - elif vm_accuracy == 0: + elif possible_inacc == 0: sys.stderr.write( "Warning: Shared memory is not reported accurately by this system.\n" ) sys.stderr.write( "Values reported could be too large, and totals are not reported\n" ) - elif vm_accuracy == 1: + elif possible_inacc == 1: sys.stderr.write( "Warning: Shared memory is slightly over-estimated by this system\n" "for each program, so totals are not reported.\n" ) sys.stderr.close() + +def get_memory_usage( pids_to_show, split_args, include_self=False, only_self=False ): + cmds = {} + shareds = {} + mem_ids = {} + count = {} + for pid in os.listdir(proc.path('')): + if not pid.isdigit(): + continue + pid = int(pid) + + # Some filters + if only_self and pid != our_pid: + continue + if pid == our_pid and not include_self: + continue + if pids_to_show is not None and pid not in pids_to_show: + continue + + try: + cmd = getCmdName(pid, split_args) + except LookupError: + #permission denied or + #kernel threads don't have exe links or + #process gone + continue + + try: + private, shared, mem_id = getMemStats(pid) + except RuntimeError: + continue #process gone + if shareds.get(cmd): + if have_pss: #add shared portion of PSS together + shareds[cmd] += shared + elif shareds[cmd] < shared: #just take largest shared val + shareds[cmd] = shared + else: + shareds[cmd] = shared + cmds[cmd] = cmds.setdefault(cmd, 0) + private + if cmd in count: + count[cmd] += 1 + else: + count[cmd] = 1 + mem_ids.setdefault(cmd, {}).update({mem_id:None}) + + #Add shared mem for each program + total = 0 + for cmd in cmds: + cmd_count = count[cmd] + if len(mem_ids[cmd]) == 1 and cmd_count > 1: + # Assume this program is using CLONE_VM without CLONE_THREAD + # so only account for one of the processes + cmds[cmd] /= cmd_count + if have_pss: + shareds[cmd] /= cmd_count + cmds[cmd] = cmds[cmd] + shareds[cmd] + total += cmds[cmd] #valid if PSS available + + sorted_cmds = sorted(cmds.items(), key=lambda x:x[1]) + sorted_cmds = [x for x in sorted_cmds if x[1]] + + return sorted_cmds, shareds, count, total + +def print_header(): + sys.stdout.write(" Private + Shared = RAM used\tProgram\n\n") + +def print_memory_usage(sorted_cmds, shareds, count, total): + for cmd in sorted_cmds: + sys.stdout.write("%8sB + %8sB = %8sB\t%s\n" % + (human(cmd[1]-shareds[cmd[0]]), + human(shareds[cmd[0]]), human(cmd[1]), + cmd_with_count(cmd[0], count[cmd[0]]))) + if have_pss: + sys.stdout.write("%s\n%s%8sB\n%s\n" % + ("-" * 33, " " * 24, human(total), "=" * 33)) + +def verify_environment(): + if os.geteuid() != 0: + sys.stderr.write("Sorry, root permission required.\n") + if __name__ == '__main__': + sys.stderr.close() + sys.exit(1) + + try: + kv = kernel_ver() + except (IOError, OSError): + val = sys.exc_info()[1] + if val.errno == errno.ENOENT: + sys.stderr.write( + "Couldn't access " + proc.path('') + "\n" + "Only GNU/Linux and FreeBSD (with linprocfs) are supported\n") + sys.exit(2) + else: + raise + +if __name__ == '__main__': + verify_environment() + split_args, pids_to_show, watch = parse_options() + + print_header() + + if watch is not None: + try: + sorted_cmds = True + while sorted_cmds: + sorted_cmds, shareds, count, total = get_memory_usage( pids_to_show, split_args ) + print_memory_usage(sorted_cmds, shareds, count, total) + time.sleep(watch) + else: + sys.stdout.write('Process does not exist anymore.\n') + except KeyboardInterrupt: + pass + else: + # This is the default behavior + sorted_cmds, shareds, count, total = get_memory_usage( pids_to_show, split_args ) + print_memory_usage(sorted_cmds, shareds, count, total) + + + # We must close explicitly, so that any EPIPE exception + # is handled by our excepthook, rather than the default + # one which is reenabled after this script finishes. + sys.stdout.close() + + vm_accuracy = shared_val_accuracy() + show_shared_val_accuracy( vm_accuracy ) -- cgit v1.2.3-24-g4f1b