From 3af71ce2688435f8828b37257c0790f6300202b1 Mon Sep 17 00:00:00 2001 From: Florian Pritz Date: Sat, 8 May 2010 20:42:44 +0200 Subject: upstream update Signed-off-by: Florian Pritz --- ps_mem.py | 121 ++++++++++++++++++++++++++++++++++++++++++++++++-------------- 1 file changed, 94 insertions(+), 27 deletions(-) diff --git a/ps_mem.py b/ps_mem.py index deae67f..88fa82a 100755 --- a/ps_mem.py +++ b/ps_mem.py @@ -35,6 +35,18 @@ # Patch from patrice.bouchand.fedora@gmail.com # V1.9 20 Feb 2008 Fix invalid values reported when PSS is available. # Reported by Andrey Borzenkov +# V2.0 15 Jan 2010 From a report by Brock Noland +# about overreporting of RAM usage of his java progs, +# handle linux clones that have pids. I.E. that have +# CLONE_VM specified without CLONE_THREAD. +# V2.1 20 Jan 2010 Append [deleted] or [updated] to programs which are +# no longer on disk or have a new version available. +# Add a --split-args option to group programs based +# on the full command line, which could be used +# to monitor separate "pmon" processes for example: +# ps_mem.py | grep [p]mon +# V2.2 16 Feb 2010 Support python 3. +# Patch from Brian Harring # Notes: # @@ -42,7 +54,7 @@ # by the shell or with env, will be merged to the interpreter # (as that's what's given to exec). For e.g. all python programs # starting with "#!/usr/bin/env python" will be grouped under python. -# You can change this by changing comm= to args= below but that will +# You can change this by using the full command line but that will # have the undesirable affect of splitting up programs started with # differing parameters (for e.g. mingetty tty[1-6]). # @@ -57,22 +69,40 @@ # Since kernel 2.6.23-rc8-mm1 PSS is available in smaps, which allows # us to calculate a more accurate value for the total RAM used by programs. # +# Programs that use CLONE_VM without CLONE_THREAD are discounted by assuming +# they're the only programs that have the same /proc/$PID/smaps file for +# each instance. This will fail if there are multiple real instances of a +# program that then use CLONE_VM without CLONE_THREAD, or if a clone changes +# its memory map while we're checksumming each /proc/$PID/smaps. +# # I don't take account of memory allocated for a program # by other programs. For e.g. memory used in the X server for # a program could be determined, but is not. import sys, os, string +try: + # md5 module is deprecated on python 2.6 + # so try the newer hashlib first + import hashlib + md5_new = hashlib.md5 +except ImportError: + import md5 + md5_new = md5.new if os.geteuid() != 0: sys.stderr.write("Sorry, root permission required.\n"); sys.exit(1) +split_args=False +if len(sys.argv)==2 and sys.argv[1] == "--split-args": + split_args = True + PAGESIZE=os.sysconf("SC_PAGE_SIZE")/1024 #KiB our_pid=os.getpid() #(major,minor,release) def kernel_ver(): - kv=open("/proc/sys/kernel/osrelease").readline().split(".")[:3] + kv=open("/proc/sys/kernel/osrelease", "rt").readline().split(".")[:3] for char in "-_": kv[2]=kv[2].split(char)[0] return (int(kv[0]), int(kv[1]), int(kv[2])) @@ -85,12 +115,18 @@ have_pss=0 #Note shared is always a subset of rss (trs is not always) def getMemStats(pid): global have_pss + mem_id = pid #unique Private_lines=[] Shared_lines=[] Pss_lines=[] - Rss=int(open("/proc/"+str(pid)+"/statm").readline().split()[1])*PAGESIZE + Rss=int(open("/proc/"+str(pid)+"/statm", "rt").readline().split()[1])*PAGESIZE if os.path.exists("/proc/"+str(pid)+"/smaps"): #stat - for line in open("/proc/"+str(pid)+"/smaps").readlines(): #open + digester = md5_new() + for line in open("/proc/"+str(pid)+"/smaps", "rb").readlines(): #open + # Note we checksum smaps as maps is usually but + # not always different for separate processes. + digester.update(line) + line = line.decode("ascii") if line.startswith("Shared"): Shared_lines.append(line) elif line.startswith("Private"): @@ -98,6 +134,7 @@ def getMemStats(pid): elif line.startswith("Pss"): have_pss=1 Pss_lines.append(line) + mem_id = digester.hexdigest() Shared=sum([int(line.split()[1]) for line in Shared_lines]) Private=sum([int(line.split()[1]) for line in Private_lines]) #Note Shared + Private = Rss above @@ -110,14 +147,32 @@ def getMemStats(pid): Shared=0 #lots of overestimation, but what can we do? Private = Rss else: - Shared=int(open("/proc/"+str(pid)+"/statm").readline().split()[2]) + Shared=int(open("/proc/"+str(pid)+"/statm", "rt").readline().split()[2]) Shared*=PAGESIZE Private = Rss - Shared - return (Private, Shared) + return (Private, Shared, mem_id) def getCmdName(pid): - cmd = file("/proc/%d/status" % pid).readline()[6:-1] - exe = os.path.basename(os.path.realpath("/proc/%d/exe" % pid)) + cmdline = open("/proc/%d/cmdline" % pid, "rt").read().split("\0") + if cmdline[-1] == '' and len(cmdline) > 1: + cmdline = cmdline[:-1] + path = os.path.realpath("/proc/%d/exe" % pid) #exception for kernel threads + if split_args: + return " ".join(cmdline) + if path.endswith(" (deleted)"): + path = path[:-10] + if os.path.exists(path): + path += " [updated]" + else: + #The path could be have prelink stuff so try cmdline + #which might have the full path present. This helped for: + #/usr/libexec/notification-area-applet.#prelink#.fX7LCT (deleted) + if os.path.exists(cmdline[0]): + path = cmdline[0] + " [updated]" + else: + path += " [deleted]" + exe = os.path.basename(path) + cmd = open("/proc/%d/status" % pid, "rt").readline()[6:-1] if exe.startswith(cmd): cmd=exe #show non truncated version #Note because we show the non truncated name @@ -128,12 +183,13 @@ def getCmdName(pid): cmds={} shareds={} +mem_ids={} count={} for pid in os.listdir("/proc/"): - try: - pid = int(pid) #note Thread IDs not listed in /proc/ which is good - if pid == our_pid: continue - except: + if not pid.isdigit(): + continue + pid = int(pid) + if pid == our_pid: continue try: cmd = getCmdName(pid) @@ -143,7 +199,7 @@ for pid in os.listdir("/proc/"): #process gone continue try: - private, shared = getMemStats(pid) + private, shared, mem_id = getMemStats(pid) except: continue #process gone if shareds.get(cmd): @@ -154,20 +210,32 @@ for pid in os.listdir("/proc/"): else: shareds[cmd]=shared cmds[cmd]=cmds.setdefault(cmd,0)+private - if count.has_key(cmd): + if cmd in count: count[cmd] += 1 else: count[cmd] = 1 + mem_ids.setdefault(cmd,{}).update({mem_id:None}) #Add shared mem for each program total=0 -for cmd in cmds.keys(): +for cmd in cmds: + cmd_count = count[cmd] + if len(mem_ids[cmd]) == 1 and cmd_count > 1: + # Assume this program is using CLONE_VM without CLONE_THREAD + # so only account for one of the processes + cmds[cmd] /= cmd_count + if have_pss: + shareds[cmd] /= cmd_count cmds[cmd]=cmds[cmd]+shareds[cmd] total+=cmds[cmd] #valid if PSS available -sort_list = cmds.items() -sort_list.sort(lambda x,y:cmp(x[1],y[1])) -sort_list=filter(lambda x:x[1],sort_list) #get rid of zero sized processes +if sys.version_info >= (2, 6): + sort_list = sorted(cmds.items(), key=lambda x:x[1]) +else: + sort_list = cmds.items() + sort_list.sort(lambda x,y:cmp(x[1],y[1])) +# list wrapping is redundant on =pyk3 however +sort_list=list(filter(lambda x:x[1],sort_list)) #get rid of zero sized processes #The following matches "du -h" output #see also human.py @@ -184,16 +252,15 @@ def cmd_with_count(cmd, count): else: return cmd -print " Private + Shared = RAM used\tProgram \n" +sys.stdout.write(" Private + Shared = RAM used\tProgram \n\n") for cmd in sort_list: - print "%8sB + %8sB = %8sB\t%s" % (human(cmd[1]-shareds[cmd[0]]), + sys.stdout.write("%8sB + %8sB = %8sB\t%s\n" % (human(cmd[1]-shareds[cmd[0]]), human(shareds[cmd[0]]), human(cmd[1]), - cmd_with_count(cmd[0], count[cmd[0]])) + cmd_with_count(cmd[0], count[cmd[0]]))) if have_pss: - print "-" * 33 - print " " * 24 + "%8sB" % human(total) - print "=" * 33 -print "\n Private + Shared = RAM used\tProgram \n" + sys.stdout.write("%s\n%s%8sB\n%s\n" % ("-" * 33, + " " * 24, human(total), "=" * 33)) +sys.stdout.write("\n Private + Shared = RAM used\tProgram \n\n") #Warn of possible inaccuracies #2 = accurate & can total @@ -203,12 +270,12 @@ print "\n Private + Shared = RAM used\tProgram \n" def shared_val_accuracy(): """http://wiki.apache.org/spamassassin/TopSharedMemoryBug""" if kv[:2] == (2,4): - if open("/proc/meminfo").read().find("Inact_") == -1: + if open("/proc/meminfo", "rt").read().find("Inact_") == -1: return 1 return 0 elif kv[:2] == (2,6): if os.path.exists("/proc/"+str(os.getpid())+"/smaps"): - if open("/proc/"+str(os.getpid())+"/smaps").read().find("Pss:")!=-1: + if open("/proc/"+str(os.getpid())+"/smaps", "rt").read().find("Pss:")!=-1: return 2 else: return 1 -- cgit v1.2.3-24-g4f1b