2 # Copyright 2013 The Chromium Authors. All rights reserved.
3 # Use of this source code is governed by a BSD-style license that can be
4 # found in the LICENSE file.
6 # A Python library to read and store procfs (/proc) information on Linux.
8 # Each information storage class in this file stores original data as original
9 # as reasonablly possible. Translation is done when requested. It is to make it
10 # always possible to probe the original data.
21 class _NullHandler(logging
.Handler
):
22 def emit(self
, record
):
26 _LOGGER
= logging
.getLogger('procfs')
27 _LOGGER
.addHandler(_NullHandler())
30 class ProcStat(object):
31 """Reads and stores information in /proc/pid/stat."""
32 _PATTERN
= re
.compile(r
'^'
35 '(?P<STATE>[RSDZTW]) '
38 '(?P<SESSION>-?[0-9]+) '
39 '(?P<TTY_NR>-?[0-9]+) '
40 '(?P<TPGID>-?[0-9]+) '
43 '(?P<CMINFIT>[0-9]+) '
45 '(?P<CMAJFIT>[0-9]+) '
50 '(?P<PRIORITY>[0-9]+) '
52 '(?P<NUM_THREADS>[0-9]+) '
53 '(?P<ITREALVALUE>[0-9]+) '
54 '(?P<STARTTIME>[0-9]+) '
58 '(?P<STARTCODE>[0-9]+) '
59 '(?P<ENDCODE>[0-9]+) '
60 '(?P<STARTSTACK>[0-9]+) '
61 '(?P<KSTKESP>[0-9]+) '
62 '(?P<KSTKEIP>[0-9]+) '
64 '(?P<BLOCKED>[0-9]+) '
65 '(?P<SIGIGNORE>[0-9]+) '
66 '(?P<SIGCATCH>[0-9]+) '
70 '(?P<EXIT_SIGNAL>[0-9]+) '
71 '(?P<PROCESSOR>[0-9]+) '
72 '(?P<RT_PRIORITY>[0-9]+) '
74 '(?P<DELAYACCT_BLKIO_TICKS>[0-9]+) '
75 '(?P<GUEST_TIME>[0-9]+) '
76 '(?P<CGUEST_TIME>[0-9]+)', re
.IGNORECASE
)
78 def __init__(self
, raw
, pid
, vsize
, rss
):
85 def load_file(stat_f
):
86 raw
= stat_f
.readlines()
87 stat
= ProcStat
._PATTERN
.match(raw
[0])
89 stat
.groupdict().get('PID'),
90 stat
.groupdict().get('VSIZE'),
91 stat
.groupdict().get('RSS'))
96 with
open(os
.path
.join('/proc', str(pid
), 'stat'), 'r') as stat_f
:
97 return ProcStat
.load_file(stat_f
)
107 return int(self
._pid
)
111 return int(self
._vsize
)
115 return int(self
._rss
)
118 class ProcStatm(object):
119 """Reads and stores information in /proc/pid/statm."""
120 _PATTERN
= re
.compile(r
'^'
122 '(?P<RESIDENT>[0-9]+) '
127 '(?P<DT>[0-9]+)', re
.IGNORECASE
)
129 def __init__(self
, raw
, size
, resident
, share
, text
, lib
, data
, dt
):
132 self
._resident
= resident
140 def load_file(statm_f
):
142 raw
= statm_f
.readlines()
143 except (IOError, OSError):
145 statm
= ProcStatm
._PATTERN
.match(raw
[0])
146 return ProcStatm(raw
,
147 statm
.groupdict().get('SIZE'),
148 statm
.groupdict().get('RESIDENT'),
149 statm
.groupdict().get('SHARE'),
150 statm
.groupdict().get('TEXT'),
151 statm
.groupdict().get('LIB'),
152 statm
.groupdict().get('DATA'),
153 statm
.groupdict().get('DT'))
158 with
open(os
.path
.join('/proc', str(pid
), 'statm'), 'r') as statm_f
:
159 return ProcStatm
.load_file(statm_f
)
160 except (IOError, OSError):
169 return int(self
._size
)
173 return int(self
._resident
)
177 return int(self
._share
)
181 return int(self
._text
)
185 return int(self
._lib
)
189 return int(self
._data
)
196 class ProcStatus(object):
197 """Reads and stores information in /proc/pid/status."""
198 _PATTERN
= re
.compile(r
'^(?P<NAME>[A-Za-z0-9_]+):\s+(?P<VALUE>.*)')
200 def __init__(self
, raw
, dct
):
202 self
._pid
= dct
.get('Pid')
203 self
._name
= dct
.get('Name')
204 self
._vm
_peak
= dct
.get('VmPeak')
205 self
._vm
_size
= dct
.get('VmSize')
206 self
._vm
_lck
= dct
.get('VmLck')
207 self
._vm
_pin
= dct
.get('VmPin')
208 self
._vm
_hwm
= dct
.get('VmHWM')
209 self
._vm
_rss
= dct
.get('VmRSS')
210 self
._vm
_data
= dct
.get('VmData')
211 self
._vm
_stack
= dct
.get('VmStk')
212 self
._vm
_exe
= dct
.get('VmExe')
213 self
._vm
_lib
= dct
.get('VmLib')
214 self
._vm
_pte
= dct
.get('VmPTE')
215 self
._vm
_swap
= dct
.get('VmSwap')
218 def load_file(status_f
):
219 raw
= status_f
.readlines()
222 status_match
= ProcStatus
._PATTERN
.match(line
)
224 match_dict
= status_match
.groupdict()
225 dct
[match_dict
['NAME']] = match_dict
['VALUE']
227 raise SyntaxError('Unknown /proc/pid/status format.')
228 return ProcStatus(raw
, dct
)
232 with
open(os
.path
.join('/proc', str(pid
), 'status'), 'r') as status_f
:
233 return ProcStatus
.load_file(status_f
)
241 return int(self
._pid
)
245 """Returns a high-water (peak) virtual memory size in kilo-bytes."""
246 if self
._vm
_peak
.endswith('kB'):
247 return int(self
._vm
_peak
.split()[0])
248 raise ValueError('VmPeak is not in kB.')
252 """Returns a virtual memory size in kilo-bytes."""
253 if self
._vm
_size
.endswith('kB'):
254 return int(self
._vm
_size
.split()[0])
255 raise ValueError('VmSize is not in kB.')
259 """Returns a high-water (peak) resident set size (RSS) in kilo-bytes."""
260 if self
._vm
_hwm
.endswith('kB'):
261 return int(self
._vm
_hwm
.split()[0])
262 raise ValueError('VmHWM is not in kB.')
266 """Returns a resident set size (RSS) in kilo-bytes."""
267 if self
._vm
_rss
.endswith('kB'):
268 return int(self
._vm
_rss
.split()[0])
269 raise ValueError('VmRSS is not in kB.')
272 class ProcMapsEntry(object):
273 """A class representing one line in /proc/pid/maps."""
276 self
, begin
, end
, readable
, writable
, executable
, private
, offset
,
277 major
, minor
, inode
, name
):
280 self
.readable
= readable
281 self
.writable
= writable
282 self
.executable
= executable
283 self
.private
= private
294 'readable': self
.readable
,
295 'writable': self
.writable
,
296 'executable': self
.executable
,
297 'private': self
.private
,
298 'offset': self
.offset
,
306 class ProcMaps(object):
307 """Reads and stores information in /proc/pid/maps."""
309 MAPS_PATTERN
= re
.compile(
310 r
'^([a-f0-9]+)-([a-f0-9]+)\s+(.)(.)(.)(.)\s+([a-f0-9]+)\s+(\S+):(\S+)\s+'
311 r
'(\d+)\s*(.*)$', re
.IGNORECASE
)
313 EXECUTABLE_PATTERN
= re
.compile(
314 r
'\S+\.(so|dll|dylib|bundle)((\.\d+)+\w*(\.\d+){0,3})?')
317 self
._sorted
_indexes
= []
318 self
._dictionary
= {}
321 def iter(self
, condition
):
323 self
._sorted
_indexes
.sort()
325 for index
in self
._sorted
_indexes
:
326 if not condition
or condition(self
._dictionary
[index
]):
327 yield self
._dictionary
[index
]
331 self
._sorted
_indexes
.sort()
333 for index
in self
._sorted
_indexes
:
334 yield self
._dictionary
[index
]
337 def load_file(maps_f
):
340 table
.append_line(line
)
346 with
open(os
.path
.join('/proc', str(pid
), 'maps'), 'r') as maps_f
:
347 return ProcMaps
.load_file(maps_f
)
348 except (IOError, OSError):
351 def append_line(self
, line
):
352 entry
= self
.parse_line(line
)
354 self
._append
_entry
(entry
)
358 def parse_line(line
):
359 matched
= ProcMaps
.MAPS_PATTERN
.match(line
)
361 return ProcMapsEntry( # pylint: disable=W0212
362 int(matched
.group(1), 16), # begin
363 int(matched
.group(2), 16), # end
364 matched
.group(3), # readable
365 matched
.group(4), # writable
366 matched
.group(5), # executable
367 matched
.group(6), # private
368 int(matched
.group(7), 16), # offset
369 matched
.group(8), # major
370 matched
.group(9), # minor
371 int(matched
.group(10), 10), # inode
372 matched
.group(11) # name
378 def constants(entry
):
379 return entry
.writable
== '-' and entry
.executable
== '-'
382 def executable(entry
):
383 return entry
.executable
== 'x'
386 def executable_and_constants(entry
):
387 return ((entry
.writable
== '-' and entry
.executable
== '-') or
388 entry
.executable
== 'x')
390 def _append_entry(self
, entry
):
391 if self
._sorted
_indexes
and self
._sorted
_indexes
[-1] > entry
.begin
:
393 self
._sorted
_indexes
.append(entry
.begin
)
394 self
._dictionary
[entry
.begin
] = entry
397 class ProcSmaps(object):
398 """Reads and stores information in /proc/pid/smaps."""
399 _SMAPS_PATTERN
= re
.compile(r
'^(?P<NAME>[A-Za-z0-9_]+):\s+(?P<VALUE>.*)')
407 def append(self
, name
, value
):
412 'Referenced': '_referenced',
413 'Private_Clean': '_private_clean',
414 'Shared_Clean': '_shared_clean',
415 'KernelPageSize': '_kernel_page_size',
416 'MMUPageSize': '_mmu_page_size',
419 self
.__setattr
__(dct
[name
], value
)
423 if self
._size
.endswith('kB'):
424 return int(self
._size
.split()[0])
425 return int(self
._size
)
429 if self
._rss
.endswith('kB'):
430 return int(self
._rss
.split()[0])
431 return int(self
._rss
)
435 if self
._pss
.endswith('kB'):
436 return int(self
._pss
.split()[0])
437 return int(self
._pss
)
439 def __init__(self
, raw
, total_dct
, maps
, vma_internals
):
441 self
._size
= total_dct
['Size']
442 self
._rss
= total_dct
['Rss']
443 self
._pss
= total_dct
['Pss']
444 self
._referenced
= total_dct
['Referenced']
445 self
._shared
_clean
= total_dct
['Shared_Clean']
446 self
._private
_clean
= total_dct
['Private_Clean']
447 self
._kernel
_page
_size
= total_dct
['KernelPageSize']
448 self
._mmu
_page
_size
= total_dct
['MMUPageSize']
450 self
._vma
_internals
= vma_internals
454 with
open(os
.path
.join('/proc', str(pid
), 'smaps'), 'r') as smaps_f
:
455 raw
= smaps_f
.readlines()
458 vma_internals
= collections
.OrderedDict()
459 total_dct
= collections
.defaultdict(int)
462 maps_match
= ProcMaps
.MAPS_PATTERN
.match(line
)
464 vma
= maps
.append_line(line
.strip())
465 vma_internals
[vma
] = ProcSmaps
.VMA()
467 smaps_match
= ProcSmaps
._SMAPS
_PATTERN
.match(line
)
469 match_dict
= smaps_match
.groupdict()
470 vma_internals
[vma
].append(match_dict
['NAME'], match_dict
['VALUE'])
471 total_dct
[match_dict
['NAME']] += int(match_dict
['VALUE'].split()[0])
473 return ProcSmaps(raw
, total_dct
, maps
, vma_internals
)
484 def referenced(self
):
485 return self
._referenced
492 def private_clean(self
):
493 return self
._private
_clean
496 def shared_clean(self
):
497 return self
._shared
_clean
500 def kernel_page_size(self
):
501 return self
._kernel
_page
_size
504 def mmu_page_size(self
):
505 return self
._mmu
_page
_size
508 def vma_internals(self
):
509 return self
._vma
_internals
512 class ProcPagemap(object):
513 """Reads and stores partial information in /proc/pid/pagemap.
515 It picks up virtual addresses to read based on ProcMaps (/proc/pid/maps).
516 See https://www.kernel.org/doc/Documentation/vm/pagemap.txt for details.
518 _BYTES_PER_PAGEMAP_VALUE
= 8
519 _BYTES_PER_OS_PAGE
= 4096
520 _VIRTUAL_TO_PAGEMAP_OFFSET
= _BYTES_PER_OS_PAGE
/ _BYTES_PER_PAGEMAP_VALUE
522 _MASK_PRESENT
= 1 << 63
523 _MASK_SWAPPED
= 1 << 62
524 _MASK_FILEPAGE_OR_SHAREDANON
= 1 << 61
525 _MASK_SOFTDIRTY
= 1 << 55
526 _MASK_PFN
= (1 << 55) - 1
529 def __init__(self
, vsize
, present
, swapped
, pageframes
):
531 self
._present
= present
532 self
._swapped
= swapped
533 self
._pageframes
= pageframes
537 return int(self
._vsize
)
541 return int(self
._present
)
545 return int(self
._swapped
)
548 def pageframes(self
):
549 return self
._pageframes
551 def __init__(self
, vsize
, present
, swapped
, vma_internals
, in_process_dup
):
553 self
._present
= present
554 self
._swapped
= swapped
555 self
._vma
_internals
= vma_internals
556 self
._in
_process
_dup
= in_process_dup
564 vma_internals
= collections
.OrderedDict()
565 process_pageframe_set
= set()
568 pagemap_fd
= os
.open(
569 os
.path
.join('/proc', str(pid
), 'pagemap'), os
.O_RDONLY
)
570 except (IOError, OSError):
576 pageframes
= collections
.defaultdict(int)
577 begin_offset
= ProcPagemap
._offset
(vma
.begin
)
578 chunk_size
= ProcPagemap
._offset
(vma
.end
) - begin_offset
580 os
.lseek(pagemap_fd
, begin_offset
, os
.SEEK_SET
)
581 buf
= os
.read(pagemap_fd
, chunk_size
)
582 except (IOError, OSError):
584 if len(buf
) < chunk_size
:
585 _LOGGER
.warn('Failed to read pagemap at 0x%x in %d.' % (vma
.begin
, pid
))
586 pagemap_values
= struct
.unpack(
587 '=%dQ' % (len(buf
) / ProcPagemap
._BYTES
_PER
_PAGEMAP
_VALUE
), buf
)
588 for pagemap_value
in pagemap_values
:
589 vsize
+= ProcPagemap
._BYTES
_PER
_OS
_PAGE
590 if pagemap_value
& ProcPagemap
._MASK
_PRESENT
:
591 if (pagemap_value
& ProcPagemap
._MASK
_PFN
) in process_pageframe_set
:
592 in_process_dup
+= ProcPagemap
._BYTES
_PER
_OS
_PAGE
594 process_pageframe_set
.add(pagemap_value
& ProcPagemap
._MASK
_PFN
)
595 if (pagemap_value
& ProcPagemap
._MASK
_PFN
) not in pageframes
:
596 present
+= ProcPagemap
._BYTES
_PER
_OS
_PAGE
597 pageframes
[pagemap_value
& ProcPagemap
._MASK
_PFN
] += 1
598 if pagemap_value
& ProcPagemap
._MASK
_SWAPPED
:
599 swapped
+= ProcPagemap
._BYTES
_PER
_OS
_PAGE
600 vma_internals
[vma
] = ProcPagemap
.VMA(vsize
, present
, swapped
, pageframes
)
601 total_present
+= present
602 total_swapped
+= swapped
609 return ProcPagemap(total_vsize
, total_present
, total_swapped
,
610 vma_internals
, in_process_dup
)
613 def _offset(virtual_address
):
614 return virtual_address
/ ProcPagemap
._VIRTUAL
_TO
_PAGEMAP
_OFFSET
618 return int(self
._vsize
)
622 return int(self
._present
)
626 return int(self
._swapped
)
629 def vma_internals(self
):
630 return self
._vma
_internals
633 class _ProcessMemory(object):
634 """Aggregates process memory information from /proc for manual testing."""
635 def __init__(self
, pid
):
644 def _read(self
, proc_file
):
646 with
open(os
.path
.join('/proc', str(self
._pid
), proc_file
), 'r') as proc_f
:
647 lines
= proc_f
.readlines()
656 self
.read_pagemap(self
._maps
)
659 self
._maps
= ProcMaps
.load(self
._pid
)
661 def read_pagemap(self
, maps
):
662 self
._pagemap
= ProcPagemap
.load(self
._pid
, maps
)
664 def read_smaps(self
):
665 self
._smaps
= ProcSmaps
.load(self
._pid
)
668 self
._stat
= ProcStat
.load(self
._pid
)
670 def read_statm(self
):
671 self
._statm
= ProcStatm
.load(self
._pid
)
673 def read_status(self
):
674 self
._status
= ProcStatus
.load(self
._pid
)
706 """The main function for manual testing."""
707 _LOGGER
.setLevel(logging
.WARNING
)
708 handler
= logging
.StreamHandler()
709 handler
.setLevel(logging
.WARNING
)
710 handler
.setFormatter(logging
.Formatter(
711 '%(asctime)s:%(name)s:%(levelname)s:%(message)s'))
712 _LOGGER
.addHandler(handler
)
719 raise SyntaxError("%s is not an integer." % arg
)
725 procs
[pid
] = _ProcessMemory(pid
)
726 procs
[pid
].read_all()
728 print '=== PID: %d ===' % pid
730 print ' stat: %d' % procs
[pid
].stat
.vsize
731 print ' statm: %d' % (procs
[pid
].statm
.size
* 4096)
732 print ' status: %d (Peak:%d)' % (procs
[pid
].status
.vm_size
* 1024,
733 procs
[pid
].status
.vm_peak
* 1024)
734 print ' smaps: %d' % (procs
[pid
].smaps
.size
* 1024)
735 print 'pagemap: %d' % procs
[pid
].pagemap
.vsize
736 print ' stat: %d' % (procs
[pid
].stat
.rss
* 4096)
737 print ' statm: %d' % (procs
[pid
].statm
.resident
* 4096)
738 print ' status: %d (Peak:%d)' % (procs
[pid
].status
.vm_rss
* 1024,
739 procs
[pid
].status
.vm_hwm
* 1024)
740 print ' smaps: %d' % (procs
[pid
].smaps
.rss
* 1024)
741 print 'pagemap: %d' % procs
[pid
].pagemap
.present
746 if __name__
== '__main__':
747 sys
.exit(main(sys
.argv
))