iotop.py

   1 #!/usr/bin/python
   2 # iotop: Display I/O usage of processes in a top like UI
   3 # Copyright (c) 2007 Guillaume Chazarain <guichaz@yahoo.fr>, GPLv2
   4 # See ./iotop.py --help for some help
   5
   6 import curses
   7 import errno
   8 import optparse
   9 import os
  10 import pwd
  11 import select
  12 import socket
  13 import struct
  14 import sys
  15 import time
  16
  17 #
  18 # Check for requirements:
  19 #   o Python >= 2.5 for AF_NETLINK sockets
  20 #   o Linux >= 2.6.20 with I/O accounting
  21 #
  22 try:
  23     socket.NETLINK_ROUTE
  24     python25 = True
  25 except AttributeError:
  26     python25 = False
  27
  28 ioaccounting = os.path.exists('/proc/self/io')
  29
  30 if not python25 or not ioaccounting:
  31     def boolean2string(boolean):
  32         return boolean and 'Found' or 'Not found'
  33     print 'Could not run iotop as some of the requirements are not met:'
  34     print '- Python >= 2.5 for AF_NETLINK support:', boolean2string(python25)
  35     print '- Linux >= 2.6.20 with I/O accounting support:', \
  36           boolean2string(ioaccounting)
  37     sys.exit(1)
  38
  39 #
  40 # Netlink stuff
  41 # Based on code from pynl80211: Netlink message generation/parsing
  42 # http://git.sipsolutions.net/?p=pynl80211.git
  43 # Copyright 2007 Johannes Berg <johannes@sipsolutions.net>
  44 # GPLv2
  45
  46 # flags
  47 NLM_F_REQUEST = 1
  48
  49 # types
  50 NLMSG_ERROR = 2
  51 NLMSG_MIN_TYPE = 0x10
  52
  53 class Attr:
  54     def __init__(self, type, str, *kw):
  55         self.type = type
  56         if len(kw):
  57             self.data = struct.pack(str, *kw)
  58         else:
  59             self.data = str
  60
  61     def _dump(self):
  62         hdr = struct.pack('HH', len(self.data)+4, self.type)
  63         length = len(self.data)
  64         pad = ((length + 4 - 1) & ~3 ) - length
  65         return hdr + self.data + '\0' * pad
  66
  67     def u16(self):
  68         return struct.unpack('H', self.data)[0]
  69
  70 class NulStrAttr(Attr):
  71     def __init__(self, type, str):
  72         Attr.__init__(self, type, '%dsB'%len(str), str, 0)
  73
  74 class U32Attr(Attr):
  75     def __init__(self, type, val):
  76         Attr.__init__(self, type, 'L', val)
  77
  78 NETLINK_GENERIC = 16
  79
  80 class Message:
  81     def __init__(self, tp, flags = 0, seq = -1, payload = []):
  82         self.type = tp
  83         self.flags = flags
  84         self.seq = seq
  85         self.pid = -1
  86         if type(payload) == list:
  87             contents = []
  88             for attr in payload:
  89                 contents.append(attr._dump())
  90             self.payload = ''.join(contents)
  91         else:
  92             self.payload = payload
  93
  94     def send(self, conn):
  95         if self.seq == -1:
  96             self.seq = conn.seq()
  97
  98         self.pid = conn.pid
  99         length = len(self.payload)
 100
 101         hdr = struct.pack('IHHII', length + 4*4, self.type, self.flags,
 102                           self.seq, self.pid)
 103         conn.send(hdr + self.payload)
 104
 105 class Connection:
 106     def __init__(self, nltype, groups=0, unexpected_msg_handler = None):
 107         self.fd = socket.socket(socket.AF_NETLINK, socket.SOCK_RAW, nltype)
 108         self.fd.setsockopt(socket.SOL_SOCKET, socket.SO_SNDBUF, 65536)
 109         self.fd.setsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF, 65536)
 110         self.fd.bind((0, groups))
 111         self.pid, self.groups = self.fd.getsockname()
 112         self._seq = 0
 113         self.unexpected = unexpected_msg_handler
 114
 115     def send(self, msg):
 116         self.fd.send(msg)
 117
 118     def recv(self):
 119         cntnts = self.fd.recv(65536)
 120         # should check msgflags for TRUNC!
 121         len, type, flags, seq, pid = struct.unpack('IHHII', cntnts[:16])
 122         m = Message(type, flags, seq, cntnts[16:])
 123         m.pid = pid
 124         if m.type == NLMSG_ERROR:
 125             errno = -struct.unpack('i', m.payload[:4])[0]
 126             if errno != 0:
 127                 e = OSError('Netlink error: %s (%d)' % \
 128                                                     (os.strerror(errno), errno))
 129                 e.errno = errno
 130         return m
 131
 132     def seq(self):
 133         self._seq += 1
 134         return self._seq
 135
 136 def parse_attributes(str):
 137     attrs = {}
 138     while str:
 139         l, tp = struct.unpack('HH', str[:4])
 140         attrs[tp] = Attr(tp, str[4:l])
 141         l = ((l + 4 - 1) & ~3 )
 142         str = str[l:]
 143     return attrs
 144
 145 CTRL_CMD_GETFAMILY = 3
 146
 147 CTRL_ATTR_FAMILY_ID = 1
 148 CTRL_ATTR_FAMILY_NAME = 2
 149
 150 class GenlHdr:
 151     def __init__(self, cmd, version = 0):
 152         self.cmd = cmd
 153         self.version = version
 154
 155     def _dump(self):
 156         return struct.pack('BBxx', self.cmd, self.version)
 157
 158 def _genl_hdr_parse(data):
 159     return GenlHdr(*struct.unpack('BBxx', data))
 160
 161 GENL_ID_CTRL = NLMSG_MIN_TYPE
 162
 163 class GeNlMessage(Message):
 164     def __init__(self, family, cmd, attrs=[], flags=0):
 165         self.cmd = cmd
 166         self.attrs = attrs
 167         self.family = family
 168         Message.__init__(self, family, flags=flags,
 169                          payload=[GenlHdr(self.cmd)] + attrs)
 170
 171 class Controller:
 172     def __init__(self, conn):
 173         self.conn = conn
 174
 175     def get_family_id(self, family):
 176         a = NulStrAttr(CTRL_ATTR_FAMILY_NAME, family)
 177         m = GeNlMessage(GENL_ID_CTRL, CTRL_CMD_GETFAMILY,
 178                         flags=NLM_F_REQUEST, attrs=[a])
 179         m.send(self.conn)
 180         m = self.conn.recv()
 181         gh = _genl_hdr_parse(m.payload[:4])
 182         attrs = parse_attributes(m.payload[4:])
 183         return attrs[CTRL_ATTR_FAMILY_ID].u16()
 184
 185 #
 186 # Netlink usage for taskstats
 187 #
 188
 189 TASKSTATS_CMD_GET = 1
 190 TASKSTATS_CMD_ATTR_PID = 1
 191 TASKSTATS_CMD_ATTR_TGID = 2
 192
 193 class TaskStatsNetlink(object):
 194     # Keep in sync with human_stats(stats, duration)
 195     members_offsets = [
 196         ('blkio_delay_total', 40),
 197         ('swapin_delay_total', 56),
 198         ('ac_etime', 144),
 199         ('read_bytes', 248),
 200         ('write_bytes', 256),
 201         ('cancelled_write_bytes', 264)
 202     ]
 203
 204     def __init__(self, options):
 205         self.options = options
 206         self.connection = Connection(NETLINK_GENERIC)
 207         controller = Controller(self.connection)
 208         self.family_id = controller.get_family_id('TASKSTATS')
 209
 210     def get_task_stats(self, pid):
 211         if self.options.processes:
 212             attr = TASKSTATS_CMD_ATTR_TGID
 213         else:
 214             attr = TASKSTATS_CMD_ATTR_PID
 215         request = GeNlMessage(self.family_id, cmd=TASKSTATS_CMD_GET,
 216                               attrs=[U32Attr(attr, pid)],
 217                               flags=NLM_F_REQUEST)
 218         request.send(self.connection)
 219         try:
 220             reply = self.connection.recv()
 221         except OSError, e:
 222             if e.errno == errno.ESRCH:
 223                 # OSError: Netlink error: No such process (3)
 224                 return
 225             raise
 226         if len(reply.payload) != 292:
 227             return
 228         reply_data = reply.payload[20:]
 229
 230         reply_length, reply_type = struct.unpack('HH', reply.payload[4:8])
 231         reply_version = struct.unpack('H', reply.payload[20:22])[0]
 232         assert (reply_length, reply_type, reply_version) == (288, attr + 3, 4)
 233
 234         res = {}
 235         for name, offset in TaskStatsNetlink.members_offsets:
 236             data = reply_data[offset: offset + 8]
 237             res[name] = struct.unpack('Q', data)[0]
 238
 239         return res
 240
 241 #
 242 # PIDs manipulations
 243 #
 244
 245 def find_uids(options):
 246     options.uids = []
 247     error = False
 248     for u in options.users or []:
 249         try:
 250             uid = int(u)
 251         except ValueError:
 252             try:
 253                 passwd = pwd.getpwnam(u)
 254             except KeyError:
 255                 print >> sys.stderr, 'Unknown user:', u
 256                 error = True
 257             else:
 258                 uid = passwd.pw_uid
 259         if not error:
 260             options.uids.append(uid)
 261     if error:
 262         sys.exit(1)
 263
 264 class pinfo(object):
 265     def __init__(self, pid, options):
 266         self.mark = False
 267         self.pid = pid
 268         self.stats = {}
 269         for name, offset in TaskStatsNetlink.members_offsets:
 270             self.stats[name] = (0, 0) # Total, Delta
 271         self.parse_status('/proc/%d/status' % pid, options)
 272
 273     def check_if_valid(self, uid, options):
 274         self.valid = not options.uids and not options.pids
 275         if not self.valid:
 276             self.valid = uid in options.uids
 277         if not self.valid:
 278             self.valid = self.pid in options.pids
 279
 280     def parse_status(self, path, options):
 281         for line in open(path):
 282             if line.startswith('Name:'):
 283                 # Name kernel threads
 284                 self.name = '[' + line.split()[1].strip() + ']'
 285             elif line.startswith('Uid:'):
 286                 uid = int(line.split()[1])
 287                 # We check monitored PIDs only here
 288                 self.check_if_valid(uid, options)
 289                 try:
 290                     self.user = pwd.getpwuid(uid).pw_name
 291                 except KeyError:
 292                     self.user = str(uid)
 293                 break
 294
 295     def add_stats(self, stats):
 296         self.stats_timestamp = time.time()
 297         for name, value in stats.iteritems():
 298             prev_value = self.stats[name][0]
 299             self.stats[name] = (value, value - prev_value)
 300
 301     def get_cmdline(self, max_length):
 302         # A process may exec, so we must always reread its cmdline
 303         try:
 304             proc_cmdline = open('/proc/%d/cmdline' % self.pid)
 305         except IOError:
 306             return '{no such process}'
 307         cmdline = proc_cmdline.read(max_length)
 308         parts = cmdline.split('\0')
 309         first_command_char = parts[0].rfind('/') + 1
 310         parts[0] = parts[0][first_command_char:]
 311         cmdline = ' '.join(parts).strip()
 312         return cmdline.encode('string_escape') or self.name
 313
 314 class ProcessList(object):
 315     def __init__(self, taskstats_connection, options):
 316         # {pid: pinfo}
 317         self.processes = {}
 318         self.taskstats_connection = taskstats_connection
 319         self.options = options
 320
 321         # A first time as we are interested in the delta
 322         self.update_process_counts()
 323
 324     def get_process(self, pid):
 325         process = self.processes.get(pid, None)
 326         if not process:
 327             try:
 328                 process = pinfo(pid, self.options)
 329             except IOError:
 330                 # IOError: [Errno 2] No such file or directory: '/proc/...'
 331                 return
 332             if not process.valid:
 333                 return
 334             self.processes[pid] = process
 335         return process
 336
 337     def list_pids(self, tgid):
 338         if self.options.processes:
 339             return [tgid]
 340         try:
 341             return map(int, os.listdir('/proc/%d/task' % tgid))
 342         except OSError:
 343             return []
 344
 345     def update_process_counts(self):
 346         total_read = total_write = 0
 347         duration = None
 348         tgids = [int(tgid) for tgid in os.listdir('/proc') if
 349                                               '0' <= tgid[0] and tgid[0] <= '9']
 350         for tgid in tgids:
 351             for pid in self.list_pids(tgid):
 352                 process = self.get_process(pid)
 353                 if process:
 354                     stats = self.taskstats_connection.get_task_stats(pid)
 355                     if stats:
 356                         process.mark = False
 357                         process.add_stats(stats)
 358                         total_read += process.stats['read_bytes'][1]
 359                         total_write += process.stats['write_bytes'][1]
 360                         if duration is None:
 361                             duration = process.stats['ac_etime'][1] / 1000000.0
 362         return total_read, total_write, duration
 363
 364     def refresh_processes(self):
 365         for process in self.processes.values():
 366             process.mark = True
 367         total_read_and_write_and_duration = self.update_process_counts()
 368         to_delete = []
 369         for pid, process in self.processes.iteritems():
 370             if process.mark:
 371                 to_delete.append(pid)
 372         for pid in to_delete:
 373             del self.processes[pid]
 374         return total_read_and_write_and_duration
 375
 376 #
 377 # Utility functions for the UI
 378 #
 379
 380 UNITS = ['B', 'K', 'M', 'G', 'T', 'P', 'E']
 381
 382 def human_bandwidth(size, duration):
 383     bw = float(size) / duration
 384     for i in xrange(len(UNITS) - 1, 0, -1):
 385         base = 1 << (10 * i)
 386         if 2 * base < size:
 387             res = '%.2f %s' % ((float(size) / base), UNITS[i])
 388             break
 389     else:
 390         res = str(size) + ' ' + UNITS[0]
 391     return res + '/s'
 392
 393 def human_stats(stats):
 394     # Keep in sync with TaskStatsNetlink.members_offsets and
 395     # IOTopUI.get_data(self)
 396     duration = stats['ac_etime'][1] / 1000000.0
 397     def delay2percent(name): # delay in ns, duration in s
 398         return '%.2f %%' % min(99.99, stats[name][1] / (duration * 10000000.0))
 399     io_delay = delay2percent('blkio_delay_total')
 400     swapin_delay = delay2percent('swapin_delay_total')
 401     read_bytes = human_bandwidth(stats['read_bytes'][1], duration)
 402     written_bytes = stats['write_bytes'][1] - stats['cancelled_write_bytes'][1]
 403     written_bytes = max(0, written_bytes)
 404     write_bytes = human_bandwidth(written_bytes, duration)
 405     return io_delay, swapin_delay, read_bytes, write_bytes
 406
 407 #
 408 # The UI
 409 #
 410
 411 class IOTopUI(object):
 412     # key, reverse
 413     sorting_keys = [
 414         (lambda p: p.pid, False),
 415         (lambda p: p.user, False),
 416         (lambda p: p.stats['read_bytes'][1], True),
 417         (lambda p: p.stats['write_bytes'][1] -
 418                    p.stats['cancelled_write_bytes'][1], True),
 419         (lambda p: p.stats['swapin_delay_total'][1], True),
 420         # The default sorting (by I/O % time) should show processes doing
 421         # only writes, without waiting on them
 422         (lambda p: p.stats['blkio_delay_total'][1] or
 423                    int(not(not(p.stats['read_bytes'][1] or
 424                                p.stats['write_bytes'][1]))), True),
 425         (lambda p: p.get_cmdline(4096), False),
 426     ]
 427
 428     def __init__(self, win, process_list, options):
 429         self.process_list = process_list
 430         self.options = options
 431         self.sorting_key = 5
 432         self.sorting_reverse = IOTopUI.sorting_keys[5][1]
 433         if not self.options.batch:
 434             self.win = win
 435             self.resize()
 436             curses.use_default_colors()
 437             curses.start_color()
 438             curses.curs_set(0)
 439
 440     def resize(self, *unused):
 441         self.height, self.width = self.win.getmaxyx()
 442
 443     def run(self):
 444         iterations = 0
 445         poll = select.poll()
 446         if not self.options.batch:
 447             poll.register(sys.stdin.fileno(), select.POLLIN|select.POLLPRI)
 448         while self.options.iterations is None or \
 449               iterations < self.options.iterations:
 450             total = self.process_list.refresh_processes()
 451             total_read, total_write, duration = total
 452             self.refresh_display(total_read, total_write, duration)
 453             if self.options.iterations is not None:
 454                 iterations += 1
 455                 if iterations >= self.options.iterations:
 456                     break
 457
 458             events = poll.poll(self.options.delay_seconds * 1000.0)
 459             if events:
 460                 key = self.win.getch()
 461                 self.handle_key(key)
 462
 463     def reverse_sorting(self):
 464         self.sorting_reverse = not self.sorting_reverse
 465
 466     def adjust_sorting_key(self, delta):
 467         orig_sorting_key = self.sorting_key
 468         self.sorting_key += delta
 469         self.sorting_key = max(0, self.sorting_key)
 470         self.sorting_key = min(len(IOTopUI.sorting_keys) - 1, self.sorting_key)
 471         if orig_sorting_key != self.sorting_key:
 472             self.sorting_reverse = IOTopUI.sorting_keys[self.sorting_key][1]
 473
 474     def handle_key(self, key):
 475         key_bindings = {
 476             ord('q'):
 477                 lambda: sys.exit(0),
 478             ord('Q'):
 479                 lambda: sys.exit(0),
 480             ord('r'):
 481                 lambda: self.reverse_sorting(),
 482             ord('R'):
 483                 lambda: self.reverse_sorting(),
 484             curses.KEY_LEFT:
 485                 lambda: self.adjust_sorting_key(-1),
 486             curses.KEY_RIGHT:
 487                 lambda: self.adjust_sorting_key(1),
 488             curses.KEY_HOME:
 489                 lambda: self.adjust_sorting_key(-len(IOTopUI.sorting_keys)),
 490             curses.KEY_END:
 491                 lambda: self.adjust_sorting_key(len(IOTopUI.sorting_keys))
 492         }
 493
 494         action = key_bindings.get(key, lambda: None)
 495         action()
 496
 497     def get_data(self):
 498         if self.options.batch:
 499             max_length = 4096
 500         else:
 501             max_length = self.width
 502         def format(p):
 503             stats = human_stats(p.stats)
 504             io_delay, swapin_delay, read_bytes, write_bytes = stats
 505             return '%5d %-8s %11s %11s %7s %7s %s' % \
 506                    (p.pid, p.user[:8], read_bytes, write_bytes, swapin_delay,
 507                     io_delay, p.get_cmdline(max_length))
 508         processes = self.process_list.processes.values()
 509         key = IOTopUI.sorting_keys[self.sorting_key][0]
 510         processes.sort(key=key, reverse=self.sorting_reverse)
 511         if not self.options.batch:
 512             del processes[self.height - 2:]
 513         return map(format, processes)
 514
 515     def refresh_display(self, total_read, total_write, duration):
 516         summary = 'Total DISK READ: %s | Total DISK WRITE: %s' % (
 517                                         human_bandwidth(total_read, duration),
 518                                         human_bandwidth(total_write, duration))
 519         titles = ['  PID', ' USER', '      DISK READ', '  DISK WRITE',
 520                   '   SWAPIN', '    IO', '    COMMAND']
 521         lines = self.get_data()
 522         if self.options.batch:
 523             print summary
 524             print ''.join(titles)
 525             for l in lines:
 526                 print l
 527         else:
 528             self.win.clear()
 529             self.win.addstr(summary)
 530             self.win.hline(1, 0, ord(' ') | curses.A_REVERSE, self.width)
 531             for i in xrange(len(titles)):
 532                 attr = curses.A_REVERSE
 533                 title = titles[i]
 534                 if i == self.sorting_key:
 535                     attr |= curses.A_BOLD
 536                     title += self.sorting_reverse and '>' or '<'
 537                 self.win.addstr(title, attr)
 538             for i in xrange(len(lines)):
 539                 self.win.insstr(i + 2, 0, lines[i])
 540             self.win.refresh()
 541
 542 def run_iotop(win, options):
 543     taskstats_connection = TaskStatsNetlink(options)
 544     process_list = ProcessList(taskstats_connection, options)
 545     ui = IOTopUI(win, process_list, options)
 546     ui.run()
 547
 548 #
 549 # Main program
 550 #
 551
 552 VERSION = '0.1'
 553
 554 USAGE = 'Usage: %s [OPTIONS]' % sys.argv[0] + '''
 555
 556 DISK READ and DISK WRITE are the block I/O bandwidth used during the sampling
 557 period. SWAPIN and IO are the percentages of time the thread spent respectively
 558 while swapping in and waiting on I/O more generally.
 559 Controls: left and right arrows to should the sorting column, r to invert the
 560 sorting order, q to quit, any other key to force a refresh'''
 561
 562 def main():
 563     parser = optparse.OptionParser(usage=USAGE, version='iotop ' + VERSION)
 564     parser.add_option('-d', '--delay', type='float', dest='delay_seconds',
 565                       help='delay between iterations [1 second]',
 566                       metavar='SEC', default=1)
 567     parser.add_option('-p', '--pid', type='int', dest='pids', action='append',
 568                       help='processes to monitor [all]', metavar='PID')
 569     parser.add_option('-u', '--user', type='str', dest='users', action='append',
 570                       help='users to monitor [all]', metavar='USER')
 571     parser.add_option('-b', '--batch', action='store_true', dest='batch',
 572                       help='non-interactive mode')
 573     parser.add_option('-P', '--processes', action='store_true',
 574                       dest='processes',
 575                       help='show only processes, not all threads')
 576     parser.add_option('-n', '--iter', type='int', dest='iterations',
 577                       metavar='NUM',
 578                       help='number of iterations before ending [infinite]')
 579     options, args = parser.parse_args()
 580     if args:
 581         parser.error('Unexpected arguments: ' + ' '.join(args))
 582     find_uids(options)
 583     options.pids = options.pids or []
 584     if options.batch:
 585         run_iotop(None, options)
 586     else:
 587         curses.wrapper(run_iotop, options)
 588
 589 if __name__ == '__main__':
 590     try:
 591         main()
 592     except KeyboardInterrupt:
 593         pass
 594     sys.exit(0)
 595