scripts/kvm/kvm_stat

   1 #!/usr/bin/python
   2 #
   3 # top-like utility for displaying kvm statistics
   4 #
   5 # Copyright 2006-2008 Qumranet Technologies
   6 # Copyright 2008-2011 Red Hat, Inc.
   7 #
   8 # Authors:
   9 #  Avi Kivity <avi@redhat.com>
  10 #
  11 # This work is licensed under the terms of the GNU GPL, version 2.  See
  12 # the COPYING file in the top-level directory.
  13
  14 import curses
  15 import sys, os, time, optparse, ctypes
  16 from ctypes import *
  17
  18 class DebugfsProvider(object):
  19     def __init__(self):
  20         self.base = '/sys/kernel/debug/kvm'
  21         self._fields = os.listdir(self.base)
  22     def fields(self):
  23         return self._fields
  24     def select(self, fields):
  25         self._fields = fields
  26     def read(self):
  27         def val(key):
  28             return int(file(self.base + '/' + key).read())
  29         return dict([(key, val(key)) for key in self._fields])
  30
  31 vmx_exit_reasons = {
  32     0: 'EXCEPTION_NMI',
  33     1: 'EXTERNAL_INTERRUPT',
  34     2: 'TRIPLE_FAULT',
  35     7: 'PENDING_INTERRUPT',
  36     8: 'NMI_WINDOW',
  37     9: 'TASK_SWITCH',
  38     10: 'CPUID',
  39     12: 'HLT',
  40     14: 'INVLPG',
  41     15: 'RDPMC',
  42     16: 'RDTSC',
  43     18: 'VMCALL',
  44     19: 'VMCLEAR',
  45     20: 'VMLAUNCH',
  46     21: 'VMPTRLD',
  47     22: 'VMPTRST',
  48     23: 'VMREAD',
  49     24: 'VMRESUME',
  50     25: 'VMWRITE',
  51     26: 'VMOFF',
  52     27: 'VMON',
  53     28: 'CR_ACCESS',
  54     29: 'DR_ACCESS',
  55     30: 'IO_INSTRUCTION',
  56     31: 'MSR_READ',
  57     32: 'MSR_WRITE',
  58     33: 'INVALID_STATE',
  59     36: 'MWAIT_INSTRUCTION',
  60     39: 'MONITOR_INSTRUCTION',
  61     40: 'PAUSE_INSTRUCTION',
  62     41: 'MCE_DURING_VMENTRY',
  63     43: 'TPR_BELOW_THRESHOLD',
  64     44: 'APIC_ACCESS',
  65     48: 'EPT_VIOLATION',
  66     49: 'EPT_MISCONFIG',
  67     54: 'WBINVD',
  68     55: 'XSETBV',
  69     56: 'APIC_WRITE',
  70     58: 'INVPCID',
  71 }
  72
  73 svm_exit_reasons = {
  74     0x000: 'READ_CR0',
  75     0x003: 'READ_CR3',
  76     0x004: 'READ_CR4',
  77     0x008: 'READ_CR8',
  78     0x010: 'WRITE_CR0',
  79     0x013: 'WRITE_CR3',
  80     0x014: 'WRITE_CR4',
  81     0x018: 'WRITE_CR8',
  82     0x020: 'READ_DR0',
  83     0x021: 'READ_DR1',
  84     0x022: 'READ_DR2',
  85     0x023: 'READ_DR3',
  86     0x024: 'READ_DR4',
  87     0x025: 'READ_DR5',
  88     0x026: 'READ_DR6',
  89     0x027: 'READ_DR7',
  90     0x030: 'WRITE_DR0',
  91     0x031: 'WRITE_DR1',
  92     0x032: 'WRITE_DR2',
  93     0x033: 'WRITE_DR3',
  94     0x034: 'WRITE_DR4',
  95     0x035: 'WRITE_DR5',
  96     0x036: 'WRITE_DR6',
  97     0x037: 'WRITE_DR7',
  98     0x040: 'EXCP_BASE',
  99     0x060: 'INTR',
 100     0x061: 'NMI',
 101     0x062: 'SMI',
 102     0x063: 'INIT',
 103     0x064: 'VINTR',
 104     0x065: 'CR0_SEL_WRITE',
 105     0x066: 'IDTR_READ',
 106     0x067: 'GDTR_READ',
 107     0x068: 'LDTR_READ',
 108     0x069: 'TR_READ',
 109     0x06a: 'IDTR_WRITE',
 110     0x06b: 'GDTR_WRITE',
 111     0x06c: 'LDTR_WRITE',
 112     0x06d: 'TR_WRITE',
 113     0x06e: 'RDTSC',
 114     0x06f: 'RDPMC',
 115     0x070: 'PUSHF',
 116     0x071: 'POPF',
 117     0x072: 'CPUID',
 118     0x073: 'RSM',
 119     0x074: 'IRET',
 120     0x075: 'SWINT',
 121     0x076: 'INVD',
 122     0x077: 'PAUSE',
 123     0x078: 'HLT',
 124     0x079: 'INVLPG',
 125     0x07a: 'INVLPGA',
 126     0x07b: 'IOIO',
 127     0x07c: 'MSR',
 128     0x07d: 'TASK_SWITCH',
 129     0x07e: 'FERR_FREEZE',
 130     0x07f: 'SHUTDOWN',
 131     0x080: 'VMRUN',
 132     0x081: 'VMMCALL',
 133     0x082: 'VMLOAD',
 134     0x083: 'VMSAVE',
 135     0x084: 'STGI',
 136     0x085: 'CLGI',
 137     0x086: 'SKINIT',
 138     0x087: 'RDTSCP',
 139     0x088: 'ICEBP',
 140     0x089: 'WBINVD',
 141     0x08a: 'MONITOR',
 142     0x08b: 'MWAIT',
 143     0x08c: 'MWAIT_COND',
 144     0x08d: 'XSETBV',
 145     0x400: 'NPF',
 146 }
 147
 148 # EC definition of HSR (from arch/arm64/include/asm/kvm_arm.h)
 149 aarch64_exit_reasons = {
 150     0x00: 'UNKNOWN',
 151     0x01: 'WFI',
 152     0x03: 'CP15_32',
 153     0x04: 'CP15_64',
 154     0x05: 'CP14_MR',
 155     0x06: 'CP14_LS',
 156     0x07: 'FP_ASIMD',
 157     0x08: 'CP10_ID',
 158     0x0C: 'CP14_64',
 159     0x0E: 'ILL_ISS',
 160     0x11: 'SVC32',
 161     0x12: 'HVC32',
 162     0x13: 'SMC32',
 163     0x15: 'SVC64',
 164     0x16: 'HVC64',
 165     0x17: 'SMC64',
 166     0x18: 'SYS64',
 167     0x20: 'IABT',
 168     0x21: 'IABT_HYP',
 169     0x22: 'PC_ALIGN',
 170     0x24: 'DABT',
 171     0x25: 'DABT_HYP',
 172     0x26: 'SP_ALIGN',
 173     0x28: 'FP_EXC32',
 174     0x2C: 'FP_EXC64',
 175     0x2F: 'SERROR',
 176     0x30: 'BREAKPT',
 177     0x31: 'BREAKPT_HYP',
 178     0x32: 'SOFTSTP',
 179     0x33: 'SOFTSTP_HYP',
 180     0x34: 'WATCHPT',
 181     0x35: 'WATCHPT_HYP',
 182     0x38: 'BKPT32',
 183     0x3A: 'VECTOR32',
 184     0x3C: 'BRK64',
 185 }
 186
 187 # From include/uapi/linux/kvm.h, KVM_EXIT_xxx
 188 userspace_exit_reasons = {
 189      0: 'UNKNOWN',
 190      1: 'EXCEPTION',
 191      2: 'IO',
 192      3: 'HYPERCALL',
 193      4: 'DEBUG',
 194      5: 'HLT',
 195      6: 'MMIO',
 196      7: 'IRQ_WINDOW_OPEN',
 197      8: 'SHUTDOWN',
 198      9: 'FAIL_ENTRY',
 199     10: 'INTR',
 200     11: 'SET_TPR',
 201     12: 'TPR_ACCESS',
 202     13: 'S390_SIEIC',
 203     14: 'S390_RESET',
 204     15: 'DCR',
 205     16: 'NMI',
 206     17: 'INTERNAL_ERROR',
 207     18: 'OSI',
 208     19: 'PAPR_HCALL',
 209     20: 'S390_UCONTROL',
 210     21: 'WATCHDOG',
 211     22: 'S390_TSCH',
 212     23: 'EPR',
 213     24: 'SYSTEM_EVENT',
 214 }
 215
 216 x86_exit_reasons = {
 217     'vmx': vmx_exit_reasons,
 218     'svm': svm_exit_reasons,
 219 }
 220
 221 sc_perf_evt_open = None
 222 exit_reasons = None
 223
 224 ioctl_numbers = {
 225     'SET_FILTER' : 0x40082406,
 226     'ENABLE'     : 0x00002400,
 227     'DISABLE'    : 0x00002401,
 228     'RESET'      : 0x00002403,
 229 }
 230
 231 def x86_init(flag):
 232     globals().update({
 233         'sc_perf_evt_open' : 298,
 234         'exit_reasons' : x86_exit_reasons[flag],
 235     })
 236
 237 def s390_init():
 238     globals().update({
 239         'sc_perf_evt_open' : 331
 240     })
 241
 242 def ppc_init():
 243     globals().update({
 244         'sc_perf_evt_open' : 319,
 245         'ioctl_numbers' : {
 246             'SET_FILTER' : 0x80002406 | (ctypes.sizeof(ctypes.c_char_p) << 16),
 247             'ENABLE'     : 0x20002400,
 248             'DISABLE'    : 0x20002401,
 249         }
 250     })
 251
 252 def aarch64_init():
 253     globals().update({
 254         'sc_perf_evt_open' : 241,
 255         'exit_reasons' : aarch64_exit_reasons,
 256     })
 257
 258 def detect_platform():
 259     if os.uname()[4].startswith('ppc'):
 260         ppc_init()
 261         return
 262     elif os.uname()[4].startswith('aarch64'):
 263         aarch64_init()
 264         return
 265
 266     for line in file('/proc/cpuinfo').readlines():
 267         if line.startswith('flags'):
 268             for flag in line.split():
 269                 if flag in x86_exit_reasons:
 270                     x86_init(flag)
 271                     return
 272         elif line.startswith('vendor_id'):
 273             for flag in line.split():
 274                 if flag == 'IBM/S390':
 275                     s390_init()
 276                     return
 277
 278 detect_platform()
 279
 280 def invert(d):
 281     return dict((x[1], x[0]) for x in d.iteritems())
 282
 283 filters = {}
 284 filters['kvm_userspace_exit'] = ('reason', invert(userspace_exit_reasons))
 285 if exit_reasons:
 286     filters['kvm_exit'] = ('exit_reason', invert(exit_reasons))
 287
 288 import struct, array
 289
 290 libc = ctypes.CDLL('libc.so.6')
 291 syscall = libc.syscall
 292 get_errno = libc.__errno_location
 293 get_errno.restype = POINTER(c_int)
 294
 295 class perf_event_attr(ctypes.Structure):
 296     _fields_ = [('type', ctypes.c_uint32),
 297                 ('size', ctypes.c_uint32),
 298                 ('config', ctypes.c_uint64),
 299                 ('sample_freq', ctypes.c_uint64),
 300                 ('sample_type', ctypes.c_uint64),
 301                 ('read_format', ctypes.c_uint64),
 302                 ('flags', ctypes.c_uint64),
 303                 ('wakeup_events', ctypes.c_uint32),
 304                 ('bp_type', ctypes.c_uint32),
 305                 ('bp_addr', ctypes.c_uint64),
 306                 ('bp_len', ctypes.c_uint64),
 307                 ]
 308 def _perf_event_open(attr, pid, cpu, group_fd, flags):
 309     return syscall(sc_perf_evt_open, ctypes.pointer(attr), ctypes.c_int(pid),
 310                    ctypes.c_int(cpu), ctypes.c_int(group_fd),
 311                    ctypes.c_long(flags))
 312
 313 PERF_TYPE_HARDWARE              = 0
 314 PERF_TYPE_SOFTWARE              = 1
 315 PERF_TYPE_TRACEPOINT            = 2
 316 PERF_TYPE_HW_CACHE              = 3
 317 PERF_TYPE_RAW                   = 4
 318 PERF_TYPE_BREAKPOINT            = 5
 319
 320 PERF_SAMPLE_IP                  = 1 << 0
 321 PERF_SAMPLE_TID                 = 1 << 1
 322 PERF_SAMPLE_TIME                = 1 << 2
 323 PERF_SAMPLE_ADDR                = 1 << 3
 324 PERF_SAMPLE_READ                = 1 << 4
 325 PERF_SAMPLE_CALLCHAIN           = 1 << 5
 326 PERF_SAMPLE_ID                  = 1 << 6
 327 PERF_SAMPLE_CPU                 = 1 << 7
 328 PERF_SAMPLE_PERIOD              = 1 << 8
 329 PERF_SAMPLE_STREAM_ID           = 1 << 9
 330 PERF_SAMPLE_RAW                 = 1 << 10
 331
 332 PERF_FORMAT_TOTAL_TIME_ENABLED  = 1 << 0
 333 PERF_FORMAT_TOTAL_TIME_RUNNING  = 1 << 1
 334 PERF_FORMAT_ID                  = 1 << 2
 335 PERF_FORMAT_GROUP               = 1 << 3
 336
 337 import re
 338
 339 sys_tracing = '/sys/kernel/debug/tracing'
 340
 341 class Group(object):
 342     def __init__(self, cpu):
 343         self.events = []
 344         self.group_leader = None
 345         self.cpu = cpu
 346     def add_event(self, name, event_set, tracepoint, filter = None):
 347         self.events.append(Event(group = self,
 348                                  name = name, event_set = event_set,
 349                                  tracepoint = tracepoint, filter = filter))
 350         if len(self.events) == 1:
 351             self.file = os.fdopen(self.events[0].fd)
 352     def read(self):
 353         bytes = 8 * (1 + len(self.events))
 354         fmt = 'xxxxxxxx' + 'q' * len(self.events)
 355         return dict(zip([event.name for event in self.events],
 356                         struct.unpack(fmt, self.file.read(bytes))))
 357
 358 class Event(object):
 359     def __init__(self, group, name, event_set, tracepoint, filter = None):
 360         self.name = name
 361         attr = perf_event_attr()
 362         attr.type = PERF_TYPE_TRACEPOINT
 363         attr.size = ctypes.sizeof(attr)
 364         id_path = os.path.join(sys_tracing, 'events', event_set,
 365                                tracepoint, 'id')
 366         id = int(file(id_path).read())
 367         attr.config = id
 368         attr.sample_type = (PERF_SAMPLE_RAW
 369                             | PERF_SAMPLE_TIME
 370                             | PERF_SAMPLE_CPU)
 371         attr.sample_period = 1
 372         attr.read_format = PERF_FORMAT_GROUP
 373         group_leader = -1
 374         if group.events:
 375             group_leader = group.events[0].fd
 376         fd = _perf_event_open(attr, -1, group.cpu, group_leader, 0)
 377         if fd == -1:
 378             err = get_errno()[0]
 379             raise Exception('perf_event_open failed, errno = ' + err.__str__())
 380         if filter:
 381             import fcntl
 382             fcntl.ioctl(fd, ioctl_numbers['SET_FILTER'], filter)
 383         self.fd = fd
 384     def enable(self):
 385         import fcntl
 386         fcntl.ioctl(self.fd, ioctl_numbers['ENABLE'], 0)
 387     def disable(self):
 388         import fcntl
 389         fcntl.ioctl(self.fd, ioctl_numbers['DISABLE'], 0)
 390     def reset(self):
 391         import fcntl
 392         fcntl.ioctl(self.fd, ioctl_numbers['RESET'], 0)
 393
 394 class TracepointProvider(object):
 395     def __init__(self):
 396         path = os.path.join(sys_tracing, 'events', 'kvm')
 397         fields = [f
 398                   for f in os.listdir(path)
 399                   if os.path.isdir(os.path.join(path, f))]
 400         extra = []
 401         for f in fields:
 402             if f in filters:
 403                 subfield, values = filters[f]
 404                 for name, number in values.iteritems():
 405                     extra.append(f + '(' + name + ')')
 406         fields += extra
 407         self._setup(fields)
 408         self.select(fields)
 409     def fields(self):
 410         return self._fields
 411
 412     def _online_cpus(self):
 413         l = []
 414         pattern = r'cpu([0-9]+)'
 415         basedir = '/sys/devices/system/cpu'
 416         for entry in os.listdir(basedir):
 417             match = re.match(pattern, entry)
 418             if not match:
 419                 continue
 420             path = os.path.join(basedir, entry, 'online')
 421             if os.path.exists(path) and open(path).read().strip() != '1':
 422                 continue
 423             l.append(int(match.group(1)))
 424         return l
 425
 426     def _setup(self, _fields):
 427         self._fields = _fields
 428         cpus = self._online_cpus()
 429         import resource
 430         nfiles = len(cpus) * 1000
 431         resource.setrlimit(resource.RLIMIT_NOFILE, (nfiles, nfiles))
 432         events = []
 433         self.group_leaders = []
 434         for cpu in cpus:
 435             group = Group(cpu)
 436             for name in _fields:
 437                 tracepoint = name
 438                 filter = None
 439                 m = re.match(r'(.*)\((.*)\)', name)
 440                 if m:
 441                     tracepoint, sub = m.groups()
 442                     filter = '%s==%d\0' % (filters[tracepoint][0],
 443                                            filters[tracepoint][1][sub])
 444                 event = group.add_event(name, event_set = 'kvm',
 445                                         tracepoint = tracepoint,
 446                                         filter = filter)
 447             self.group_leaders.append(group)
 448     def select(self, fields):
 449         for group in self.group_leaders:
 450             for event in group.events:
 451                 if event.name in fields:
 452                     event.reset()
 453                     event.enable()
 454                 else:
 455                     event.disable()
 456     def read(self):
 457         from collections import defaultdict
 458         ret = defaultdict(int)
 459         for group in self.group_leaders:
 460             for name, val in group.read().iteritems():
 461                 ret[name] += val
 462         return ret
 463
 464 class Stats:
 465     def __init__(self, providers, fields = None):
 466         self.providers = providers
 467         self.fields_filter = fields
 468         self._update()
 469     def _update(self):
 470         def wanted(key):
 471             import re
 472             if not self.fields_filter:
 473                 return True
 474             return re.match(self.fields_filter, key) is not None
 475         self.values = dict()
 476         for d in providers:
 477             provider_fields = [key for key in d.fields() if wanted(key)]
 478             for key in provider_fields:
 479                 self.values[key] = None
 480             d.select(provider_fields)
 481     def set_fields_filter(self, fields_filter):
 482         self.fields_filter = fields_filter
 483         self._update()
 484     def get(self):
 485         for d in providers:
 486             new = d.read()
 487             for key in d.fields():
 488                 oldval = self.values.get(key, (0, 0))
 489                 newval = new[key]
 490                 newdelta = None
 491                 if oldval is not None:
 492                     newdelta = newval - oldval[0]
 493                 self.values[key] = (newval, newdelta)
 494         return self.values
 495
 496 if not os.access('/sys/kernel/debug', os.F_OK):
 497     print 'Please enable CONFIG_DEBUG_FS in your kernel'
 498     sys.exit(1)
 499 if not os.access('/sys/kernel/debug/kvm', os.F_OK):
 500     print "Please mount debugfs ('mount -t debugfs debugfs /sys/kernel/debug')"
 501     print "and ensure the kvm modules are loaded"
 502     sys.exit(1)
 503
 504 label_width = 40
 505 number_width = 10
 506
 507 def tui(screen, stats):
 508     curses.use_default_colors()
 509     curses.noecho()
 510     drilldown = False
 511     fields_filter = stats.fields_filter
 512     def update_drilldown():
 513         if not fields_filter:
 514             if drilldown:
 515                 stats.set_fields_filter(None)
 516             else:
 517                 stats.set_fields_filter(r'^[^\(]*$')
 518     update_drilldown()
 519     def refresh(sleeptime):
 520         screen.erase()
 521         screen.addstr(0, 0, 'kvm statistics')
 522         screen.addstr(2, 1, 'Event')
 523         screen.addstr(2, 1 + label_width + number_width - len('Total'), 'Total')
 524         screen.addstr(2, 1 + label_width + number_width + 8 - len('Current'), 'Current')
 525         row = 3
 526         s = stats.get()
 527         def sortkey(x):
 528             if s[x][1]:
 529                 return (-s[x][1], -s[x][0])
 530             else:
 531                 return (0, -s[x][0])
 532         for key in sorted(s.keys(), key = sortkey):
 533             if row >= screen.getmaxyx()[0]:
 534                 break
 535             values = s[key]
 536             if not values[0] and not values[1]:
 537                 break
 538             col = 1
 539             screen.addstr(row, col, key)
 540             col += label_width
 541             screen.addstr(row, col, '%10d' % (values[0],))
 542             col += number_width
 543             if values[1] is not None:
 544                 screen.addstr(row, col, '%8d' % (values[1] / sleeptime,))
 545             row += 1
 546         screen.refresh()
 547
 548     sleeptime = 0.25
 549     while True:
 550         refresh(sleeptime)
 551         curses.halfdelay(int(sleeptime * 10))
 552         sleeptime = 3
 553         try:
 554             c = screen.getkey()
 555             if c == 'x':
 556                 drilldown = not drilldown
 557                 update_drilldown()
 558             if c == 'q':
 559                 break
 560         except KeyboardInterrupt:
 561             break
 562         except curses.error:
 563             continue
 564
 565 def batch(stats):
 566     s = stats.get()
 567     time.sleep(1)
 568     s = stats.get()
 569     for key in sorted(s.keys()):
 570         values = s[key]
 571         print '%-22s%10d%10d' % (key, values[0], values[1])
 572
 573 def log(stats):
 574     keys = sorted(stats.get().iterkeys())
 575     def banner():
 576         for k in keys:
 577             print '%10s' % k[0:9],
 578         print
 579     def statline():
 580         s = stats.get()
 581         for k in keys:
 582             print ' %9d' % s[k][1],
 583         print
 584     line = 0
 585     banner_repeat = 20
 586     while True:
 587         time.sleep(1)
 588         if line % banner_repeat == 0:
 589             banner()
 590         statline()
 591         line += 1
 592
 593 options = optparse.OptionParser()
 594 options.add_option('-1', '--once', '--batch',
 595                    action = 'store_true',
 596                    default = False,
 597                    dest = 'once',
 598                    help = 'run in batch mode for one second',
 599                    )
 600 options.add_option('-l', '--log',
 601                    action = 'store_true',
 602                    default = False,
 603                    dest = 'log',
 604                    help = 'run in logging mode (like vmstat)',
 605                    )
 606 options.add_option('-t', '--tracepoints',
 607                    action = 'store_true',
 608                    default = False,
 609                    dest = 'tracepoints',
 610                    help = 'retrieve statistics from tracepoints',
 611                    )
 612 options.add_option('-d', '--debugfs',
 613                    action = 'store_true',
 614                    default = False,
 615                    dest = 'debugfs',
 616                    help = 'retrieve statistics from debugfs',
 617                    )
 618 options.add_option('-f', '--fields',
 619                    action = 'store',
 620                    default = None,
 621                    dest = 'fields',
 622                    help = 'fields to display (regex)',
 623                    )
 624 (options, args) = options.parse_args(sys.argv)
 625
 626 providers = []
 627 if options.tracepoints:
 628     providers.append(TracepointProvider())
 629 if options.debugfs:
 630     providers.append(DebugfsProvider())
 631
 632 if len(providers) == 0:
 633     try:
 634         providers = [TracepointProvider()]
 635     except:
 636         providers = [DebugfsProvider()]
 637
 638 stats = Stats(providers, fields = options.fields)
 639
 640 if options.log:
 641     log(stats)
 642 elif not options.once:
 643     import curses.wrapper
 644     curses.wrapper(tui, stats)
 645 else:
 646     batch(stats)