scripts/tracetool/__init__.py

   1 # -*- coding: utf-8 -*-
   2
   3 """
   4 Machinery for generating tracing-related intermediate files.
   5 """
   6
   7 __author__     = "Lluís Vilanova <vilanova@ac.upc.edu>"
   8 __copyright__  = "Copyright 2012-2017, Lluís Vilanova <vilanova@ac.upc.edu>"
   9 __license__    = "GPL version 2 or (at your option) any later version"
  10
  11 __maintainer__ = "Stefan Hajnoczi"
  12 __email__      = "stefanha@redhat.com"
  13
  14
  15 import re
  16 import sys
  17 import weakref
  18
  19 import tracetool.format
  20 import tracetool.backend
  21 import tracetool.transform
  22
  23
  24 def error_write(*lines):
  25     """Write a set of error lines."""
  26     sys.stderr.writelines("\n".join(lines) + "\n")
  27
  28 def error(*lines):
  29     """Write a set of error lines and exit."""
  30     error_write(*lines)
  31     sys.exit(1)
  32
  33
  34 out_lineno = 1
  35 out_filename = '<none>'
  36 out_fobj = sys.stdout
  37
  38 def out_open(filename):
  39     global out_filename, out_fobj
  40     out_filename = filename
  41     out_fobj = open(filename, 'wt')
  42
  43 def out(*lines, **kwargs):
  44     """Write a set of output lines.
  45
  46     You can use kwargs as a shorthand for mapping variables when formatting all
  47     the strings in lines.
  48
  49     The 'out_lineno' kwarg is automatically added to reflect the current output
  50     file line number. The 'out_next_lineno' kwarg is also automatically added
  51     with the next output line number. The 'out_filename' kwarg is automatically
  52     added with the output filename.
  53     """
  54     global out_lineno
  55     output = []
  56     for l in lines:
  57         kwargs['out_lineno'] = out_lineno
  58         kwargs['out_next_lineno'] = out_lineno + 1
  59         kwargs['out_filename'] = out_filename
  60         output.append(l % kwargs)
  61         out_lineno += 1
  62
  63     out_fobj.writelines("\n".join(output) + "\n")
  64
  65 # We only want to allow standard C types or fixed sized
  66 # integer types. We don't want QEMU specific types
  67 # as we can't assume trace backends can resolve all the
  68 # typedefs
  69 ALLOWED_TYPES = [
  70     "int",
  71     "long",
  72     "short",
  73     "char",
  74     "bool",
  75     "unsigned",
  76     "signed",
  77     "int8_t",
  78     "uint8_t",
  79     "int16_t",
  80     "uint16_t",
  81     "int32_t",
  82     "uint32_t",
  83     "int64_t",
  84     "uint64_t",
  85     "void",
  86     "size_t",
  87     "ssize_t",
  88     "uintptr_t",
  89     "ptrdiff_t",
  90     # Magic substitution is done by tracetool
  91     "TCGv",
  92 ]
  93
  94 def validate_type(name):
  95     bits = name.split(" ")
  96     for bit in bits:
  97         bit = re.sub("\*", "", bit)
  98         if bit == "":
  99             continue
 100         if bit == "const":
 101             continue
 102         if bit not in ALLOWED_TYPES:
 103             raise ValueError("Argument type '%s' is not allowed. "
 104                              "Only standard C types and fixed size integer "
 105                              "types should be used. struct, union, and "
 106                              "other complex pointer types should be "
 107                              "declared as 'void *'" % name)
 108
 109 class Arguments:
 110     """Event arguments description."""
 111
 112     def __init__(self, args):
 113         """
 114         Parameters
 115         ----------
 116         args :
 117             List of (type, name) tuples or Arguments objects.
 118         """
 119         self._args = []
 120         for arg in args:
 121             if isinstance(arg, Arguments):
 122                 self._args.extend(arg._args)
 123             else:
 124                 self._args.append(arg)
 125
 126     def copy(self):
 127         """Create a new copy."""
 128         return Arguments(list(self._args))
 129
 130     @staticmethod
 131     def build(arg_str):
 132         """Build and Arguments instance from an argument string.
 133
 134         Parameters
 135         ----------
 136         arg_str : str
 137             String describing the event arguments.
 138         """
 139         res = []
 140         for arg in arg_str.split(","):
 141             arg = arg.strip()
 142             if not arg:
 143                 raise ValueError("Empty argument (did you forget to use 'void'?)")
 144             if arg == 'void':
 145                 continue
 146
 147             if '*' in arg:
 148                 arg_type, identifier = arg.rsplit('*', 1)
 149                 arg_type += '*'
 150                 identifier = identifier.strip()
 151             else:
 152                 arg_type, identifier = arg.rsplit(None, 1)
 153
 154             validate_type(arg_type)
 155             res.append((arg_type, identifier))
 156         return Arguments(res)
 157
 158     def __getitem__(self, index):
 159         if isinstance(index, slice):
 160             return Arguments(self._args[index])
 161         else:
 162             return self._args[index]
 163
 164     def __iter__(self):
 165         """Iterate over the (type, name) pairs."""
 166         return iter(self._args)
 167
 168     def __len__(self):
 169         """Number of arguments."""
 170         return len(self._args)
 171
 172     def __str__(self):
 173         """String suitable for declaring function arguments."""
 174         if len(self._args) == 0:
 175             return "void"
 176         else:
 177             return ", ".join([ " ".join([t, n]) for t,n in self._args ])
 178
 179     def __repr__(self):
 180         """Evaluable string representation for this object."""
 181         return "Arguments(\"%s\")" % str(self)
 182
 183     def names(self):
 184         """List of argument names."""
 185         return [ name for _, name in self._args ]
 186
 187     def types(self):
 188         """List of argument types."""
 189         return [ type_ for type_, _ in self._args ]
 190
 191     def casted(self):
 192         """List of argument names casted to their type."""
 193         return ["(%s)%s" % (type_, name) for type_, name in self._args]
 194
 195     def transform(self, *trans):
 196         """Return a new Arguments instance with transformed types.
 197
 198         The types in the resulting Arguments instance are transformed according
 199         to tracetool.transform.transform_type.
 200         """
 201         res = []
 202         for type_, name in self._args:
 203             res.append((tracetool.transform.transform_type(type_, *trans),
 204                         name))
 205         return Arguments(res)
 206
 207
 208 class Event(object):
 209     """Event description.
 210
 211     Attributes
 212     ----------
 213     name : str
 214         The event name.
 215     fmt : str
 216         The event format string.
 217     properties : set(str)
 218         Properties of the event.
 219     args : Arguments
 220         The event arguments.
 221     lineno : int
 222         The line number in the input file.
 223     filename : str
 224         The path to the input file.
 225
 226     """
 227
 228     _CRE = re.compile("((?P<props>[\w\s]+)\s+)?"
 229                       "(?P<name>\w+)"
 230                       "\((?P<args>[^)]*)\)"
 231                       "\s*"
 232                       "(?:(?:(?P<fmt_trans>\".+),)?\s*(?P<fmt>\".+))?"
 233                       "\s*")
 234
 235     _VALID_PROPS = set(["disable", "tcg", "tcg-trans", "tcg-exec", "vcpu"])
 236
 237     def __init__(self, name, props, fmt, args, lineno, filename, orig=None,
 238                  event_trans=None, event_exec=None):
 239         """
 240         Parameters
 241         ----------
 242         name : string
 243             Event name.
 244         props : list of str
 245             Property names.
 246         fmt : str, list of str
 247             Event printing format string(s).
 248         args : Arguments
 249             Event arguments.
 250         lineno : int
 251             The line number in the input file.
 252         filename : str
 253             The path to the input file.
 254         orig : Event or None
 255             Original Event before transformation/generation.
 256         event_trans : Event or None
 257             Generated translation-time event ("tcg" property).
 258         event_exec : Event or None
 259             Generated execution-time event ("tcg" property).
 260
 261         """
 262         self.name = name
 263         self.properties = props
 264         self.fmt = fmt
 265         self.args = args
 266         self.lineno = int(lineno)
 267         self.filename = str(filename)
 268         self.event_trans = event_trans
 269         self.event_exec = event_exec
 270
 271         if len(args) > 10:
 272             raise ValueError("Event '%s' has more than maximum permitted "
 273                              "argument count" % name)
 274
 275         if orig is None:
 276             self.original = weakref.ref(self)
 277         else:
 278             self.original = orig
 279
 280         unknown_props = set(self.properties) - self._VALID_PROPS
 281         if len(unknown_props) > 0:
 282             raise ValueError("Unknown properties: %s"
 283                              % ", ".join(unknown_props))
 284         assert isinstance(self.fmt, str) or len(self.fmt) == 2
 285
 286     def copy(self):
 287         """Create a new copy."""
 288         return Event(self.name, list(self.properties), self.fmt,
 289                      self.args.copy(), self.lineno, self.filename,
 290                      self, self.event_trans, self.event_exec)
 291
 292     @staticmethod
 293     def build(line_str, lineno, filename):
 294         """Build an Event instance from a string.
 295
 296         Parameters
 297         ----------
 298         line_str : str
 299             Line describing the event.
 300         lineno : int
 301             Line number in input file.
 302         filename : str
 303             Path to input file.
 304         """
 305         m = Event._CRE.match(line_str)
 306         assert m is not None
 307         groups = m.groupdict('')
 308
 309         name = groups["name"]
 310         props = groups["props"].split()
 311         fmt = groups["fmt"]
 312         fmt_trans = groups["fmt_trans"]
 313         if fmt.find("%m") != -1 or fmt_trans.find("%m") != -1:
 314             raise ValueError("Event format '%m' is forbidden, pass the error "
 315                              "as an explicit trace argument")
 316         if fmt.endswith(r'\n"'):
 317             raise ValueError("Event format must not end with a newline "
 318                              "character")
 319
 320         if len(fmt_trans) > 0:
 321             fmt = [fmt_trans, fmt]
 322         args = Arguments.build(groups["args"])
 323
 324         if "tcg-trans" in props:
 325             raise ValueError("Invalid property 'tcg-trans'")
 326         if "tcg-exec" in props:
 327             raise ValueError("Invalid property 'tcg-exec'")
 328         if "tcg" not in props and not isinstance(fmt, str):
 329             raise ValueError("Only events with 'tcg' property can have two format strings")
 330         if "tcg" in props and isinstance(fmt, str):
 331             raise ValueError("Events with 'tcg' property must have two format strings")
 332
 333         event = Event(name, props, fmt, args, lineno, filename)
 334
 335         # add implicit arguments when using the 'vcpu' property
 336         import tracetool.vcpu
 337         event = tracetool.vcpu.transform_event(event)
 338
 339         return event
 340
 341     def __repr__(self):
 342         """Evaluable string representation for this object."""
 343         if isinstance(self.fmt, str):
 344             fmt = self.fmt
 345         else:
 346             fmt = "%s, %s" % (self.fmt[0], self.fmt[1])
 347         return "Event('%s %s(%s) %s')" % (" ".join(self.properties),
 348                                           self.name,
 349                                           self.args,
 350                                           fmt)
 351     # Star matching on PRI is dangerous as one might have multiple
 352     # arguments with that format, hence the non-greedy version of it.
 353     _FMT = re.compile("(%[\d\.]*\w+|%.*?PRI\S+)")
 354
 355     def formats(self):
 356         """List conversion specifiers in the argument print format string."""
 357         assert not isinstance(self.fmt, list)
 358         return self._FMT.findall(self.fmt)
 359
 360     QEMU_TRACE               = "trace_%(name)s"
 361     QEMU_TRACE_NOCHECK       = "_nocheck__" + QEMU_TRACE
 362     QEMU_TRACE_TCG           = QEMU_TRACE + "_tcg"
 363     QEMU_DSTATE              = "_TRACE_%(NAME)s_DSTATE"
 364     QEMU_BACKEND_DSTATE      = "TRACE_%(NAME)s_BACKEND_DSTATE"
 365     QEMU_EVENT               = "_TRACE_%(NAME)s_EVENT"
 366
 367     def api(self, fmt=None):
 368         if fmt is None:
 369             fmt = Event.QEMU_TRACE
 370         return fmt % {"name": self.name, "NAME": self.name.upper()}
 371
 372     def transform(self, *trans):
 373         """Return a new Event with transformed Arguments."""
 374         return Event(self.name,
 375                      list(self.properties),
 376                      self.fmt,
 377                      self.args.transform(*trans),
 378                      self.lineno,
 379                      self.filename,
 380                      self)
 381
 382
 383 def read_events(fobj, fname):
 384     """Generate the output for the given (format, backends) pair.
 385
 386     Parameters
 387     ----------
 388     fobj : file
 389         Event description file.
 390     fname : str
 391         Name of event file
 392
 393     Returns a list of Event objects
 394     """
 395
 396     events = []
 397     for lineno, line in enumerate(fobj, 1):
 398         if line[-1] != '\n':
 399             raise ValueError("%s does not end with a new line" % fname)
 400         if not line.strip():
 401             continue
 402         if line.lstrip().startswith('#'):
 403             continue
 404
 405         try:
 406             event = Event.build(line, lineno, fname)
 407         except ValueError as e:
 408             arg0 = 'Error at %s:%d: %s' % (fname, lineno, e.args[0])
 409             e.args = (arg0,) + e.args[1:]
 410             raise
 411
 412         # transform TCG-enabled events
 413         if "tcg" not in event.properties:
 414             events.append(event)
 415         else:
 416             event_trans = event.copy()
 417             event_trans.name += "_trans"
 418             event_trans.properties += ["tcg-trans"]
 419             event_trans.fmt = event.fmt[0]
 420             # ignore TCG arguments
 421             args_trans = []
 422             for atrans, aorig in zip(
 423                     event_trans.transform(tracetool.transform.TCG_2_HOST).args,
 424                     event.args):
 425                 if atrans == aorig:
 426                     args_trans.append(atrans)
 427             event_trans.args = Arguments(args_trans)
 428
 429             event_exec = event.copy()
 430             event_exec.name += "_exec"
 431             event_exec.properties += ["tcg-exec"]
 432             event_exec.fmt = event.fmt[1]
 433             event_exec.args = event_exec.args.transform(tracetool.transform.TCG_2_HOST)
 434
 435             new_event = [event_trans, event_exec]
 436             event.event_trans, event.event_exec = new_event
 437
 438             events.extend(new_event)
 439
 440     return events
 441
 442
 443 class TracetoolError (Exception):
 444     """Exception for calls to generate."""
 445     pass
 446
 447
 448 def try_import(mod_name, attr_name=None, attr_default=None):
 449     """Try to import a module and get an attribute from it.
 450
 451     Parameters
 452     ----------
 453     mod_name : str
 454         Module name.
 455     attr_name : str, optional
 456         Name of an attribute in the module.
 457     attr_default : optional
 458         Default value if the attribute does not exist in the module.
 459
 460     Returns
 461     -------
 462     A pair indicating whether the module could be imported and the module or
 463     object or attribute value.
 464     """
 465     try:
 466         module = __import__(mod_name, globals(), locals(), ["__package__"])
 467         if attr_name is None:
 468             return True, module
 469         return True, getattr(module, str(attr_name), attr_default)
 470     except ImportError:
 471         return False, None
 472
 473
 474 def generate(events, group, format, backends,
 475              binary=None, probe_prefix=None):
 476     """Generate the output for the given (format, backends) pair.
 477
 478     Parameters
 479     ----------
 480     events : list
 481         list of Event objects to generate for
 482     group: str
 483         Name of the tracing group
 484     format : str
 485         Output format name.
 486     backends : list
 487         Output backend names.
 488     binary : str or None
 489         See tracetool.backend.dtrace.BINARY.
 490     probe_prefix : str or None
 491         See tracetool.backend.dtrace.PROBEPREFIX.
 492     """
 493     # fix strange python error (UnboundLocalError tracetool)
 494     import tracetool
 495
 496     format = str(format)
 497     if len(format) == 0:
 498         raise TracetoolError("format not set")
 499     if not tracetool.format.exists(format):
 500         raise TracetoolError("unknown format: %s" % format)
 501
 502     if len(backends) == 0:
 503         raise TracetoolError("no backends specified")
 504     for backend in backends:
 505         if not tracetool.backend.exists(backend):
 506             raise TracetoolError("unknown backend: %s" % backend)
 507     backend = tracetool.backend.Wrapper(backends, format)
 508
 509     import tracetool.backend.dtrace
 510     tracetool.backend.dtrace.BINARY = binary
 511     tracetool.backend.dtrace.PROBEPREFIX = probe_prefix
 512
 513     tracetool.format.generate(events, format, backend, group)