scripts/tracetool/__init__.py

   1 # -*- coding: utf-8 -*-
   2
   3 """
   4 Machinery for generating tracing-related intermediate files.
   5 """
   6
   7 __author__     = "Lluís Vilanova <vilanova@ac.upc.edu>"
   8 __copyright__  = "Copyright 2012-2017, Lluís Vilanova <vilanova@ac.upc.edu>"
   9 __license__    = "GPL version 2 or (at your option) any later version"
  10
  11 __maintainer__ = "Stefan Hajnoczi"
  12 __email__      = "stefanha@linux.vnet.ibm.com"
  13
  14
  15 import re
  16 import sys
  17 import weakref
  18
  19 import tracetool.format
  20 import tracetool.backend
  21 import tracetool.transform
  22
  23
  24 def error_write(*lines):
  25     """Write a set of error lines."""
  26     sys.stderr.writelines("\n".join(lines) + "\n")
  27
  28 def error(*lines):
  29     """Write a set of error lines and exit."""
  30     error_write(*lines)
  31     sys.exit(1)
  32
  33
  34 def out(*lines, **kwargs):
  35     """Write a set of output lines.
  36
  37     You can use kwargs as a shorthand for mapping variables when formating all
  38     the strings in lines.
  39     """
  40     lines = [ l % kwargs for l in lines ]
  41     sys.stdout.writelines("\n".join(lines) + "\n")
  42
  43 # We only want to allow standard C types or fixed sized
  44 # integer types. We don't want QEMU specific types
  45 # as we can't assume trace backends can resolve all the
  46 # typedefs
  47 ALLOWED_TYPES = [
  48     "int",
  49     "long",
  50     "short",
  51     "char",
  52     "bool",
  53     "unsigned",
  54     "signed",
  55     "int8_t",
  56     "uint8_t",
  57     "int16_t",
  58     "uint16_t",
  59     "int32_t",
  60     "uint32_t",
  61     "int64_t",
  62     "uint64_t",
  63     "void",
  64     "size_t",
  65     "ssize_t",
  66     "uintptr_t",
  67     "ptrdiff_t",
  68     # Magic substitution is done by tracetool
  69     "TCGv",
  70 ]
  71
  72 def validate_type(name):
  73     bits = name.split(" ")
  74     for bit in bits:
  75         bit = re.sub("\*", "", bit)
  76         if bit == "":
  77             continue
  78         if bit == "const":
  79             continue
  80         if bit not in ALLOWED_TYPES:
  81             raise ValueError("Argument type '%s' is not in whitelist. "
  82                              "Only standard C types and fixed size integer "
  83                              "types should be used. struct, union, and "
  84                              "other complex pointer types should be "
  85                              "declared as 'void *'" % name)
  86
  87 class Arguments:
  88     """Event arguments description."""
  89
  90     def __init__(self, args):
  91         """
  92         Parameters
  93         ----------
  94         args :
  95             List of (type, name) tuples or Arguments objects.
  96         """
  97         self._args = []
  98         for arg in args:
  99             if isinstance(arg, Arguments):
 100                 self._args.extend(arg._args)
 101             else:
 102                 self._args.append(arg)
 103
 104     def copy(self):
 105         """Create a new copy."""
 106         return Arguments(list(self._args))
 107
 108     @staticmethod
 109     def build(arg_str):
 110         """Build and Arguments instance from an argument string.
 111
 112         Parameters
 113         ----------
 114         arg_str : str
 115             String describing the event arguments.
 116         """
 117         res = []
 118         for arg in arg_str.split(","):
 119             arg = arg.strip()
 120             if not arg:
 121                 raise ValueError("Empty argument (did you forget to use 'void'?)")
 122             if arg == 'void':
 123                 continue
 124
 125             if '*' in arg:
 126                 arg_type, identifier = arg.rsplit('*', 1)
 127                 arg_type += '*'
 128                 identifier = identifier.strip()
 129             else:
 130                 arg_type, identifier = arg.rsplit(None, 1)
 131
 132             validate_type(arg_type)
 133             res.append((arg_type, identifier))
 134         return Arguments(res)
 135
 136     def __getitem__(self, index):
 137         if isinstance(index, slice):
 138             return Arguments(self._args[index])
 139         else:
 140             return self._args[index]
 141
 142     def __iter__(self):
 143         """Iterate over the (type, name) pairs."""
 144         return iter(self._args)
 145
 146     def __len__(self):
 147         """Number of arguments."""
 148         return len(self._args)
 149
 150     def __str__(self):
 151         """String suitable for declaring function arguments."""
 152         if len(self._args) == 0:
 153             return "void"
 154         else:
 155             return ", ".join([ " ".join([t, n]) for t,n in self._args ])
 156
 157     def __repr__(self):
 158         """Evaluable string representation for this object."""
 159         return "Arguments(\"%s\")" % str(self)
 160
 161     def names(self):
 162         """List of argument names."""
 163         return [ name for _, name in self._args ]
 164
 165     def types(self):
 166         """List of argument types."""
 167         return [ type_ for type_, _ in self._args ]
 168
 169     def casted(self):
 170         """List of argument names casted to their type."""
 171         return ["(%s)%s" % (type_, name) for type_, name in self._args]
 172
 173     def transform(self, *trans):
 174         """Return a new Arguments instance with transformed types.
 175
 176         The types in the resulting Arguments instance are transformed according
 177         to tracetool.transform.transform_type.
 178         """
 179         res = []
 180         for type_, name in self._args:
 181             res.append((tracetool.transform.transform_type(type_, *trans),
 182                         name))
 183         return Arguments(res)
 184
 185
 186 class Event(object):
 187     """Event description.
 188
 189     Attributes
 190     ----------
 191     name : str
 192         The event name.
 193     fmt : str
 194         The event format string.
 195     properties : set(str)
 196         Properties of the event.
 197     args : Arguments
 198         The event arguments.
 199
 200     """
 201
 202     _CRE = re.compile("((?P<props>[\w\s]+)\s+)?"
 203                       "(?P<name>\w+)"
 204                       "\((?P<args>[^)]*)\)"
 205                       "\s*"
 206                       "(?:(?:(?P<fmt_trans>\".+),)?\s*(?P<fmt>\".+))?"
 207                       "\s*")
 208
 209     _VALID_PROPS = set(["disable", "tcg", "tcg-trans", "tcg-exec", "vcpu"])
 210
 211     def __init__(self, name, props, fmt, args, orig=None,
 212                  event_trans=None, event_exec=None):
 213         """
 214         Parameters
 215         ----------
 216         name : string
 217             Event name.
 218         props : list of str
 219             Property names.
 220         fmt : str, list of str
 221             Event printing format string(s).
 222         args : Arguments
 223             Event arguments.
 224         orig : Event or None
 225             Original Event before transformation/generation.
 226         event_trans : Event or None
 227             Generated translation-time event ("tcg" property).
 228         event_exec : Event or None
 229             Generated execution-time event ("tcg" property).
 230
 231         """
 232         self.name = name
 233         self.properties = props
 234         self.fmt = fmt
 235         self.args = args
 236         self.event_trans = event_trans
 237         self.event_exec = event_exec
 238
 239         if len(args) > 10:
 240             raise ValueError("Event '%s' has more than maximum permitted "
 241                              "argument count" % name)
 242
 243         if orig is None:
 244             self.original = weakref.ref(self)
 245         else:
 246             self.original = orig
 247
 248         unknown_props = set(self.properties) - self._VALID_PROPS
 249         if len(unknown_props) > 0:
 250             raise ValueError("Unknown properties: %s"
 251                              % ", ".join(unknown_props))
 252         assert isinstance(self.fmt, str) or len(self.fmt) == 2
 253
 254     def copy(self):
 255         """Create a new copy."""
 256         return Event(self.name, list(self.properties), self.fmt,
 257                      self.args.copy(), self, self.event_trans, self.event_exec)
 258
 259     @staticmethod
 260     def build(line_str):
 261         """Build an Event instance from a string.
 262
 263         Parameters
 264         ----------
 265         line_str : str
 266             Line describing the event.
 267         """
 268         m = Event._CRE.match(line_str)
 269         assert m is not None
 270         groups = m.groupdict('')
 271
 272         name = groups["name"]
 273         props = groups["props"].split()
 274         fmt = groups["fmt"]
 275         fmt_trans = groups["fmt_trans"]
 276         if fmt.find("%m") != -1 or fmt_trans.find("%m") != -1:
 277             raise ValueError("Event format '%m' is forbidden, pass the error "
 278                              "as an explicit trace argument")
 279         if fmt.endswith(r'\n"'):
 280             raise ValueError("Event format must not end with a newline "
 281                              "character")
 282
 283         if len(fmt_trans) > 0:
 284             fmt = [fmt_trans, fmt]
 285         args = Arguments.build(groups["args"])
 286
 287         if "tcg-trans" in props:
 288             raise ValueError("Invalid property 'tcg-trans'")
 289         if "tcg-exec" in props:
 290             raise ValueError("Invalid property 'tcg-exec'")
 291         if "tcg" not in props and not isinstance(fmt, str):
 292             raise ValueError("Only events with 'tcg' property can have two format strings")
 293         if "tcg" in props and isinstance(fmt, str):
 294             raise ValueError("Events with 'tcg' property must have two format strings")
 295
 296         event = Event(name, props, fmt, args)
 297
 298         # add implicit arguments when using the 'vcpu' property
 299         import tracetool.vcpu
 300         event = tracetool.vcpu.transform_event(event)
 301
 302         return event
 303
 304     def __repr__(self):
 305         """Evaluable string representation for this object."""
 306         if isinstance(self.fmt, str):
 307             fmt = self.fmt
 308         else:
 309             fmt = "%s, %s" % (self.fmt[0], self.fmt[1])
 310         return "Event('%s %s(%s) %s')" % (" ".join(self.properties),
 311                                           self.name,
 312                                           self.args,
 313                                           fmt)
 314     # Star matching on PRI is dangerous as one might have multiple
 315     # arguments with that format, hence the non-greedy version of it.
 316     _FMT = re.compile("(%[\d\.]*\w+|%.*?PRI\S+)")
 317
 318     def formats(self):
 319         """List conversion specifiers in the argument print format string."""
 320         assert not isinstance(self.fmt, list)
 321         return self._FMT.findall(self.fmt)
 322
 323     QEMU_TRACE               = "trace_%(name)s"
 324     QEMU_TRACE_NOCHECK       = "_nocheck__" + QEMU_TRACE
 325     QEMU_TRACE_TCG           = QEMU_TRACE + "_tcg"
 326     QEMU_DSTATE              = "_TRACE_%(NAME)s_DSTATE"
 327     QEMU_BACKEND_DSTATE      = "TRACE_%(NAME)s_BACKEND_DSTATE"
 328     QEMU_EVENT               = "_TRACE_%(NAME)s_EVENT"
 329
 330     def api(self, fmt=None):
 331         if fmt is None:
 332             fmt = Event.QEMU_TRACE
 333         return fmt % {"name": self.name, "NAME": self.name.upper()}
 334
 335     def transform(self, *trans):
 336         """Return a new Event with transformed Arguments."""
 337         return Event(self.name,
 338                      list(self.properties),
 339                      self.fmt,
 340                      self.args.transform(*trans),
 341                      self)
 342
 343
 344 def read_events(fobj, fname):
 345     """Generate the output for the given (format, backends) pair.
 346
 347     Parameters
 348     ----------
 349     fobj : file
 350         Event description file.
 351     fname : str
 352         Name of event file
 353
 354     Returns a list of Event objects
 355     """
 356
 357     events = []
 358     for lineno, line in enumerate(fobj, 1):
 359         if line[-1] != '\n':
 360             raise ValueError("%s does not end with a new line" % fname)
 361         if not line.strip():
 362             continue
 363         if line.lstrip().startswith('#'):
 364             continue
 365
 366         try:
 367             event = Event.build(line)
 368         except ValueError as e:
 369             arg0 = 'Error at %s:%d: %s' % (fname, lineno, e.args[0])
 370             e.args = (arg0,) + e.args[1:]
 371             raise
 372
 373         # transform TCG-enabled events
 374         if "tcg" not in event.properties:
 375             events.append(event)
 376         else:
 377             event_trans = event.copy()
 378             event_trans.name += "_trans"
 379             event_trans.properties += ["tcg-trans"]
 380             event_trans.fmt = event.fmt[0]
 381             # ignore TCG arguments
 382             args_trans = []
 383             for atrans, aorig in zip(
 384                     event_trans.transform(tracetool.transform.TCG_2_HOST).args,
 385                     event.args):
 386                 if atrans == aorig:
 387                     args_trans.append(atrans)
 388             event_trans.args = Arguments(args_trans)
 389
 390             event_exec = event.copy()
 391             event_exec.name += "_exec"
 392             event_exec.properties += ["tcg-exec"]
 393             event_exec.fmt = event.fmt[1]
 394             event_exec.args = event_exec.args.transform(tracetool.transform.TCG_2_HOST)
 395
 396             new_event = [event_trans, event_exec]
 397             event.event_trans, event.event_exec = new_event
 398
 399             events.extend(new_event)
 400
 401     return events
 402
 403
 404 class TracetoolError (Exception):
 405     """Exception for calls to generate."""
 406     pass
 407
 408
 409 def try_import(mod_name, attr_name=None, attr_default=None):
 410     """Try to import a module and get an attribute from it.
 411
 412     Parameters
 413     ----------
 414     mod_name : str
 415         Module name.
 416     attr_name : str, optional
 417         Name of an attribute in the module.
 418     attr_default : optional
 419         Default value if the attribute does not exist in the module.
 420
 421     Returns
 422     -------
 423     A pair indicating whether the module could be imported and the module or
 424     object or attribute value.
 425     """
 426     try:
 427         module = __import__(mod_name, globals(), locals(), ["__package__"])
 428         if attr_name is None:
 429             return True, module
 430         return True, getattr(module, str(attr_name), attr_default)
 431     except ImportError:
 432         return False, None
 433
 434
 435 def generate(events, group, format, backends,
 436              binary=None, probe_prefix=None):
 437     """Generate the output for the given (format, backends) pair.
 438
 439     Parameters
 440     ----------
 441     events : list
 442         list of Event objects to generate for
 443     group: str
 444         Name of the tracing group
 445     format : str
 446         Output format name.
 447     backends : list
 448         Output backend names.
 449     binary : str or None
 450         See tracetool.backend.dtrace.BINARY.
 451     probe_prefix : str or None
 452         See tracetool.backend.dtrace.PROBEPREFIX.
 453     """
 454     # fix strange python error (UnboundLocalError tracetool)
 455     import tracetool
 456
 457     format = str(format)
 458     if len(format) == 0:
 459         raise TracetoolError("format not set")
 460     if not tracetool.format.exists(format):
 461         raise TracetoolError("unknown format: %s" % format)
 462
 463     if len(backends) == 0:
 464         raise TracetoolError("no backends specified")
 465     for backend in backends:
 466         if not tracetool.backend.exists(backend):
 467             raise TracetoolError("unknown backend: %s" % backend)
 468     backend = tracetool.backend.Wrapper(backends, format)
 469
 470     import tracetool.backend.dtrace
 471     tracetool.backend.dtrace.BINARY = binary
 472     tracetool.backend.dtrace.PROBEPREFIX = probe_prefix
 473
 474     tracetool.format.generate(events, format, backend, group)