ppapi/generate_ppapi_size_checks.py

   1 #!/usr/bin/python
   2
   3 # Copyright (c) 2010 The Chromium Authors. All rights reserved.
   4 # Use of this source code is governed by a BSD-style license that can be
   5 # found in the LICENSE file.
   6
   7 """This script should be run manually on occasion to make sure all PPAPI types
   8 have appropriate size checking.
   9
  10 """
  11
  12 import optparse
  13 import os
  14 import subprocess
  15 import sys
  16
  17
  18 # The string that the PrintNamesAndSizes plugin uses to indicate a type is
  19 # expected to have architecture-dependent size.
  20 ARCH_DEPENDENT_STRING = "ArchDependentSize"
  21
  22
  23
  24 class SourceLocation:
  25
  26   """A class representing the source location of a definiton."""
  27
  28   def __init__(self, filename="", start_line=-1, end_line=-1):
  29     self.filename = os.path.normpath(filename)
  30     self.start_line = start_line
  31     self.end_line = end_line
  32
  33
  34
  35 class TypeInfo:
  36
  37   """A class representing information about a C++ type.  It contains the
  38   following fields:
  39    - kind:  The Clang TypeClassName (Record, Enum, Typedef, Union, etc)
  40    - name:  The unmangled string name of the type.
  41    - size:  The size in bytes of the type.
  42    - arch_dependent:  True if the type may have architecture dependent size
  43                       according to PrintNamesAndSizes.  False otherwise.  Types
  44                       which are considered architecture-dependent from 32-bit
  45                       to 64-bit are pointers, longs, unsigned longs, and any
  46                       type that contains an architecture-dependent type.
  47    - source_location:  A SourceLocation describing where the type is defined.
  48    - target:  The target Clang was compiling when it found the type definition.
  49               This is used only for diagnostic output.
  50    - parsed_line:  The line which Clang output which was used to create this
  51                    TypeInfo (as the info_string parameter to __init__).  This is
  52                    used only for diagnostic output.
  53   """
  54
  55   def __init__(self, info_string, target):
  56     """Create a TypeInfo from a given info_string.  Also store the name of the
  57     target for which the TypeInfo was first created just so we can print useful
  58     error information.
  59     info_string is a comma-delimited string of the following form:
  60     kind,name,size,arch_dependent,source_file,start_line,end_line
  61     Where:
  62    - kind:  The Clang TypeClassName (Record, Enum, Typedef, Union, etc)
  63    - name:  The unmangled string name of the type.
  64    - size:  The size in bytes of the type.
  65    - arch_dependent:  'ArchDependentSize' if the type has architecture-dependent
  66                       size, NotArchDependentSize otherwise.
  67    - source_file:  The source file in which the type is defined.
  68    - first_line:  The first line of the definition (counting from 0).
  69    - last_line:  The last line of the definition (counting from 0).
  70    This should match the output of the PrintNamesAndSizes plugin.
  71    """
  72     [self.kind, self.name, self.size, arch_dependent_string, source_file,
  73         start_line, end_line] = info_string.split(',')
  74     self.target = target
  75     self.parsed_line = info_string
  76     # Note that Clang counts line numbers from 1, but we want to count from 0.
  77     self.source_location = SourceLocation(source_file,
  78                                           int(start_line)-1,
  79                                           int(end_line)-1)
  80     self.arch_dependent = (arch_dependent_string == ARCH_DEPENDENT_STRING)
  81
  82
  83 class FilePatch:
  84
  85   """A class representing a set of line-by-line changes to a particular file.
  86   None of the changes are applied until Apply is called.  All line numbers are
  87   counted from 0.
  88   """
  89
  90   def __init__(self, filename):
  91     self.filename = filename
  92     self.linenums_to_delete = set()
  93     # A dictionary from line number to an array of strings to be inserted at
  94     # that line number.
  95     self.lines_to_add = {}
  96
  97   def Delete(self, start_line, end_line):
  98     """Make the patch delete the lines starting with |start_line| up to but not
  99     including |end_line|.
 100     """
 101     self.linenums_to_delete |= set(range(start_line, end_line))
 102
 103   def Add(self, text, line_number):
 104     """Add the given text before the text on the given line number."""
 105     if line_number in self.lines_to_add:
 106       self.lines_to_add[line_number].append(text)
 107     else:
 108       self.lines_to_add[line_number] = [text]
 109
 110   def Apply(self):
 111     """Apply the patch by writing it to self.filename."""
 112     # Read the lines of the existing file in to a list.
 113     sourcefile = open(self.filename, "r")
 114     file_lines = sourcefile.readlines()
 115     sourcefile.close()
 116     # Now apply the patch.  Our strategy is to keep the array at the same size,
 117     # and just edit strings in the file_lines list as necessary.  When we delete
 118     # lines, we just blank the line and keep it in the list.  When we add lines,
 119     # we just prepend the added source code to the start of the existing line at
 120     # that line number.  This way, all the line numbers we cached from calls to
 121     # Add and Delete remain valid list indices, and we don't have to worry about
 122     # maintaining any offsets.  Each element of file_lines at the end may
 123     # contain any number of lines (0 or more) delimited by carriage returns.
 124     for linenum_to_delete in self.linenums_to_delete:
 125       file_lines[linenum_to_delete] = "";
 126     for linenum, sourcelines in self.lines_to_add.items():
 127       # Sort the lines we're adding so we get relatively consistent results.
 128       sourcelines.sort()
 129       # Prepend the new lines.  When we output
 130       file_lines[linenum] = "".join(sourcelines) + file_lines[linenum]
 131     newsource = open(self.filename, "w")
 132     for line in file_lines:
 133       newsource.write(line)
 134     newsource.close()
 135
 136
 137 def CheckAndInsert(typeinfo, typeinfo_map):
 138   """Check if a TypeInfo exists already in the given map with the same name.  If
 139   so, make sure the size is consistent.
 140   - If the name exists but the sizes do not match, print a message and
 141     exit with non-zero exit code.
 142   - If the name exists and the sizes match, do nothing.
 143   - If the name does not exist, insert the typeinfo in to the map.
 144
 145   """
 146   # If the type is unnamed, ignore it.
 147   if typeinfo.name == "":
 148     return
 149   # If the size is 0, ignore it.
 150   elif int(typeinfo.size) == 0:
 151     return
 152   # If the type is not defined under ppapi, ignore it.
 153   elif typeinfo.source_location.filename.find("ppapi") == -1:
 154     return
 155   # If the type is defined under GLES2, ignore it.
 156   elif typeinfo.source_location.filename.find("GLES2") > -1:
 157     return
 158   # If the type is an interface (by convention, starts with PPP_ or PPB_),
 159   # ignore it.
 160   elif (typeinfo.name[:4] == "PPP_") or (typeinfo.name[:4] == "PPB_"):
 161     return
 162   elif typeinfo.name in typeinfo_map:
 163     if typeinfo.size != typeinfo_map[typeinfo.name].size:
 164       print "Error: '" + typeinfo.name + "' is", \
 165           typeinfo_map[typeinfo.name].size, \
 166           "bytes on target '" + typeinfo_map[typeinfo.name].target + \
 167           "', but", typeinfo.size, "on target '" + typeinfo.target + "'"
 168       print typeinfo_map[typeinfo.name].parsed_line
 169       print typeinfo.parsed_line
 170       sys.exit(1)
 171     else:
 172       # It's already in the map and the sizes match.
 173       pass
 174   else:
 175     typeinfo_map[typeinfo.name] = typeinfo
 176
 177
 178 def ProcessTarget(clang_command, target, types):
 179   """Run clang using the given clang_command for the given target string.  Parse
 180   the output to create TypeInfos for each discovered type.  Insert each type in
 181   to the 'types' dictionary.  If the type already exists in the types
 182   dictionary, make sure that the size matches what's already in the map.  If
 183   not, exit with an error message.
 184   """
 185   p = subprocess.Popen(clang_command + " -triple " + target,
 186                        shell=True,
 187                        stdout=subprocess.PIPE)
 188   lines = p.communicate()[0].split()
 189   for line in lines:
 190     typeinfo = TypeInfo(line, target)
 191     CheckAndInsert(typeinfo, types)
 192
 193
 194 def ToAssertionCode(typeinfo):
 195   """Convert the TypeInfo to an appropriate C compile assertion.
 196   If it's a struct (Record in Clang terminology), we want a line like this:
 197     PP_COMPILE_ASSERT_STRUCT_SIZE_IN_BYTES(<name>, <size>);\n
 198   Enums:
 199     PP_COMPILE_ASSERT_ENUM_SIZE_IN_BYTES(<name>, <size>);\n
 200   Typedefs:
 201     PP_COMPILE_ASSERT_SIZE_IN_BYTES(<name>, <size>);\n
 202
 203   """
 204   line = "PP_COMPILE_ASSERT_"
 205   if typeinfo.kind == "Enum":
 206     line += "ENUM_"
 207   elif typeinfo.kind == "Record":
 208     line += "STRUCT_"
 209   line += "SIZE_IN_BYTES("
 210   line += typeinfo.name
 211   line += ", "
 212   line += typeinfo.size
 213   line += ");\n"
 214   return line
 215
 216
 217 def IsMacroDefinedName(typename):
 218   """Return true iff the given typename came from a PPAPI compile assertion."""
 219   return typename.find("PP_Dummy_Struct_For_") == 0
 220
 221
 222 COPYRIGHT_STRING_C = \
 223 """/* Copyright (c) 2010 The Chromium Authors. All rights reserved.
 224  * Use of this source code is governed by a BSD-style license that can be
 225  * found in the LICENSE file.
 226  *
 227  * This file has compile assertions for the sizes of types that are dependent
 228  * on the architecture for which they are compiled (i.e., 32-bit vs 64-bit).
 229  */
 230
 231 """
 232
 233
 234 def WriteArchSpecificCode(types, root, filename):
 235   """Write a header file that contains a compile-time assertion for the size of
 236      each of the given typeinfos, in to a file named filename rooted at root.
 237   """
 238   assertion_lines = [ToAssertionCode(typeinfo) for typeinfo in types]
 239   assertion_lines.sort()
 240   outfile = open(os.path.join(root, filename), "w")
 241   header_guard = "PPAPI_TESTS_" + filename.upper().replace(".", "_") + "_"
 242   outfile.write(COPYRIGHT_STRING_C)
 243   outfile.write('#ifndef ' + header_guard + '\n')
 244   outfile.write('#define ' + header_guard + '\n\n')
 245   outfile.write('#include "ppapi/tests/test_struct_sizes.c"\n\n')
 246   for line in assertion_lines:
 247     outfile.write(line)
 248   outfile.write('\n#endif  /* ' + header_guard + ' */\n')
 249
 250
 251 def main(argv):
 252   # See README file for example command-line invocation.  This script runs the
 253   # PrintNamesAndSizes Clang plugin with 'test_struct_sizes.c' as input, which
 254   # should include all C headers and all existing size checks.  It runs the
 255   # plugin multiple times;  once for each of a set of targets, some 32-bit and
 256   # some 64-bit.  It verifies that wherever possible, types have a consistent
 257   # size on both platforms.  Types that can't easily have consistent size (e.g.
 258   # ones that contain a pointer) are checked to make sure they are consistent
 259   # for all 32-bit platforms and consistent on all 64-bit platforms, but the
 260   # sizes on 32 vs 64 are allowed to differ.
 261   #
 262   # Then, if all the types have consistent size as expected, compile assertions
 263   # are added to the source code.  Types whose size is independent of
 264   # architectureacross have their compile assertions placed immediately after
 265   # their definition in the C API header.  Types whose size differs on 32-bit
 266   # vs 64-bit have a compile assertion placed in each of:
 267   # ppapi/tests/arch_dependent_sizes_32.h and
 268   # ppapi/tests/arch_dependent_sizes_64.h.
 269   #
 270   # Note that you should always check the results of the tool to make sure
 271   # they are sane.
 272   parser = optparse.OptionParser()
 273   parser.add_option(
 274       '-c', '--clang-path', dest='clang_path',
 275       default=(''),
 276       help='the path to the clang binary (default is to get it from your path)')
 277   parser.add_option(
 278       '-p', '--plugin', dest='plugin',
 279       default='tests/clang/libPrintNamesAndSizes.so',
 280       help='The path to the PrintNamesAndSizes plugin library.')
 281   parser.add_option(
 282       '--targets32', dest='targets32',
 283       default='i386-pc-linux,arm-pc-linux,i386-pc-win32',
 284       help='Which 32-bit target triples to provide to clang.')
 285   parser.add_option(
 286       '--targets64', dest='targets64',
 287       default='x86_64-pc-linux,x86_64-pc-win',
 288       help='Which 32-bit target triples to provide to clang.')
 289   parser.add_option(
 290       '-r', '--ppapi-root', dest='ppapi_root',
 291       default='.',
 292       help='The root directory of ppapi.')
 293   options, args = parser.parse_args(argv)
 294   if args:
 295     parser.print_help()
 296     print 'ERROR: invalid argument'
 297     sys.exit(1)
 298
 299   clang_executable = os.path.join(options.clang_path, 'clang')
 300   clang_command = clang_executable + " -cc1" \
 301       + " -load " + options.plugin \
 302       + " -plugin PrintNamesAndSizes" \
 303       + " -I" + os.path.join(options.ppapi_root, "..") \
 304       + " " \
 305       + os.path.join(options.ppapi_root, "tests", "test_struct_sizes.c")
 306
 307   # Dictionaries mapping type names to TypeInfo objects.
 308   # Types that have size dependent on architecture, for 32-bit
 309   types32 = {}
 310   # Types that have size dependent on architecture, for 64-bit
 311   types64 = {}
 312   # Note that types32 and types64 should contain the same types, but with
 313   # different sizes.
 314
 315   # Types whose size should be consistent regardless of architecture.
 316   types_independent = {}
 317
 318   # Now run clang for each target.  Along the way, make sure architecture-
 319   # dependent types are consistent sizes on all 32-bit platforms and consistent
 320   # on all 64-bit platforms.
 321   targets32 = options.targets32.split(',');
 322   for target in targets32:
 323     # For each 32-bit target, run the PrintNamesAndSizes Clang plugin to get
 324     # information about all types in the translation unit, and add a TypeInfo
 325     # for each of them to types32.  If any size mismatches are found,
 326     # ProcessTarget will spit out an error and exit.
 327     ProcessTarget(clang_command, target, types32)
 328   targets64 = options.targets64.split(',');
 329   for target in targets64:
 330     # Do the same as above for each 64-bit target;  put all types in types64.
 331     ProcessTarget(clang_command, target, types64)
 332
 333   # Now for each dictionary, find types whose size are consistent regardless of
 334   # architecture, and move those in to types_independent.  Anywhere sizes
 335   # differ, make sure they are expected to be architecture-dependent based on
 336   # their structure.  If we find types which could easily be consistent but
 337   # aren't, spit out an error and exit.
 338   types_independent = {}
 339   for typename, typeinfo32 in types32.items():
 340     if (typename in types64):
 341       typeinfo64 = types64[typename]
 342       if (typeinfo64.size == typeinfo32.size):
 343         # The types are the same size, so we can treat it as arch-independent.
 344         types_independent[typename] = typeinfo32
 345         del types32[typename]
 346         del types64[typename]
 347       elif (typeinfo32.arch_dependent or typeinfo64.arch_dependent):
 348         # The type is defined in such a way that it would be difficult to make
 349         # its size consistent.  E.g., it has pointers.  We'll leave it in the
 350         # arch-dependent maps so that we can put arch-dependent size checks in
 351         # test code.
 352         pass
 353       else:
 354         # The sizes don't match, but there's no reason they couldn't.  It's
 355         # probably due to an alignment mismatch between Win32/NaCl vs Linux32/
 356         # Mac32.
 357         print "Error: '" + typename + "' is", typeinfo32.size, \
 358             "bytes on target '" + typeinfo32.target + \
 359             "', but", typeinfo64.size, "on target '" + typeinfo64.target + "'"
 360         print typeinfo32.parsed_line
 361         print typeinfo64.parsed_line
 362         sys.exit(1)
 363     else:
 364       print "WARNING:  Type '", typename, "' was defined for target '",
 365       print typeinfo32.target, ", but not for any 64-bit targets."
 366
 367   # Now we have all the information we need to generate our static assertions.
 368   # Types that have consistent size across architectures will have the static
 369   # assertion placed immediately after their definition.  Types whose size
 370   # depends on 32-bit vs 64-bit architecture will have checks placed in
 371   # tests/arch_dependent_sizes_32/64.h.
 372
 373   # This dictionary maps file names to FilePatch objects.  We will add items
 374   # to it as needed.  Each FilePatch represents a set of changes to make to the
 375   # associated file (additions and deletions).
 376   file_patches = {}
 377
 378   # Find locations of existing macros, and just delete them all.  Note that
 379   # normally, only things in 'types_independent' need to be deleted, as arch-
 380   # dependent checks exist in tests/arch_dependent_sizes_32/64.h, which are
 381   # always completely over-ridden.  However, it's possible that a type that used
 382   # to be arch-independent has changed to now be arch-dependent (e.g., because
 383   # a pointer was added), and we want to delete the old check in that case.
 384   for name, typeinfo in \
 385       types_independent.items() + types32.items() + types64.items():
 386     if IsMacroDefinedName(name):
 387       sourcefile = typeinfo.source_location.filename
 388       if sourcefile not in file_patches:
 389         file_patches[sourcefile] = FilePatch(sourcefile)
 390       file_patches[sourcefile].Delete(typeinfo.source_location.start_line,
 391                                       typeinfo.source_location.end_line+1)
 392
 393   # Add a compile-time assertion for each type whose size is independent of
 394   # architecture.  These assertions go immediately after the class definition.
 395   for name, typeinfo in types_independent.items():
 396     # Ignore dummy types that were defined by macros and also ignore types that
 397     # are 0 bytes (i.e., typedefs to void).
 398     if not IsMacroDefinedName(name) and typeinfo.size > 0:
 399       sourcefile = typeinfo.source_location.filename
 400       if sourcefile not in file_patches:
 401         file_patches[sourcefile] = FilePatch(sourcefile)
 402       # Add the assertion code just after the definition of the type.
 403       # E.g.:
 404       # struct Foo {
 405       #   int32_t x;
 406       # };
 407       # PP_COMPILE_ASSERT_STRUCT_SIZE_IN_BYTES(Foo, 4); <---Add this line
 408       file_patches[sourcefile].Add(ToAssertionCode(typeinfo),
 409                                    typeinfo.source_location.end_line+1)
 410
 411   # Apply our patches.  This actually edits the files containing the definitions
 412   # for the types in types_independent.
 413   for filename, patch in file_patches.items():
 414     patch.Apply()
 415
 416   # Write out a file of checks for 32-bit architectures and a separate file for
 417   # 64-bit architectures.  These only have checks for types that are
 418   # architecture-dependent.
 419   c_source_root = os.path.join(options.ppapi_root, "tests")
 420   WriteArchSpecificCode(types32.values(),
 421                         c_source_root,
 422                         "arch_dependent_sizes_32.h")
 423   WriteArchSpecificCode(types64.values(),
 424                         c_source_root,
 425                         "arch_dependent_sizes_64.h")
 426
 427   return 0
 428
 429
 430 if __name__ == '__main__':
 431     sys.exit(main(sys.argv[1:]))
 432