contrib/header-tools/headerutils.py

   1 #! /usr/bin/python2
   2 import os.path
   3 import sys
   4 import shlex
   5 import re
   6 import subprocess
   7 import shutil
   8 import pickle
   9
  10 import multiprocessing
  11
  12 def find_pound_include (line, use_outside, use_slash):
  13   inc = re.findall (ur"^\s*#\s*include\s*\"(.+?)\"", line)
  14   if len(inc) == 1:
  15     nm = inc[0]
  16     if use_outside or os.path.exists (nm):
  17       if use_slash or '/' not in nm:
  18         return nm
  19   return ""
  20
  21 def find_system_include (line):
  22   inc = re.findall (ur"^\s*#\s*include\s*<(.+?)>", line)
  23   if len(inc) == 1:
  24     return inc[0]
  25   return ""
  26
  27 def find_pound_define (line):
  28   inc = re.findall (ur"^\s*#\s*define ([A-Za-z0-9_]+)", line)
  29   if len(inc) != 0:
  30     if len(inc) > 1:
  31       print "What? more than 1 match in #define??"
  32       print inc
  33       sys.exit(5)
  34     return inc[0];
  35   return ""
  36
  37 def is_pound_if (line):
  38   inc = re.findall ("^\s*#\s*if\s", line)
  39   if not inc:
  40     inc = re.findall ("^\s*#\s*if[n]?def\s", line)
  41   if inc:
  42     return True
  43   return False
  44
  45 def is_pound_endif (line):
  46   inc = re.findall ("^\s*#\s*endif", line)
  47   if inc:
  48     return True
  49   return False
  50
  51 def find_pound_if (line):
  52   inc = re.findall (ur"^\s*#\s*if\s+(.*)", line)
  53   if len(inc) == 0:
  54     inc = re.findall (ur"^\s*#\s*elif\s+(.*)", line)
  55   if len(inc) > 0:
  56     inc2 = re.findall (ur"defined\s*\((.+?)\)", inc[0])
  57     inc3 = re.findall (ur"defined\s+([a-zA-Z0-9_]+)", inc[0])
  58     for yy in inc3:
  59       inc2.append (yy)
  60     return inc2
  61   else:
  62     inc = re.findall (ur"^\s*#\s*ifdef\s(.*)", line)
  63     if len(inc) == 0:
  64       inc = re.findall (ur"^\s*#\s*ifndef\s(.*)", line)
  65     if len(inc) > 0:
  66       inc2 = re.findall ("[A-Za-z_][A-Za-z_0-9]*", inc[0])
  67       return inc2
  68   if len(inc) == 0:
  69     return list ()
  70   print "WTF. more than one line returned for find_pound_if"
  71   print inc
  72   sys.exit(5)
  73
  74
  75 # IINFO - this is a vector of include information. It consists of 7 elements.
  76 # [0] - base name of the file
  77 # [1] - path leading to this file.
  78 # [2] - orderd list of all headers directly included by this file.
  79 # [3] - Ordered list of any headers included within condionally compiled code.
  80 #       headers files are expected to have all includes one level deep due to
  81 #       the omnipresent guards at the top of the file.
  82 # [4] - List of all macros which are consumed (used) within this file.
  83 # [5] - list of all macros which may be defined in this file.
  84 # [6] - The source code for this file, if cached.
  85 # [7] - line number info for any headers in the source file.  Indexed by base
  86 #       name, returning the line the include is on.
  87
  88 empty_iinfo =  ("", "", list(), list(), list(), list(), list())
  89
  90 # This function will process a file and extract interesting information.
  91 # DO_MACROS indicates whether macros defined and used should be recorded.
  92 # KEEP_SRC indicates the source for the file should be cached.
  93 def process_include_info (filen, do_macros, keep_src):
  94   header = False
  95   if not os.path.exists (filen):
  96     return empty_iinfo
  97
  98   sfile = open (filen, "r");
  99   data = sfile.readlines()
 100   sfile.close()
 101
 102   # Ignore the initial #ifdef HEADER_H in header files
 103   if filen[-2:] == ".h":
 104     nest = -1
 105     header = True
 106   else:
 107     nest = 0
 108
 109   macout = list ()
 110   macin = list()
 111   incl = list()
 112   cond_incl = list()
 113   src_line = { }
 114   guard = ""
 115
 116   for line in (data):
 117     if is_pound_if (line):
 118       nest += 1
 119     elif is_pound_endif (line):
 120       nest -= 1
 121
 122     nm = find_pound_include (line, True, True)
 123     if nm != "" and nm not in incl and nm[-2:] == ".h":
 124       incl.append (nm)
 125       if nest > 0:
 126         cond_incl.append (nm)
 127       if keep_src:
 128         src_line[nm] = line
 129       continue
 130
 131     if do_macros:
 132       d = find_pound_define (line)
 133       if d:
 134         if d not in macout:
 135           macout.append (d);
 136           continue
 137
 138       d = find_pound_if (line)
 139       if d:
 140         # The first #if in a header file should be the guard
 141         if header and len (d) == 1 and guard == "":
 142           if d[0][-2:] == "_H":
 143             guard = d
 144           else:
 145             guard = "Guess there was no guard..."
 146         else:
 147           for mac in d:
 148             if mac != "defined" and mac not in macin:
 149               macin.append (mac);
 150
 151   if not keep_src:
 152     data = list()
 153
 154   return (os.path.basename (filen), os.path.dirname (filen), incl, cond_incl,
 155           macin, macout, data, src_line)
 156
 157 # Extract header info, but no macros or source code.
 158 def process_ii (filen):
 159   return process_include_info (filen, False, False)
 160
 161 # Extract header information, and collect macro information.
 162 def process_ii_macro (filen):
 163   return process_include_info (filen, True, False)
 164
 165 # Extract header information, cache the source lines.
 166 def process_ii_src (filen):
 167   return process_include_info (filen, False, True)
 168
 169 # Extract header information, coolewc macro info and cache the source lines.
 170 def process_ii_macro_src (filen):
 171   return process_include_info (filen, True, True)
 172
 173
 174 def ii_base (iinfo):
 175   return iinfo[0]
 176
 177 def ii_path (iinfo):
 178   return iinfo[1]
 179
 180 def ii_include_list (iinfo):
 181   return iinfo[2]
 182
 183 def ii_include_list_cond (iinfo):
 184   return iinfo[3]
 185
 186 def ii_include_list_non_cond (iinfo):
 187   l = ii_include_list (iinfo)
 188   for n in ii_include_list_cond (iinfo):
 189     l.remove (n)
 190   return l
 191
 192 def ii_macro_consume (iinfo):
 193   return iinfo[4]
 194
 195 def ii_macro_define (iinfo):
 196   return iinfo[5]
 197
 198 def ii_src (iinfo):
 199   return iinfo[6]
 200
 201 def ii_src_line (iinfo):
 202   return iinfo[7]
 203
 204 def ii_read (fname):
 205   f = open (fname, 'rb')
 206   incl = pickle.load (f)
 207   consumes = pickle.load (f)
 208   defines = pickle.load (f)
 209   obj = (fname,fname,incl,list(), list(), consumes, defines, list(), list())
 210   return obj
 211
 212 def ii_write (fname, obj):
 213   f = open (fname, 'wb')
 214   pickle.dump (obj[2], f)
 215   pickle.dump (obj[4], f)
 216   pickle.dump (obj[5], f)
 217   f.close ()
 218
 219 # execute a system command which returns file names
 220 def execute_command (command):
 221   files = list()
 222   f = os.popen (command)
 223   for x in f:
 224     if x[0:2] == "./":
 225       fn = x.rstrip()[2:]
 226     else:
 227       fn = x.rstrip()
 228     files.append(fn)
 229   return files
 230
 231 # Try to locate a build directory from PATH
 232 def find_gcc_bld_dir (path):
 233   blddir = ""
 234   # Look for blddir/gcc/tm.h
 235   command = "find " + path + " -mindepth 2 -maxdepth 3 -name tm.h"
 236   files = execute_command (command)
 237   for y in files:
 238     p = os.path.dirname (y)
 239     if os.path.basename (p) == "gcc":
 240       blddir = p
 241       break
 242   # If not found, try looking a bit deeper
 243   # Dont look this deep initially because a lot of cross target builds may show
 244   # up in the list before a native build... but those are better than nothing.
 245   if not blddir:
 246     command = "find " + path + " -mindepth 3 -maxdepth 5 -name tm.h"
 247     files = execute_command (command)
 248     for y in files:
 249       p = os.path.dirname (y)
 250       if os.path.basename (p) == "gcc":
 251         blddir = p
 252         break
 253
 254   return blddir
 255
 256
 257 # Find files matching pattern NAME, return in a list.
 258 # CURRENT is True if you want to include the current directory
 259 # DEEPER is True if you want to search 3 levels below the current directory
 260 # any files with testsuite diurectories are ignored
 261
 262 def find_gcc_files (name, current, deeper):
 263   files = list()
 264   command = ""
 265   if current:
 266     if not deeper:
 267       command = "find -maxdepth 1 -name " + name + " -not -path \"./testsuite/*\""
 268     else:
 269       command = "find -maxdepth 4 -name " + name + " -not -path \"./testsuite/*\""
 270   else:
 271     if deeper:
 272       command = "find -maxdepth 4 -mindepth 2 -name " + name + " -not -path \"./testsuite/*\""
 273
 274   if command != "":
 275     files = execute_command (command)
 276
 277   return files
 278
 279 # find the list of unique include names found in a file.
 280 def find_unique_include_list_src (data):
 281   found = list ()
 282   for line in data:
 283     d = find_pound_include (line, True, True)
 284     if d and d not in found and d[-2:] == ".h":
 285       found.append (d)
 286   return found
 287
 288 # find the list of unique include names found in a file.
 289 def find_unique_include_list (filen):
 290   data = open (filen).read().splitlines()
 291   return find_unique_include_list_src (data)
 292
 293
 294 # Create the macin, macout, and incl vectors for a file FILEN.
 295 # macin are the macros that are used in #if* conditional expressions
 296 # macout are the macros which are #defined
 297 # incl is the list of incluide files encountered
 298 # returned as a tuple of the filename followed by the triplet of lists
 299 # (filen, macin, macout, incl)
 300
 301 def create_macro_in_out (filen):
 302   sfile = open (filen, "r");
 303   data = sfile.readlines()
 304   sfile.close()
 305
 306   macout = list ()
 307   macin = list()
 308   incl = list()
 309
 310   for line in (data):
 311     d = find_pound_define (line)
 312     if d != "":
 313       if d not in macout:
 314         macout.append (d);
 315       continue
 316
 317     d = find_pound_if (line)
 318     if len(d) != 0:
 319       for mac in d:
 320         if mac != "defined" and mac not in macin:
 321           macin.append (mac);
 322       continue
 323
 324     nm = find_pound_include (line, True, True)
 325     if nm != "" and nm not in incl:
 326       incl.append (nm)
 327
 328   return (filen, macin, macout, incl)
 329
 330 # create the macro information for filen, and create .macin, .macout, and .incl
 331 # files.  Return the created macro tuple.
 332 def create_include_data_files (filen):
 333
 334   macros = create_macro_in_out (filen)
 335   depends = macros[1]
 336   defines = macros[2]
 337   incls = macros[3]
 338
 339   disp_message = filen
 340   if len (defines) > 0:
 341     disp_message = disp_message + " " + str(len (defines)) + " #defines"
 342   dfile = open (filen + ".macout", "w")
 343   for x in defines:
 344     dfile.write (x + "\n")
 345   dfile.close ()
 346
 347   if len (depends) > 0:
 348     disp_message = disp_message + " " + str(len (depends)) + " #if dependencies"
 349   dfile = open (filen + ".macin", "w")
 350   for x in depends:
 351     dfile.write (x + "\n")
 352   dfile.close ()
 353
 354   if len (incls) > 0:
 355     disp_message = disp_message + " " + str(len (incls)) + " #includes"
 356   dfile = open (filen + ".incl", "w")
 357   for x in incls:
 358     dfile.write (x + "\n")
 359   dfile.close ()
 360
 361   return macros
 362
 363
 364
 365 # extract data for include file name_h and enter it into the dictionary.
 366 # this does not change once read in.  use_requires is True if you want to
 367 # prime the values with already created .requires and .provides files.
 368 def get_include_data (name_h, use_requires):
 369   macin = list()
 370   macout = list()
 371   incl = list ()
 372   if use_requires and os.path.exists (name_h + ".requires"):
 373     macin = open (name_h + ".requires").read().splitlines()
 374   elif os.path.exists (name_h + ".macin"):
 375     macin = open (name_h + ".macin").read().splitlines()
 376
 377   if use_requires and os.path.exists (name_h + ".provides"):
 378     macout  = open (name_h + ".provides").read().splitlines()
 379   elif os.path.exists (name_h + ".macout"):
 380     macout  = open (name_h + ".macout").read().splitlines()
 381
 382   if os.path.exists (name_h + ".incl"):
 383     incl = open (name_h + ".incl").read().splitlines()
 384
 385   if len(macin) == 0 and len(macout) == 0 and len(incl) == 0:
 386     return ()
 387   data = ( name_h, macin, macout, incl )
 388   return data
 389
 390 # find FIND in src, and replace it with the list of headers in REPLACE.
 391 # Remove any duplicates of FIND in REPLACE, and if some of the REPLACE
 392 # headers occur earlier in the include chain, leave them.
 393 # Return the new SRC only if anything changed.
 394 def find_replace_include (find, replace, src):
 395   res = list()
 396   seen = { }
 397   anything = False
 398   for line in src:
 399     inc = find_pound_include (line, True, True)
 400     if inc == find:
 401       for y in replace:
 402         if seen.get(y) == None:
 403           res.append("#include \""+y+"\"\n")
 404           seen[y] = True
 405           if y != find:
 406             anything = True
 407 # if find isnt in the replacement list, then we are deleting FIND, so changes.
 408       if find not in replace:
 409         anything = True
 410     else:
 411       if inc in replace:
 412         if seen.get(inc) == None:
 413           res.append (line)
 414           seen[inc] = True
 415       else:
 416         res.append (line)
 417
 418   if (anything):
 419     return res
 420   else:
 421     return list()
 422
 423
 424 # pass in a require and provide dictionary to be read in.
 425 def read_require_provides (require, provide):
 426   if not os.path.exists ("require-provide.master"):
 427     print "require-provide.master file is not available. please run data collection."
 428     sys.exit(1)
 429   incl_list = open("require-provide.master").read().splitlines()
 430   for f in incl_list:
 431     if os.path.exists (f+".requires"):
 432       require[os.path.basename (f)] = open (f + ".requires").read().splitlines()
 433     else:
 434       require[os.path.basename (f)] = list ()
 435     if os.path.exists (f+".provides"):
 436       provide[os.path.basename (f)] = open (f + ".provides").read().splitlines()
 437     else:
 438       provide [os.path.basename (f)] = list ()
 439
 440
 441 def build_include_list (filen):
 442   include_files = list()
 443   sfile = open (filen, "r")
 444   data = sfile.readlines()
 445   sfile.close()
 446   for line in data:
 447     nm = find_pound_include (line, False, False)
 448     if nm != "" and nm[-2:] == ".h":
 449       if nm not in include_files:
 450         include_files.append(nm)
 451   return include_files
 452
 453 def build_reverse_include_list (filen):
 454   include_files = list()
 455   sfile = open (filen, "r")
 456   data = sfile.readlines()
 457   sfile.close()
 458   for line in reversed(data):
 459     nm = find_pound_include (line, False, False)
 460     if nm != "":
 461       if nm not in include_files:
 462         include_files.append(nm)
 463   return include_files
 464
 465 # Get compilation return code, and compensate for a warning that we want to
 466 # consider an error when it comes to inlined templates.
 467 def get_make_rc (rc, output):
 468   rc = rc % 1280
 469   if rc == 0:
 470     # This is not considered an error during compilation of an individual file,
 471     # but it will cause an error during link if it isn't defined.  If this
 472     # warning is seen during compiling a file, make it a build error so we
 473     # don't remove the header.
 474     h = re.findall ("warning: inline function.*used but never defined", output)
 475     if len(h) != 0:
 476       rc = 1
 477   return rc;
 478
 479 def get_make_output (build_dir, make_opt):
 480   devnull = open('/dev/null', 'w')
 481   at_a_time = multiprocessing.cpu_count() * 2
 482   make = "make -j"+str(at_a_time)+ " "
 483   if build_dir != "":
 484     command = "cd " + build_dir +"; " + make + make_opt
 485   else:
 486     command = make + make_opt
 487   process = subprocess.Popen(command, stdout=devnull, stderr=subprocess.PIPE, shell=True)
 488   output = process.communicate();
 489   rc = get_make_rc (process.returncode, output[1])
 490   return (rc , output[1])
 491
 492 def spawn_makes (command_list):
 493   devnull = open('/dev/null', 'w')
 494   rc = (0,"", "")
 495   proc_res = list()
 496   text = "  Trying target builds : "
 497   for command_pair in command_list:
 498     tname = command_pair[0]
 499     command = command_pair[1]
 500     text += tname + ", "
 501     c = subprocess.Popen(command, bufsize=-1, stdout=devnull, stderr=subprocess.PIPE, shell=True)
 502     proc_res.append ((c, tname))
 503
 504   print text[:-2]
 505
 506   for p in proc_res:
 507     output = p[0].communicate()
 508     ret = (get_make_rc (p[0].returncode, output[1]), output[1], p[1])
 509     if (ret[0] != 0):
 510       # Just record the first one.
 511       if rc[0] == 0:
 512         rc = ret;
 513   return rc
 514
 515 def get_make_output_parallel (targ_list, make_opt, at_a_time):
 516   command = list()
 517   targname = list()
 518   if at_a_time == 0:
 519     at_a_time = multiprocessing.cpu_count() * 2
 520   proc_res = [0] * at_a_time
 521   for x in targ_list:
 522     if make_opt[-2:] == ".o":
 523       s = "cd " + x[1] + "/gcc/; make " + make_opt
 524     else:
 525       s = "cd " + x[1] +"; make " + make_opt
 526     command.append ((x[0],s))
 527
 528   num = len(command)
 529   rc = (0,"", "")
 530   loops = num // at_a_time
 531
 532   if (loops > 0):
 533     for idx in range (loops):
 534       ret = spawn_makes (command[idx*at_a_time:(idx+1)*at_a_time])
 535       if ret[0] != 0:
 536         rc = ret
 537         break
 538
 539   if (rc[0] == 0):
 540     leftover = num % at_a_time
 541     if (leftover > 0):
 542       ret = spawn_makes (command[-leftover:])
 543       if ret[0] != 0:
 544         rc = ret
 545
 546   return rc
 547
 548
 549 def readwholefile (src_file):
 550   sfile = open (src_file, "r")
 551   src_data = sfile.readlines()
 552   sfile.close()
 553   return src_data
 554