tools/symsrc/source_index.py

   1 #!/usr/bin/env python
   2 # Copyright (c) 2011 The Chromium Authors. All rights reserved.
   3 # Use of this source code is governed by a BSD-style license that can be
   4 # found in the LICENSE file.
   5
   6 """Usage: <win-path-to-pdb.pdb>
   7 This tool will take a PDB on the command line, extract the source files that
   8 were used in building the PDB, query SVN for which repository and revision
   9 these files are at, and then finally write this information back into the PDB
  10 in a format that the debugging tools understand.  This allows for automatic
  11 source debugging, as all of the information is contained in the PDB, and the
  12 debugger can go out and fetch the source files via SVN.
  13
  14 You most likely want to run these immediately after a build, since the source
  15 input files need to match the generated PDB, and we want the correct SVN
  16 revision information for the exact files that were used for the build.
  17
  18 The following files from a windbg + source server installation are expected
  19 to reside in the same directory as this python script:
  20   dbghelp.dll
  21   pdbstr.exe
  22   srctool.exe
  23
  24 NOTE: Expected to run under a native win32 python, NOT cygwin.  All paths are
  25 dealt with as win32 paths, since we have to interact with the Microsoft tools.
  26 """
  27
  28 import sys
  29 import os
  30 import time
  31 import subprocess
  32 import tempfile
  33
  34 # This serves two purposes.  First, it acts as a whitelist, and only files
  35 # from repositories listed here will be source indexed.  Second, it allows us
  36 # to map from one SVN URL to another, so we can map to external SVN servers.
  37 REPO_MAP = {
  38   "svn://chrome-svn/blink": "http://src.chromium.org/blink",
  39   "svn://chrome-svn/chrome": "http://src.chromium.org/chrome",
  40   "svn://chrome-svn/multivm": "http://src.chromium.org/multivm",
  41   "svn://chrome-svn/native_client": "http://src.chromium.org/native_client",
  42   "svn://chrome-svn.corp.google.com/blink": "http://src.chromium.org/blink",
  43   "svn://chrome-svn.corp.google.com/chrome": "http://src.chromium.org/chrome",
  44   "svn://chrome-svn.corp.google.com/multivm": "http://src.chromium.org/multivm",
  45   "svn://chrome-svn.corp.google.com/native_client":
  46       "http://src.chromium.org/native_client",
  47   "svn://svn-mirror.golo.chromium.org/blink": "http://src.chromium.org/blink",
  48   "svn://svn-mirror.golo.chromium.org/chrome": "http://src.chromium.org/chrome",
  49   "svn://svn-mirror.golo.chromium.org/multivm":
  50       "http://src.chromium.org/multivm",
  51   "svn://svn-mirror.golo.chromium.org/native_client":
  52       "http://src.chromium.org/native_client",
  53   "http://v8.googlecode.com/svn": None,
  54   "http://google-breakpad.googlecode.com/svn": None,
  55   "http://googletest.googlecode.com/svn": None,
  56   "http://open-vcdiff.googlecode.com/svn": None,
  57   "http://google-url.googlecode.com/svn": None,
  58 }
  59
  60
  61 def FindFile(filename):
  62   """Return the full windows path to a file in the same dir as this code."""
  63   thisdir = os.path.dirname(os.path.join(os.path.curdir, __file__))
  64   return os.path.abspath(os.path.join(thisdir, filename))
  65
  66
  67 def ExtractSourceFiles(pdb_filename):
  68   """Extract a list of local paths of the source files from a PDB."""
  69   srctool = subprocess.Popen([FindFile('srctool.exe'), '-r', pdb_filename],
  70                              stdout=subprocess.PIPE, stderr=subprocess.PIPE)
  71   filelist = srctool.stdout.read()
  72   res = srctool.wait()
  73   if res != 0 or filelist.startswith("srctool: "):
  74     raise "srctool failed: " + filelist
  75   return [x for x in filelist.split('\r\n') if len(x) != 0]
  76
  77
  78 def ReadSourceStream(pdb_filename):
  79   """Read the contents of the source information stream from a PDB."""
  80   srctool = subprocess.Popen([FindFile('pdbstr.exe'),
  81                               '-r', '-s:srcsrv',
  82                               '-p:%s' % pdb_filename],
  83                              stdout=subprocess.PIPE, stderr=subprocess.PIPE)
  84   data = srctool.stdout.read()
  85   res = srctool.wait()
  86
  87   if (res != 0 and res != -1) or data.startswith("pdbstr: "):
  88     raise "pdbstr failed: " + data
  89   return data
  90
  91
  92 def WriteSourceStream(pdb_filename, data):
  93   """Write the contents of the source information stream to a PDB."""
  94   # Write out the data to a temporary filename that we can pass to pdbstr.
  95   (f, fname) = tempfile.mkstemp()
  96   f = os.fdopen(f, "wb")
  97   f.write(data)
  98   f.close()
  99
 100   srctool = subprocess.Popen([FindFile('pdbstr.exe'),
 101                               '-w', '-s:srcsrv',
 102                               '-i:%s' % fname,
 103                               '-p:%s' % pdb_filename],
 104                              stdout=subprocess.PIPE, stderr=subprocess.PIPE)
 105   data = srctool.stdout.read()
 106   res = srctool.wait()
 107
 108   if (res != 0 and res != -1) or data.startswith("pdbstr: "):
 109     raise "pdbstr failed: " + data
 110
 111   os.unlink(fname)
 112
 113
 114 # TODO for performance, we should probably work in directories instead of
 115 # files.  I'm scared of DEPS and generated files, so for now we query each
 116 # individual file, and don't make assumptions that all files in the same
 117 # directory are part of the same repository or at the same revision number.
 118 def ExtractSvnInfo(local_filename):
 119   """Calls svn info to extract the repository, path, and revision."""
 120   # We call svn.bat to make sure and get the depot tools SVN and not cygwin.
 121   srctool = subprocess.Popen(['svn.bat', 'info', local_filename],
 122                              stdout=subprocess.PIPE, stderr=subprocess.PIPE)
 123   info = srctool.stdout.read()
 124   res = srctool.wait()
 125   if res != 0:
 126     return None
 127   # Hack up into a dictionary of the fields printed by svn info.
 128   vals = dict((y.split(': ', 2) for y in info.split('\r\n') if y))
 129
 130   root = vals['Repository Root']
 131   if not vals['URL'].startswith(root):
 132     raise "URL is not inside of the repository root?!?"
 133   path = vals['URL'][len(root):]
 134   rev  = int(vals['Revision'])
 135
 136   return [root, path, rev]
 137
 138
 139 def UpdatePDB(pdb_filename, verbose=False):
 140   """Update a pdb file with source information."""
 141   dir_blacklist = { }
 142   # TODO(deanm) look into "compressing" our output, by making use of vars
 143   # and other things, so we don't need to duplicate the repo path and revs.
 144   lines = [
 145     'SRCSRV: ini ------------------------------------------------',
 146     'VERSION=1',
 147     'INDEXVERSION=2',
 148     'VERCTRL=Subversion',
 149     'DATETIME=%s' % time.asctime(),
 150     'SRCSRV: variables ------------------------------------------',
 151     'SVN_EXTRACT_TARGET_DIR=%targ%\%fnbksl%(%var3%)\%var4%',
 152     'SVN_EXTRACT_TARGET=%svn_extract_target_dir%\%fnfile%(%var1%)',
 153     'SVN_EXTRACT_CMD=cmd /c mkdir "%svn_extract_target_dir%" && cmd /c svn cat "%var2%%var3%@%var4%" --non-interactive > "%svn_extract_target%"',
 154     'SRCSRVTRG=%SVN_extract_target%',
 155     'SRCSRVCMD=%SVN_extract_cmd%',
 156     'SRCSRV: source files ---------------------------------------',
 157   ]
 158
 159   if ReadSourceStream(pdb_filename):
 160     raise "PDB already has source indexing information!"
 161
 162   filelist = ExtractSourceFiles(pdb_filename)
 163   for filename in filelist:
 164     filedir = os.path.dirname(filename)
 165
 166     if verbose:
 167       print "Processing: %s" % filename
 168     # This directory is blacklisted, either because it's not part of the SVN
 169     # repository, or from one we're not interested in indexing.
 170     if dir_blacklist.get(filedir, False):
 171       if verbose:
 172         print "  skipping, directory is blacklisted."
 173       continue
 174
 175     info = ExtractSvnInfo(filename)
 176
 177     # Skip the file if it's not under an svn repository.  To avoid constantly
 178     # querying SVN for files outside of SVN control (for example, the CRT
 179     # sources), check if the directory is outside of SVN and blacklist it.
 180     if not info:
 181       if not ExtractSvnInfo(filedir):
 182         dir_blacklist[filedir] = True
 183       if verbose:
 184         print "  skipping, file is not in an SVN repository"
 185       continue
 186
 187     root = info[0]
 188     path = info[1]
 189     rev  = info[2]
 190
 191     # Check if file was from a svn repository we don't know about, or don't
 192     # want to index.  Blacklist the entire directory.
 193     if not REPO_MAP.has_key(info[0]):
 194       if verbose:
 195         print "  skipping, file is from an unknown SVN repository %s" % root
 196       dir_blacklist[filedir] = True
 197       continue
 198
 199     # We might want to map an internal repository URL to an external repository.
 200     if REPO_MAP[root]:
 201       root = REPO_MAP[root]
 202
 203     lines.append('%s*%s*%s*%s' % (filename, root, path, rev))
 204     if verbose:
 205       print "  indexed file."
 206
 207   lines.append('SRCSRV: end ------------------------------------------------')
 208
 209   WriteSourceStream(pdb_filename, '\r\n'.join(lines))
 210
 211
 212 def main():
 213   if len(sys.argv) < 2 or len(sys.argv) > 3:
 214     print "usage: file.pdb [-v]"
 215     return 1
 216
 217   verbose = False
 218   if len(sys.argv) == 3:
 219     verbose = (sys.argv[2] == '-v')
 220
 221   UpdatePDB(sys.argv[1], verbose=verbose)
 222   return 0
 223
 224
 225 if __name__ == '__main__':
 226   sys.exit(main())