Make MaybeInterpret, NeedsInterpret, IsDirtyOrInTableOpDirty inline
[LibreOffice.git] / bin / symbolstore.py
blobe81c529259bd98743a39ae30b3ccbe61d51818e2
1 #!/usr/bin/env python
2 # ***** BEGIN LICENSE BLOCK *****
3 # Version: MPL 1.1/GPL 2.0/LGPL 2.1
5 # The contents of this file are subject to the Mozilla Public License Version
6 # 1.1 (the "License"); you may not use this file except in compliance with
7 # the License. You may obtain a copy of the License at
8 # http://www.mozilla.org/MPL/
10 # Software distributed under the License is distributed on an "AS IS" basis,
11 # WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
12 # for the specific language governing rights and limitations under the
13 # License.
15 # The Original Code is mozilla.org code.
17 # The Initial Developer of the Original Code is
18 # The Mozilla Foundation
19 # Portions created by the Initial Developer are Copyright (C) 2007
20 # the Initial Developer. All Rights Reserved.
22 # Contributor(s):
23 # Ted Mielczarek <ted.mielczarek@gmail.com>
24 # Ben Turner <mozilla@songbirdnest.com>
26 # Alternatively, the contents of this file may be used under the terms of
27 # either the GNU General Public License Version 2 or later (the "GPL"), or
28 # the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
29 # in which case the provisions of the GPL or the LGPL are applicable instead
30 # of those above. If you wish to allow use of your version of this file only
31 # under the terms of either the GPL or the LGPL, and not to allow others to
32 # use your version of this file under the terms of the MPL, indicate your
33 # decision by deleting the provisions above and replace them with the notice
34 # and other provisions required by the GPL or the LGPL. If you do not delete
35 # the provisions above, a recipient may use your version of this file under
36 # the terms of any one of the MPL, the GPL or the LGPL.
38 # ***** END LICENSE BLOCK *****
40 # Usage: symbolstore.py <params> <dump_syms path> <symbol store path>
41 # <debug info files or dirs>
42 # Runs dump_syms on each debug info file specified on the command line,
43 # then places the resulting symbol file in the proper directory
44 # structure in the symbol store path. Accepts multiple files
45 # on the command line, so can be called as part of a pipe using
46 # find <dir> | xargs symbolstore.pl <dump_syms> <storepath>
47 # But really, you might just want to pass it <dir>.
49 # Parameters accepted:
50 # -c : Copy debug info files to the same directory structure
51 # as sym files
52 # -a "<archs>" : Run dump_syms -a <arch> for each space separated
53 # cpu architecture in <archs> (only on OS X)
54 # -s <srcdir> : Use <srcdir> as the top source directory to
55 # generate relative filenames.
57 import sys
58 import os
59 import re
60 import shutil
61 from optparse import OptionParser
63 # Utility classes
65 class VCSFileInfo:
66 """ A base class for version-controlled file information. Ensures that the
67 following attributes are generated only once (successfully):
69 self.root
70 self.clean_root
71 self.revision
72 self.filename
74 The attributes are generated by a single call to the GetRoot,
75 GetRevision, and GetFilename methods. Those methods are explicitly not
76 implemented here and must be implemented in derived classes. """
78 def __init__(self, file):
79 if not file:
80 raise ValueError
81 self.file = file
83 def __getattr__(self, name):
84 """ __getattr__ is only called for attributes that are not set on self,
85 so setting self.[attr] will prevent future calls to the GetRoot,
86 GetRevision, and GetFilename methods. We don't set the values on
87 failure on the off chance that a future call might succeed. """
89 if name == "root":
90 root = self.GetRoot()
91 if root:
92 self.root = root
93 return root
95 elif name == "clean_root":
96 clean_root = self.GetCleanRoot()
97 if clean_root:
98 self.clean_root = clean_root
99 return clean_root
101 elif name == "revision":
102 revision = self.GetRevision()
103 if revision:
104 self.revision = revision
105 return revision
107 elif name == "filename":
108 filename = self.GetFilename()
109 if filename:
110 self.filename = filename
111 return filename
113 raise AttributeError
115 def GetRoot(self):
116 """ This method should return the unmodified root for the file or 'None'
117 on failure. """
118 raise NotImplementedError
120 def GetCleanRoot(self):
121 """ This method should return the repository root for the file or 'None'
122 on failure. """
123 raise NotImplementedErrors
125 def GetRevision(self):
126 """ This method should return the revision number for the file or 'None'
127 on failure. """
128 raise NotImplementedError
130 def GetFilename(self):
131 """ This method should return the repository-specific filename for the
132 file or 'None' on failure. """
133 raise NotImplementedError
135 class CVSFileInfo(VCSFileInfo):
136 """ A class to maintain version information for files in a CVS repository.
137 Derived from VCSFileInfo. """
139 def __init__(self, file, srcdir):
140 VCSFileInfo.__init__(self, file)
141 self.srcdir = srcdir
143 def GetRoot(self):
144 (path, filename) = os.path.split(self.file)
145 root = os.path.join(path, "CVS", "Root")
146 if not os.path.isfile(root):
147 return None
148 f = open(root, "r")
149 root_name = f.readline().strip()
150 f.close()
151 if root_name:
152 return root_name
153 print >> sys.stderr, "Failed to get CVS Root for %s" % filename
154 return None
156 def GetCleanRoot(self):
157 parts = self.root.split('@')
158 if len(parts) > 1:
159 # we don't want the extra colon
160 return parts[1].replace(":","")
161 print >> sys.stderr, "Failed to get CVS Root for %s" % filename
162 return None
164 def GetRevision(self):
165 (path, filename) = os.path.split(self.file)
166 entries = os.path.join(path, "CVS", "Entries")
167 if not os.path.isfile(entries):
168 return None
169 f = open(entries, "r")
170 for line in f:
171 parts = line.split("/")
172 if len(parts) > 1 and parts[1] == filename:
173 return parts[2]
174 print >> sys.stderr, "Failed to get CVS Revision for %s" % filename
175 return None
177 def GetFilename(self):
178 file = self.file
179 if self.revision and self.clean_root:
180 if self.srcdir:
181 # strip the base path off
182 # but we actually want the last dir in srcdir
183 file = os.path.normpath(file)
184 # the lower() is to handle win32+vc8, where
185 # the source filenames come out all lowercase,
186 # but the srcdir can be mixed case
187 if file.lower().startswith(self.srcdir.lower()):
188 file = file[len(self.srcdir):]
189 (head, tail) = os.path.split(self.srcdir)
190 if tail == "":
191 tail = os.path.basename(head)
192 file = tail + file
193 return "cvs:%s:%s:%s" % (self.clean_root, file, self.revision)
194 return file
196 class SVNFileInfo(VCSFileInfo):
197 url = None
198 repo = None
199 svndata = {}
201 # This regex separates protocol and optional username/password from a url.
202 # For instance, all the following urls will be transformed into
203 # 'foo.com/bar':
205 # http://foo.com/bar
206 # svn+ssh://user@foo.com/bar
207 # svn+ssh://user:pass@foo.com/bar
209 rootRegex = re.compile(r'^\S+?:/+(?:[^\s/]*@)?(\S+)$')
211 def __init__(self, file):
212 """ We only want to run subversion's info tool once so pull all the data
213 here. """
215 VCSFileInfo.__init__(self, file)
217 if os.path.isfile(file):
218 command = os.popen("svn info %s" % file, "r")
219 for line in command:
220 # The last line of the output is usually '\n'
221 if line.strip() == '':
222 continue
223 # Split into a key/value pair on the first colon
224 key, value = line.split(':', 1)
225 if key in ["Repository Root", "Revision", "URL"]:
226 self.svndata[key] = value.strip()
228 exitStatus = command.close()
229 if exitStatus:
230 print >> sys.stderr, "Failed to get SVN info for %s" % file
232 def GetRoot(self):
233 key = "Repository Root"
234 if key in self.svndata:
235 match = self.rootRegex.match(self.svndata[key])
236 if match:
237 return match.group(1)
238 print >> sys.stderr, "Failed to get SVN Root for %s" % self.file
239 return None
241 # File bug to get this teased out from the current GetRoot, this is temporary
242 def GetCleanRoot(self):
243 return self.root
245 def GetRevision(self):
246 key = "Revision"
247 if key in self.svndata:
248 return self.svndata[key]
249 print >> sys.stderr, "Failed to get SVN Revision for %s" % self.file
250 return None
252 def GetFilename(self):
253 if self.root and self.revision:
254 if "URL" in self.svndata and "Repository Root" in self.svndata:
255 url, repo = self.svndata["URL"], self.svndata["Repository Root"]
256 file = url[len(repo) + 1:]
257 return "svn:%s:%s:%s" % (self.root, file, self.revision)
258 print >> sys.stderr, "Failed to get SVN Filename for %s" % self.file
259 return self.file
261 # Utility functions
263 # A cache of files for which VCS info has already been determined. Used to
264 # prevent extra filesystem activity or process launching.
265 vcsFileInfoCache = {}
267 def GetVCSFilename(file, srcdir):
268 """Given a full path to a file, and the top source directory,
269 look for version control information about this file, and return
270 a tuple containing
271 1) a specially formatted filename that contains the VCS type,
272 VCS location, relative filename, and revision number, formatted like:
273 vcs:vcs location:filename:revision
274 For example:
275 cvs:cvs.mozilla.org/cvsroot:mozilla/browser/app/nsBrowserApp.cpp:1.36
276 2) the unmodified root information if it exists"""
277 (path, filename) = os.path.split(file)
278 if path == '' or filename == '':
279 return (file, None)
281 fileInfo = None
282 root = ''
283 if file in vcsFileInfoCache:
284 # Already cached this info, use it.
285 fileInfo = vcsFileInfoCache[file]
286 else:
287 if os.path.isdir(os.path.join(path, "CVS")):
288 fileInfo = CVSFileInfo(file, srcdir)
289 if fileInfo:
290 root = fileInfo.root
291 elif os.path.isdir(os.path.join(path, ".svn")) or \
292 os.path.isdir(os.path.join(path, "_svn")):
293 fileInfo = SVNFileInfo(file);
294 vcsFileInfoCache[file] = fileInfo
296 if fileInfo:
297 file = fileInfo.filename
299 # we want forward slashes on win32 paths
300 return (file.replace("\\", "/"), root)
302 def GetPlatformSpecificDumper(**kwargs):
303 """This function simply returns a instance of a subclass of Dumper
304 that is appropriate for the current platform."""
305 return {'win32': Dumper_Win32,
306 'cygwin': Dumper_Win32,
307 'linux2': Dumper_Linux,
308 'sunos5': Dumper_Solaris,
309 'darwin': Dumper_Mac}[sys.platform](**kwargs)
311 def SourceIndex(fileStream, outputPath, cvs_root):
312 """Takes a list of files, writes info to a data block in a .stream file"""
313 # Creates a .pdb.stream file in the mozilla\objdir to be used for source indexing
314 # Create the srcsrv data block that indexes the pdb file
315 result = True
316 pdbStreamFile = open(outputPath, "w")
317 pdbStreamFile.write('''SRCSRV: ini ------------------------------------------------\r\nVERSION=1\r\nSRCSRV: variables ------------------------------------------\r\nCVS_EXTRACT_CMD=%fnchdir%(%targ%)cvs.exe -d %fnvar%(%var2%) checkout -r %var4% -d %var4% -N %var3%\r\nMYSERVER=''')
318 pdbStreamFile.write(cvs_root)
319 pdbStreamFile.write('''\r\nSRCSRVTRG=%targ%\%var4%\%fnbksl%(%var3%)\r\nSRCSRVCMD=%CVS_EXTRACT_CMD%\r\nSRCSRV: source files ---------------------------------------\r\n''')
320 pdbStreamFile.write(fileStream) # can't do string interpolation because the source server also uses this and so there are % in the above
321 pdbStreamFile.write("SRCSRV: end ------------------------------------------------\r\n\n")
322 pdbStreamFile.close()
323 return result
325 class Dumper:
326 """This class can dump symbols from a file with debug info, and
327 store the output in a directory structure that is valid for use as
328 a Breakpad symbol server. Requires a path to a dump_syms binary--
329 |dump_syms| and a directory to store symbols in--|symbol_path|.
330 Optionally takes a list of processor architectures to process from
331 each debug file--|archs|, the full path to the top source
332 directory--|srcdir|, for generating relative source file names,
333 and an option to copy debug info files alongside the dumped
334 symbol files--|copy_debug|, mostly useful for creating a
335 Microsoft Symbol Server from the resulting output.
337 You don't want to use this directly if you intend to call
338 ProcessDir. Instead, call GetPlatformSpecificDumper to
339 get an instance of a subclass."""
340 def __init__(self, dump_syms, symbol_path,
341 archs=None, srcdir=None, copy_debug=False, vcsinfo=False, srcsrv=False):
342 # popen likes absolute paths, at least on windows
343 self.dump_syms = dump_syms
344 self.symbol_path = symbol_path
345 if archs is None:
346 # makes the loop logic simpler
347 self.archs = ['']
348 else:
349 self.archs = ['-a %s' % a for a in archs.split()]
350 if srcdir is not None:
351 self.srcdir = os.path.normpath(srcdir)
352 else:
353 self.srcdir = None
354 self.copy_debug = copy_debug
355 self.vcsinfo = vcsinfo
356 self.srcsrv = srcsrv
358 # subclasses override this
359 def ShouldProcess(self, file):
360 return False
362 def RunFileCommand(self, file):
363 """Utility function, returns the output of file(1)"""
364 try:
365 # we use -L to read the targets of symlinks,
366 # and -b to print just the content, not the filename
367 return os.popen("file -Lb " + file).read()
368 except:
369 return ""
371 # This is a no-op except on Win32
372 def FixFilenameCase(self, file):
373 return file
375 # This is a no-op except on Win32
376 def SourceServerIndexing(self, debug_file, guid, sourceFileStream, cvs_root):
377 return ""
379 # subclasses override this if they want to support this
380 def CopyDebug(self, file, debug_file, guid):
381 pass
383 def Process(self, file_or_dir):
384 "Process a file or all the (valid) files in a directory."
385 if os.path.isdir(file_or_dir):
386 return self.ProcessDir(file_or_dir)
387 elif os.path.isfile(file_or_dir):
388 return self.ProcessFile(file_or_dir)
389 # maybe it doesn't exist?
390 return False
392 def ProcessDir(self, dir):
393 """Process all the valid files in this directory. Valid files
394 are determined by calling ShouldProcess."""
395 result = True
396 for root, dirs, files in os.walk(dir):
397 for f in files:
398 fullpath = os.path.join(root, f)
399 if self.ShouldProcess(fullpath):
400 if not self.ProcessFile(fullpath):
401 result = False
402 return result
404 def ProcessFile(self, file):
405 """Dump symbols from this file into a symbol file, stored
406 in the proper directory structure in |symbol_path|."""
407 result = False
408 sourceFileStream = ''
409 # tries to get cvsroot from the .mozconfig first - if it's not set
410 # the tinderbox cvs_path will be assigned further down
411 cvs_root = os.environ.get("SRCSRV_ROOT")
412 for arch in self.archs:
413 try:
414 cmd = os.popen("%s %s %s" % (self.dump_syms, arch, file), "r")
415 module_line = cmd.next()
416 if module_line.startswith("MODULE"):
417 # MODULE os cpu guid debug_file
418 (guid, debug_file) = (module_line.split())[3:5]
419 # strip off .pdb extensions, and append .sym
420 sym_file = re.sub("\.pdb$", "", debug_file) + ".sym"
421 # we do want forward slashes here
422 rel_path = os.path.join(debug_file,
423 guid,
424 sym_file).replace("\\", "/")
425 full_path = os.path.normpath(os.path.join(self.symbol_path,
426 rel_path))
427 try:
428 os.makedirs(os.path.dirname(full_path))
429 except OSError: # already exists
430 pass
431 f = open(full_path, "w")
432 f.write(module_line)
433 # now process the rest of the output
434 for line in cmd:
435 if line.startswith("FILE"):
436 # FILE index filename
437 (x, index, filename) = line.split(None, 2)
438 if sys.platform == "sunos5":
439 start = filename.find(self.srcdir)
440 if start == -1:
441 start = 0
442 filename = filename[start:]
443 filename = self.FixFilenameCase(filename.rstrip())
444 sourcepath = filename
445 if self.vcsinfo:
446 (filename, rootname) = GetVCSFilename(filename, self.srcdir)
447 # sets cvs_root in case the loop through files were to end on an empty rootname
448 if cvs_root is None:
449 if rootname:
450 cvs_root = rootname
451 # gather up files with cvs for indexing
452 if filename.startswith("cvs"):
453 (ver, checkout, source_file, revision) = filename.split(":", 3)
454 sourceFileStream += sourcepath + "*MYSERVER*" + source_file + '*' + revision + "\r\n"
455 f.write("FILE %s %s\n" % (index, filename))
456 else:
457 # pass through all other lines unchanged
458 f.write(line)
459 f.close()
460 cmd.close()
461 # we output relative paths so callers can get a list of what
462 # was generated
463 print rel_path
464 if self.copy_debug:
465 self.CopyDebug(file, debug_file, guid)
466 if self.srcsrv:
467 # Call on SourceServerIndexing
468 result = self.SourceServerIndexing(debug_file, guid, sourceFileStream, cvs_root)
469 result = True
470 except StopIteration:
471 pass
472 except:
473 print >> sys.stderr, "Unexpected error: ", sys.exc_info()[0]
474 raise
475 return result
477 # Platform-specific subclasses. For the most part, these just have
478 # logic to determine what files to extract symbols from.
480 class Dumper_Win32(Dumper):
481 fixedFilenameCaseCache = {}
483 def ShouldProcess(self, file):
484 """This function will allow processing of pdb files that have dll
485 or exe files with the same base name next to them."""
486 if file.endswith(".pdb"):
487 (path,ext) = os.path.splitext(file)
488 if os.path.isfile(path + ".exe") or os.path.isfile(path + ".dll") or os.path.isfile(path + ".bin"):
489 return True
490 return False
492 def FixFilenameCase(self, file):
493 """Recent versions of Visual C++ put filenames into
494 PDB files as all lowercase. If the file exists
495 on the local filesystem, fix it."""
497 # Use a cached version if we have one.
498 if file in self.fixedFilenameCaseCache:
499 return self.fixedFilenameCaseCache[file]
501 result = file
503 (path, filename) = os.path.split(file)
504 if os.path.isdir(path):
505 lc_filename = filename.lower()
506 for f in os.listdir(path):
507 if f.lower() == lc_filename:
508 result = os.path.join(path, f)
509 break
511 # Cache the corrected version to avoid future filesystem hits.
512 self.fixedFilenameCaseCache[file] = result
513 return result
515 def CopyDebug(self, file, debug_file, guid):
516 rel_path = os.path.join(debug_file,
517 guid,
518 debug_file).replace("\\", "/")
519 print rel_path
520 full_path = os.path.normpath(os.path.join(self.symbol_path,
521 rel_path))
522 shutil.copyfile(file, full_path)
523 pass
525 def SourceServerIndexing(self, debug_file, guid, sourceFileStream, cvs_root):
526 # Creates a .pdb.stream file in the mozilla\objdir to be used for source indexing
527 cwd = os.getcwd()
528 streamFilename = debug_file + ".stream"
529 stream_output_path = os.path.join(cwd, streamFilename)
530 # Call SourceIndex to create the .stream file
531 result = SourceIndex(sourceFileStream, stream_output_path, cvs_root)
533 if self.copy_debug:
534 pdbstr_path = os.environ.get("PDBSTR_PATH")
535 pdbstr = os.path.normpath(pdbstr_path)
536 pdb_rel_path = os.path.join(debug_file, guid, debug_file)
537 pdb_filename = os.path.normpath(os.path.join(self.symbol_path, pdb_rel_path))
538 # move to the dir with the stream files to call pdbstr
539 os.chdir(os.path.dirname(stream_output_path))
540 os.spawnv(os.P_WAIT, pdbstr, [pdbstr, "-w", "-p:" + pdb_filename, "-i:" + streamFilename, "-s:srcsrv"])
541 # clean up all the .stream files when done
542 os.remove(stream_output_path)
543 return result
545 class Dumper_Linux(Dumper):
546 def ShouldProcess(self, file):
547 """This function will allow processing of files that are
548 executable, or end with the .so extension, and additionally
549 file(1) reports as being ELF files. It expects to find the file
550 command in PATH."""
551 if file.endswith(".so") or file.endswith(".bin") or os.access(file, os.X_OK):
552 return self.RunFileCommand(file).startswith("ELF")
553 return False
555 def CopyDebug(self, file, debug_file, guid):
556 # We want to strip out the debug info, and add a
557 # .gnu_debuglink section to the object, so the debugger can
558 # actually load our debug info later.
559 file_dbg = file + ".dbg"
560 os.system("objcopy --only-keep-debug %s %s" % (file, file_dbg))
561 os.system("objcopy --add-gnu-debuglink=%s %s" % (file_dbg, file))
563 rel_path = os.path.join(debug_file,
564 guid,
565 debug_file + ".dbg")
566 full_path = os.path.normpath(os.path.join(self.symbol_path,
567 rel_path))
568 shutil.copyfile(file_dbg, full_path)
569 # gzip the shipped debug files
570 os.system("gzip %s" % full_path)
571 print rel_path + ".gz"
573 class Dumper_Solaris(Dumper):
574 def RunFileCommand(self, file):
575 """Utility function, returns the output of file(1)"""
576 try:
577 output = os.popen("file " + file).read()
578 return output.split('\t')[1];
579 except:
580 return ""
582 def ShouldProcess(self, file):
583 """This function will allow processing of files that are
584 executable, or end with the .so extension, and additionally
585 file(1) reports as being ELF files. It expects to find the file
586 command in PATH."""
587 if file.endswith(".so") or os.access(file, os.X_OK):
588 return self.RunFileCommand(file).startswith("ELF")
589 return False
591 class Dumper_Mac(Dumper):
592 def ShouldProcess(self, file):
593 """This function will allow processing of files that are
594 executable, or end with the .dylib extension, and additionally
595 file(1) reports as being Mach-O files. It expects to find the file
596 command in PATH."""
597 if file.endswith(".dylib") or os.access(file, os.X_OK):
598 return self.RunFileCommand(file).startswith("Mach-O")
599 return False
601 # Entry point if called as a standalone program
602 def main():
603 parser = OptionParser(usage="usage: %prog [options] <dump_syms binary> <symbol store path> <debug info files>")
604 parser.add_option("-c", "--copy",
605 action="store_true", dest="copy_debug", default=False,
606 help="Copy debug info files into the same directory structure as symbol files")
607 parser.add_option("-a", "--archs",
608 action="store", dest="archs",
609 help="Run dump_syms -a <arch> for each space separated cpu architecture in ARCHS (only on OS X)")
610 parser.add_option("-s", "--srcdir",
611 action="store", dest="srcdir",
612 help="Use SRCDIR to determine relative paths to source files")
613 parser.add_option("-v", "--vcs-info",
614 action="store_true", dest="vcsinfo",
615 help="Try to retrieve VCS info for each FILE listed in the output")
616 parser.add_option("-i", "--source-index",
617 action="store_true", dest="srcsrv", default=False,
618 help="Add source index information to debug files, making them suitable for use in a source server.")
619 (options, args) = parser.parse_args()
621 #check to see if the pdbstr.exe exists
622 if options.srcsrv:
623 pdbstr = os.environ.get("PDBSTR_PATH")
624 if not os.path.exists(pdbstr):
625 print >> sys.stderr, "Invalid path to pdbstr.exe - please set/check PDBSTR_PATH.\n"
626 sys.exit(1)
628 if len(args) < 3:
629 parser.error("not enough arguments")
630 exit(1)
632 dumper = GetPlatformSpecificDumper(dump_syms=args[0],
633 symbol_path=args[1],
634 copy_debug=options.copy_debug,
635 archs=options.archs,
636 srcdir=options.srcdir,
637 vcsinfo=options.vcsinfo,
638 srcsrv=options.srcsrv)
639 for arg in args[2:]:
640 dumper.Process(arg)
642 # run main if run directly
643 if __name__ == "__main__":
644 main()