extras/breakpad/symb_upload.py

   1 #! /usr/bin/env python3
   2 import os
   3 import sys
   4 import argparse
   5 import subprocess
   6 import logging
   7 import requests
   8 import io
   9 import shutil
  10 import typing
  11
  12 class Dumper:
  13     def __init__(self, strip_path: str = None):
  14         self.strip_path = strip_path
  15
  16     def can_process(self, fpath: str):
  17         return False
  18
  19     def dump(self, fpath: str):
  20         assert(False)
  21
  22     def _preparse_dump(self, source: str):
  23         meta = {}
  24         dest = io.StringIO()
  25         if  not source.startswith("MODULE"):
  26             logging.error("file doesn't starst with MODULE")
  27             return None, None
  28         for line in source.split("\n"):
  29             if line.startswith("MODULE"):
  30                 #MODULE <os> <arch> <buildid> <filename>
  31                 line_split = line.split(" ")
  32                 if len(line_split) != 5:
  33                     logging.error("malformed MODULE entry")
  34                     return None, None
  35                 _, _os, cpu, buildid, filename  =  line_split
  36                 if filename.endswith(".dbg"):
  37                     filename = filename[:-4]
  38                 meta["os"] = _os
  39                 meta["cpu"] = cpu
  40                 meta["debug_file"] = filename
  41                 meta["code_file"] = filename
  42                 #see CompactIdentifier in symbol_upload.cc
  43                 meta["debug_identifier"] = buildid.replace("-", "")
  44                 dest.write("MODULE {} {} {} {}".format(_os, cpu, buildid, filename))
  45                 dest.write("\n")
  46             elif line.startswith("FILE"):
  47                 #FILE <LINE> <PATH>
  48                 _, line, *path_split =  line.split(" ")
  49                 path = " ".join(path_split)
  50                 path = os.path.normpath(path)
  51
  52                 if self.strip_path and path.startswith(self.strip_path):
  53                     path = os.path.relpath(path, self.strip_path)
  54
  55                 dest.write("FILE {} {}\n".format(line, path))
  56             else:
  57                 dest.write(line)
  58                 dest.write("\n")
  59         dest.seek(0)
  60         return meta, dest
  61
  62 class WindowDumper(Dumper):
  63     def __init__(self, *args, **kwargs):
  64         super().__init__(*args, **kwargs)
  65
  66     def can_process(self, fpath: str):
  67         return any(fpath.endswith(ext) for ext in ["dbg", "dll", "exe"])
  68
  69     def dump(self, fpath: str):
  70         proc = subprocess.run(
  71             ["dump_syms_win", "-r", fpath],
  72             stdout=subprocess.PIPE,
  73             stderr=subprocess.PIPE,
  74         )
  75         if  proc.returncode != 0:
  76             logging.error("unable to extract symbols from {}".format(fpath))
  77             logging.error(proc.stderr)
  78             return None, None
  79         return self._preparse_dump(proc.stdout.decode("utf8"))
  80
  81 class MacDumper(Dumper):
  82     def __init__(self, *args, **kwargs):
  83         super().__init__(*args, **kwargs)
  84
  85     # Helper to check Mach-O header
  86     def is_mach_o(self, fpath: str):
  87         file = open(fpath, "rb")
  88         header = file.read(4)
  89         file.close()
  90         # MH_MAGIC
  91         if b'\xFE\xED\xFA\xCE' == header:
  92             return True
  93         # MH_CIGAM
  94         elif b'\xCE\xFA\xED\xFE' == header:
  95             return True
  96         # MH_MAGIC_64
  97         elif b'\xFE\xED\xFA\xCF' == header:
  98             return True
  99         # MH_CIGAM_64
 100         elif b'\xCF\xFA\xED\xFE' == header:
 101             return True
 102         return False
 103
 104     def can_process(self, fpath: str):
 105         if fpath.endswith(".dylib") or os.access(fpath, os.X_OK):
 106             return self.is_mach_o(fpath) and not os.path.islink(fpath)
 107         return False
 108
 109     def dump(self, fpath: str):
 110         dsymbundle = fpath + ".dSYM"
 111         if os.path.exists(dsymbundle):
 112             shutil.rmtree(dsymbundle)
 113
 114         #generate symbols file
 115         proc = subprocess.run(
 116             ["dsymutil", fpath],
 117             stdout=subprocess.DEVNULL,
 118             stderr=subprocess.PIPE,
 119             check=True
 120         )
 121         if  proc.returncode != 0:
 122             logging.error("unable to run dsymutil on {}:".format(fpath))
 123             logging.error(proc.stderr)
 124             return None, None
 125         if not os.path.exists(dsymbundle):
 126             logging.error("No symbols in {}".format(fpath))
 127             return None, None
 128
 129         proc = subprocess.run(
 130             ["dump_syms", "-r", "-g", dsymbundle, fpath],
 131             stdout=subprocess.PIPE,
 132             stderr=subprocess.PIPE,
 133         )
 134
 135         # Cleanup dsymbundle file
 136         shutil.rmtree(dsymbundle)
 137
 138         if  proc.returncode != 0:
 139             logging.error("unable to extract symbols from {}:".format(fpath))
 140             logging.error(proc.stderr)
 141             return None, None
 142
 143         return self._preparse_dump(proc.stdout.decode("utf8"))
 144
 145
 146 class OutputStore:
 147     def store(self, dump: typing.io.TextIO, meta):
 148         assert(False)
 149
 150 class HTTPOutputStore(OutputStore):
 151     def __init__(self, url : str, version = None, prod = None):
 152         super().__init__()
 153         self.url = url
 154         self.extra_args = {}
 155         if version:
 156             self.extra_args["ver"] = version
 157         if prod:
 158             self.extra_args["prod"] = prod
 159
 160     def store(self, dump: typing.io.TextIO, meta):
 161         post_args = {**meta, **self.extra_args}
 162         r = requests.post(self.url, post_args, files={"symfile": dump})
 163         if not r.ok:
 164             logging.error("Unable to perform request, ret {}".format(r.status_code))
 165             r.raise_for_status()
 166
 167 class LocalDirOutputStore(OutputStore):
 168     def __init__(self, rootdir: str):
 169         super().__init__()
 170         self.rootdir = rootdir
 171
 172     def store(self, dump: typing.io, meta):
 173         basepath = os.path.join(self.rootdir, meta["debug_file"], meta["debug_identifier"])
 174         if not os.path.exists(basepath):
 175             os.makedirs(basepath)
 176         with open(os.path.join(basepath, meta["debug_file"] + ".sym"), "w+") as fd:
 177             shutil.copyfileobj(dump, fd)
 178
 179 def process_dir(sourcedir, dumper, store):
 180     for root, dirnames, filenames, in os.walk(sourcedir):
 181         for fname in filenames:
 182             if not dumper.can_process(os.path.join(root, fname)):
 183                 continue
 184             logging.info("processing {}".format(fname))
 185             meta, dump = dumper.dump(os.path.join(root, fname))
 186             if meta is None or dump is None:
 187                 logging.warning("unable to dump {}".format(fname))
 188                 continue
 189             store.store(dump, meta)
 190
 191
 192 def main():
 193     parser = argparse.ArgumentParser(description='extract symbols for breakpad and upload or store them')
 194     parser.add_argument("sourcedir", help="source directory")
 195     parser.add_argument("--upload-url", metavar="URL", dest="uploadurl", type=str, help="upload url")
 196     parser.add_argument("--strip-path", metavar="PATH", dest="strippath", type=str, help="strip path prefix")
 197     parser.add_argument("-p","--platform",metavar="OS", dest="platform",
 198                         choices=["mac", "linux", "win"], required=True, help="symbol platform (mac, linux, win)")
 199     parser.add_argument("--output-dir", metavar="DIRECTORY", dest="outdir", type=str, help="output directory")
 200     parser.add_argument("--version", metavar="VERSION", dest="version", type=str, help="specify symbol version for uploading")
 201     parser.add_argument("--prod", metavar="PRODUCT", dest="prod", type=str, help="specify product name for uploading")
 202     parser.add_argument("--log", metavar="LOGLEVEL", dest="log", type=str, help="log level (INFO, WARNING, ERROR)")
 203     args = parser.parse_args()
 204
 205     if args.log:
 206         numeric_level = getattr(logging, args.log.upper(), None)
 207         if not isinstance(numeric_level, int):
 208             raise ValueError("Invalid log level: {}".format(loglevel))
 209         logging.basicConfig(format='%(levelname)s: %(message)s', level=numeric_level)
 210
 211
 212     if args.platform == "win":
 213         dumper = WindowDumper(strip_path=args.strippath)
 214     elif args.platform == "mac":
 215         dumper = MacDumper(strip_path=args.strippath)
 216     else:
 217         logging.error("Dumper {} is not implemented yet".format(args.platform))
 218         exit(1)
 219
 220     if args.uploadurl:
 221         store=HTTPOutputStore(args.uploadurl, version=args.version, prod=args.prod)
 222     elif args.outdir:
 223         store=LocalDirOutputStore(args.outdir)
 224     else:
 225         logging.error("You must chose either --output-dir or --upload-url")
 226         exit(1)
 227
 228     process_dir(args.sourcedir, dumper, store)
 229
 230
 231 if __name__ == "__main__":
 232     assert(sys.version_info >= (3,5))
 233     main()