toolkit/crashreporter/tools/upload_symbols.py

   1 #!/usr/bin/env python3
   2 #
   3 # This Source Code Form is subject to the terms of the Mozilla Public
   4 # License, v. 2.0. If a copy of the MPL was not distributed with this
   5 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
   6 #
   7 # This script uploads a symbol archive file from a path or URL passed on the commandline
   8 # to the symbol server at https://symbols.mozilla.org/ .
   9 #
  10 # Using this script requires you to have generated an authentication
  11 # token in the symbol server web interface. You must store the token in a Taskcluster
  12 # secret as the JSON blob `{"token": "<token>"}` and set the `SYMBOL_SECRET`
  13 # environment variable to the name of the Taskcluster secret. Alternately,
  14 # you can put the token in a file and set `SOCORRO_SYMBOL_UPLOAD_TOKEN_FILE`
  15 # environment variable to the path to the file.
  16
  17 import argparse
  18 import logging
  19 import os
  20 import sys
  21 import tempfile
  22
  23 import redo
  24 import requests
  25
  26 log = logging.getLogger("upload-symbols")
  27 log.setLevel(logging.INFO)
  28
  29 DEFAULT_URL = "https://symbols.mozilla.org/upload/"
  30 TASKCLUSTER_PROXY_URL = os.environ.get("TASKCLUSTER_PROXY_URL", "http://taskcluster")
  31 TASKCLUSTER_ROOT_URL = os.environ.get(
  32     "TASKCLUSTER_ROOT_URL", "https://firefox-ci-tc.services.mozilla.com"
  33 )
  34 MAX_RETRIES = 7
  35 MAX_ZIP_SIZE = 500000000  # 500 MB
  36
  37
  38 def print_error(r):
  39     if r.status_code < 400:
  40         log.error("Error: bad auth token? ({0}: {1})".format(r.status_code, r.reason))
  41     else:
  42         log.error("Error: got HTTP response {0}: {1}".format(r.status_code, r.reason))
  43
  44     log.error(
  45         "Response body:\n{sep}\n{body}\n{sep}\n".format(sep="=" * 20, body=r.text)
  46     )
  47
  48
  49 def get_taskcluster_secret(secret_name):
  50     secrets_url = "{}/secrets/v1/secret/{}".format(TASKCLUSTER_PROXY_URL, secret_name)
  51     log.info(
  52         'Using symbol upload token from the secrets service: "{}"'.format(secrets_url)
  53     )
  54     res = requests.get(secrets_url)
  55     res.raise_for_status()
  56     secret = res.json()
  57     auth_token = secret["secret"]["token"]
  58
  59     return auth_token
  60
  61
  62 def get_taskcluster_artifact_urls(task_id):
  63     artifacts_url = "{}/queue/v1/task/{}/artifacts".format(
  64         TASKCLUSTER_PROXY_URL, task_id
  65     )
  66     res = requests.get(artifacts_url)
  67     res.raise_for_status()
  68     return [
  69         "{}/api/queue/v1/task/{}/artifacts/{}".format(
  70             TASKCLUSTER_ROOT_URL, task_id, artifact["name"]
  71         )
  72         for artifact in res.json()["artifacts"]
  73         if artifact["name"].startswith("public/build/target.crashreporter-symbols")
  74     ]
  75
  76
  77 def main():
  78     logging.basicConfig()
  79     parser = argparse.ArgumentParser(
  80         description="Upload symbols in ZIP using token from Taskcluster secrets service."
  81     )
  82     parser.add_argument(
  83         "archive",
  84         help="Symbols archive file - URL or path to local file",
  85         nargs="*",
  86     )
  87     parser.add_argument(
  88         "--ignore-missing", help="No error on missing files", action="store_true"
  89     )
  90     parser.add_argument("--task-id", help="Taskcluster task id to use symbols from")
  91     args = parser.parse_args()
  92
  93     def check_file_exists(url):
  94         for i, _ in enumerate(redo.retrier(attempts=MAX_RETRIES), start=1):
  95             try:
  96                 resp = requests.head(url, allow_redirects=True)
  97                 return resp.status_code == requests.codes.ok
  98             except requests.exceptions.RequestException as e:
  99                 log.error("Error: {0}".format(e))
 100             log.info("Retrying...")
 101         return False
 102
 103     if args.task_id:
 104         args.archive.extend(get_taskcluster_artifact_urls(args.task_id))
 105
 106     for archive in args.archive:
 107         error = False
 108         if archive.startswith("http"):
 109             is_existing = check_file_exists(archive)
 110         else:
 111             is_existing = os.path.isfile(archive)
 112
 113         if not is_existing:
 114             if args.ignore_missing:
 115                 log.info('Archive file "{0}" does not exist!'.format(args.archive))
 116             else:
 117                 log.error(
 118                     'Error: archive file "{0}" does not exist!'.format(args.archive)
 119                 )
 120                 error = True
 121         if error:
 122             return 1
 123
 124     try:
 125         tmpdir = tempfile.TemporaryDirectory()
 126         zip_paths = convert_zst_archives(args.archive, tmpdir)
 127         for zip_path in zip_paths:
 128             result = upload_symbols(zip_path)
 129             if result:
 130                 return result
 131         return 0
 132     finally:
 133         tmpdir.cleanup()
 134
 135
 136 def convert_zst_archives(archives, tmpdir):
 137     for archive in archives:
 138         if archive.endswith(".tar.zst"):
 139             yield from convert_zst_archive(archive, tmpdir)
 140         else:
 141             yield archive
 142
 143
 144 def convert_zst_archive(zst_archive, tmpdir):
 145     """
 146     Convert a .tar.zst file to a zip file
 147
 148     Our build tasks output .tar.zst files, but the tecken server only allows
 149     .zip files to be uploaded.
 150
 151     :param zst_archive: path or URL to a .tar.zst source file
 152     :param tmpdir: TemporaryDirectory to store the output zip file in
 153     :returns: path to output zip file
 154     """
 155     import concurrent.futures
 156     import gzip
 157     import itertools
 158     import tarfile
 159
 160     import zstandard
 161     from mozpack.files import File
 162     from mozpack.mozjar import Deflater, JarWriter
 163
 164     def iter_files_from_tar(reader):
 165         ctx = zstandard.ZstdDecompressor()
 166         uncompressed = ctx.stream_reader(reader)
 167         with tarfile.open(mode="r|", fileobj=uncompressed, bufsize=1024 * 1024) as tar:
 168             while True:
 169                 info = tar.next()
 170                 if info is None:
 171                     break
 172                 data = tar.extractfile(info).read()
 173                 yield (info.name, data)
 174
 175     def prepare_from(archive, tmpdir):
 176         if archive.startswith("http"):
 177             resp = requests.get(archive, allow_redirects=True, stream=True)
 178             resp.raise_for_status()
 179             reader = resp.raw
 180             # Work around taskcluster generic-worker possibly gzipping the tar.zst.
 181             if resp.headers.get("Content-Encoding") == "gzip":
 182                 reader = gzip.GzipFile(fileobj=reader)
 183         else:
 184             reader = open(archive, "rb")
 185
 186         def handle_file(data):
 187             name, data = data
 188             log.info("Compressing %s", name)
 189             path = os.path.join(tmpdir, name.lstrip("/"))
 190             if name.endswith(".dbg"):
 191                 os.makedirs(os.path.dirname(path), exist_ok=True)
 192                 with open(path, "wb") as fh:
 193                     with gzip.GzipFile(fileobj=fh, mode="wb", compresslevel=5) as c:
 194                         c.write(data)
 195                 return (name + ".gz", File(path))
 196             elif name.endswith(".dSYM.tar"):
 197                 import bz2
 198
 199                 os.makedirs(os.path.dirname(path), exist_ok=True)
 200                 with open(path, "wb") as fh:
 201                     fh.write(bz2.compress(data))
 202                 return (name + ".bz2", File(path))
 203             elif name.endswith((".pdb", ".exe", ".dll")):
 204                 import subprocess
 205
 206                 makecab = os.environ.get("MAKECAB", "makecab")
 207                 os.makedirs(os.path.dirname(path), exist_ok=True)
 208                 with open(path, "wb") as fh:
 209                     fh.write(data)
 210
 211                 subprocess.check_call(
 212                     [makecab, "-D", "CompressionType=MSZIP", path, path + "_"],
 213                     stdout=subprocess.DEVNULL,
 214                     stderr=subprocess.STDOUT,
 215                 )
 216
 217                 return (name[:-1] + "_", File(path + "_"))
 218             else:
 219                 deflater = Deflater(compress_level=5)
 220                 deflater.write(data)
 221                 return (name, deflater)
 222
 223         with concurrent.futures.ThreadPoolExecutor(
 224             max_workers=os.cpu_count()
 225         ) as executor:
 226             yield from executor.map(handle_file, iter_files_from_tar(reader))
 227
 228         reader.close()
 229
 230     zip_paths_iter = iter(
 231         os.path.join(tmpdir.name, "symbols{}.zip".format("" if i == 1 else i))
 232         for i in itertools.count(start=1)
 233     )
 234     zip_path = next(zip_paths_iter)
 235     log.info('Preparing symbol archive "{0}" from "{1}"'.format(zip_path, zst_archive))
 236     for i, _ in enumerate(redo.retrier(attempts=MAX_RETRIES), start=1):
 237         zip_paths = []
 238         jar = None
 239         try:
 240             for name, data in prepare_from(zst_archive, tmpdir.name):
 241                 if not jar:
 242                     jar = JarWriter(zip_path)
 243                     zip_paths.append(zip_path)
 244                     size = 0
 245                 log.info("Adding %s", name)
 246                 jar.add(name, data, compress=not isinstance(data, File))
 247                 size += data.size() if isinstance(data, File) else data.compressed_size
 248                 if size > MAX_ZIP_SIZE:
 249                     jar.finish()
 250                     jar = None
 251                     zip_path = next(zip_paths_iter)
 252                     log.info('Continuing with symbol archive "{}"'.format(zip_path))
 253             if jar:
 254                 jar.finish()
 255             return zip_paths
 256         except requests.exceptions.RequestException as e:
 257             log.error("Error: {0}".format(e))
 258             log.info("Retrying...")
 259
 260     return []
 261
 262
 263 def upload_symbols(zip_path):
 264     """
 265     Upload symbols to the tecken server
 266
 267     :param zip_path: path to the zip file to upload
 268     :returns: 0 indicates the upload was successful, non-zero indicates an
 269               error that should be used for the script's exit code
 270     """
 271     secret_name = os.environ.get("SYMBOL_SECRET")
 272     if secret_name is not None:
 273         auth_token = get_taskcluster_secret(secret_name)
 274     elif "SOCORRO_SYMBOL_UPLOAD_TOKEN_FILE" in os.environ:
 275         token_file = os.environ["SOCORRO_SYMBOL_UPLOAD_TOKEN_FILE"]
 276
 277         if not os.path.isfile(token_file):
 278             log.error(
 279                 'SOCORRO_SYMBOL_UPLOAD_TOKEN_FILE "{0}" does not exist!'.format(
 280                     token_file
 281                 )
 282             )
 283             return 1
 284         auth_token = open(token_file, "r").read().strip()
 285     else:
 286         log.error(
 287             "You must set the SYMBOL_SECRET or SOCORRO_SYMBOL_UPLOAD_TOKEN_FILE "
 288             "environment variables!"
 289         )
 290         return 1
 291
 292     # Allow overwriting of the upload url with an environmental variable
 293     if "SOCORRO_SYMBOL_UPLOAD_URL" in os.environ:
 294         url = os.environ["SOCORRO_SYMBOL_UPLOAD_URL"]
 295     else:
 296         url = DEFAULT_URL
 297
 298     log.info('Uploading symbol file "{0}" to "{1}"'.format(zip_path, url))
 299
 300     for i, _ in enumerate(redo.retrier(attempts=MAX_RETRIES), start=1):
 301         log.info("Attempt %d of %d..." % (i, MAX_RETRIES))
 302         try:
 303             if zip_path.startswith("http"):
 304                 zip_arg = {"data": {"url": zip_path}}
 305             else:
 306                 zip_arg = {"files": {"symbols.zip": open(zip_path, "rb")}}
 307             r = requests.post(
 308                 url,
 309                 headers={"Auth-Token": auth_token},
 310                 allow_redirects=False,
 311                 # Allow a longer read timeout because uploading by URL means the server
 312                 # has to fetch the entire zip file, which can take a while. The load balancer
 313                 # in front of symbols.mozilla.org has a 300 second timeout, so we'll use that.
 314                 timeout=(300, 300),
 315                 **zip_arg
 316             )
 317             # 408, 429 or any 5XX is likely to be a transient failure.
 318             # Break out for success or other error codes.
 319             if r.ok or (r.status_code < 500 and (r.status_code not in (408, 429))):
 320                 break
 321             print_error(r)
 322         except requests.exceptions.RequestException as e:
 323             log.error("Error: {0}".format(e))
 324         log.info("Retrying...")
 325     else:
 326         log.warning("Maximum retries hit, giving up!")
 327         return 1
 328
 329     if r.status_code >= 200 and r.status_code < 300:
 330         log.info("Uploaded successfully!")
 331         return 0
 332
 333     print_error(r)
 334     return 1
 335
 336
 337 if __name__ == "__main__":
 338     sys.exit(main())