build/gen_mach_buildprops.py

   1 #!/usr/bin/python
   2 #
   3 # This Source Code Form is subject to the terms of the Mozilla Public
   4 # License, v. 2.0. If a copy of the MPL was not distributed with this
   5 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
   6
   7 import sys
   8 import os
   9 import hashlib
  10 import json
  11 import re
  12 import errno
  13 from argparse import ArgumentParser
  14
  15 def getFileHashAndSize(filename):
  16     sha512Hash = 'UNKNOWN'
  17     size = 'UNKNOWN'
  18
  19     try:
  20         # open in binary mode to make sure we get consistent results
  21         # across all platforms
  22         f = open(filename, "rb")
  23         shaObj = hashlib.sha512(f.read())
  24         sha512Hash = shaObj.hexdigest()
  25
  26         size = os.path.getsize(filename)
  27     except:
  28         pass
  29
  30     return (sha512Hash, size)
  31
  32 def getMarProperties(filename, partial=False):
  33     if not os.path.exists(filename):
  34         return {}
  35     (mar_hash, mar_size) = getFileHashAndSize(filename)
  36     martype = 'partial' if partial else 'complete'
  37     return {
  38         '%sMarFilename' % martype: os.path.basename(filename),
  39         '%sMarSize' % martype: mar_size,
  40         '%sMarHash' % martype: mar_hash,
  41     }
  42
  43 def getUrlProperties(filename, package):
  44     # let's create a switch case using name-spaces/dict
  45     # rather than a long if/else with duplicate code
  46     property_conditions = [
  47         # key: property name, value: condition
  48         ('symbolsUrl', lambda m: m.endswith('crashreporter-symbols.zip') or
  49                        m.endswith('crashreporter-symbols-full.zip')),
  50         ('testsUrl', lambda m: m.endswith(('tests.tar.bz2', 'tests.zip'))),
  51         ('unsignedApkUrl', lambda m: m.endswith('apk') and
  52                            'unsigned-unaligned' in m),
  53         ('robocopApkUrl', lambda m: m.endswith('apk') and 'robocop' in m),
  54         ('jsshellUrl', lambda m: 'jsshell-' in m and m.endswith('.zip')),
  55         ('completeMarUrl', lambda m: m.endswith('.complete.mar')),
  56         ('partialMarUrl', lambda m: m.endswith('.mar') and '.partial.' in m),
  57         ('codeCoverageURL', lambda m: m.endswith('code-coverage-gcno.zip')),
  58         ('packageUrl', lambda m: m.endswith(package)),
  59     ]
  60     url_re = re.compile(r'''^(https?://.*?\.(?:tar\.bz2|dmg|zip|apk|rpm|mar|tar\.gz))$''')
  61     properties = {}
  62
  63     try:
  64         with open(filename) as f:
  65             for line in f:
  66                 m = url_re.match(line)
  67                 if m:
  68                     m = m.group(1)
  69                     for prop, condition in property_conditions:
  70                         if condition(m):
  71                             properties.update({prop: m})
  72                             break
  73     except IOError as e:
  74         if e.errno != errno.ENOENT:
  75             raise
  76         properties = {prop: 'UNKNOWN' for prop, condition in property_conditions}
  77     return properties
  78
  79 def getPartialInfo(props):
  80     return [{
  81         "from_buildid": props.get("previous_buildid"),
  82         "size": props.get("partialMarSize"),
  83         "hash": props.get("partialMarHash"),
  84         "url": props.get("partialMarUrl"),
  85     }]
  86
  87 if __name__ == '__main__':
  88     parser = ArgumentParser(description='Generate mach_build_properties.json for automation builds.')
  89     parser.add_argument("--complete-mar-file", required=True,
  90                         action="store", dest="complete_mar_file",
  91                         help="Path to the complete MAR file, relative to the objdir.")
  92     parser.add_argument("--partial-mar-file", required=False,
  93                         action="store", dest="partial_mar_file",
  94                         help="Path to the partial MAR file, relative to the objdir.")
  95     parser.add_argument("--upload-output", required=True,
  96                         action="store", dest="upload_output",
  97                         help="Path to the text output of 'make upload'")
  98     parser.add_argument("--upload-files", required=True, nargs="+",
  99                         action="store", dest="upload_files",
 100                         help="List of files to be uploaded.")
 101     parser.add_argument("--package", required=True,
 102                         action="store", dest="package",
 103                         help="Filename of the build package")
 104     args = parser.parse_args()
 105
 106     json_data = getMarProperties(args.complete_mar_file)
 107     json_data.update(getUrlProperties(args.upload_output, args.package))
 108     if args.partial_mar_file:
 109         json_data.update(getMarProperties(args.partial_mar_file, partial=True))
 110
 111         # Pull the previous buildid from the partial mar filename.
 112         res = re.match(r'.*\.([0-9]+)-[0-9]+.mar', args.partial_mar_file)
 113         if res:
 114             json_data['previous_buildid'] = res.group(1)
 115
 116             # Set partialInfo to be a collection of the partial mar properties
 117             # useful for balrog.
 118             json_data['partialInfo'] = getPartialInfo(json_data)
 119
 120     json_data['uploadFiles'] = args.upload_files
 121     json_data['packageFilename'] = args.package
 122
 123     with open('mach_build_properties.json', 'w') as outfile:
 124         json.dump(json_data, outfile, indent=4)