tools/update-packaging/make_incremental_updates.py

   1 # This Source Code Form is subject to the terms of the Mozilla Public
   2 # License, v. 2.0. If a copy of the MPL was not distributed with this
   3 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
   4
   5 import os
   6 import shutil
   7 import hashlib
   8 import re
   9 import sys
  10 import getopt
  11 import time
  12 import tempfile
  13 import io
  14
  15
  16 class PatchInfo:
  17     """ Represents the meta-data associated with a patch
  18         work_dir = working dir where files are stored for this patch
  19         archive_files = list of files to include in this patch
  20         manifestv2 = set of manifest version 2 patch instructions
  21         manifestv3 = set of manifest version 3 patch instructions
  22         file_exclusion_list =
  23         files to exclude from this patch. names without slashes will be
  24         excluded anywhere in the directory hiearchy.   names with slashes
  25         will only be excluded at that exact path
  26         """
  27
  28     def __init__(self, work_dir, file_exclusion_list, path_exclusion_list):
  29         self.work_dir = work_dir
  30         self.archive_files = []
  31         self.manifestv2 = []
  32         self.manifestv3 = []
  33         self.file_exclusion_list = file_exclusion_list
  34         self.path_exclusion_list = path_exclusion_list
  35
  36     def append_add_instruction(self, filename):
  37         """ Appends an add instruction for this patch.
  38             if filename starts with distribution/extensions/.*/ this will add an
  39             add-if instruction that will add the file if the parent directory
  40             of the file exists.  This was ported from
  41             mozilla/tools/update-packaging/common.sh's make_add_instruction.
  42         """
  43         m = re.match("((?:|.*/)distribution/extensions/.*)/", filename)
  44         if m:
  45             # Directory immediately following extensions is used for the test
  46             testdir = m.group(1)
  47             print('     add-if "' + testdir + '" "' + filename + '"')
  48             self.manifestv2.append('add-if "' + testdir + '" "' + filename + '"')
  49             self.manifestv3.append('add-if "' + testdir + '" "' + filename + '"')
  50         else:
  51             print('        add "' + filename + '"')
  52             self.manifestv2.append('add "' + filename + '"')
  53             self.manifestv3.append('add "' + filename + '"')
  54
  55     def append_add_if_not_instruction(self, filename):
  56         """ Appends an add-if-not instruction to the version 3 manifest for this patch.
  57             This was ported from mozilla/tools/update-packaging/common.sh's
  58             make_add_if_not_instruction.
  59         """
  60         print(' add-if-not "' + filename + '" "' + filename + '"')
  61         self.manifestv3.append('add-if-not "' + filename + '" "' + filename + '"')
  62
  63     def append_patch_instruction(self, filename, patchname):
  64         """ Appends a patch instruction for this patch.
  65
  66             filename = file to patch
  67             patchname = patchfile to apply to file
  68
  69             if filename starts with distribution/extensions/.*/ this will add a
  70             patch-if instruction that will patch the file if the parent
  71             directory of the file exists. This was ported from
  72             mozilla/tools/update-packaging/common.sh's make_patch_instruction.
  73         """
  74         m = re.match("((?:|.*/)distribution/extensions/.*)/", filename)
  75         if m:
  76             testdir = m.group(1)
  77             print('   patch-if "' + testdir + '" "' + patchname + '" "' + filename + '"')
  78             self.manifestv2.append('patch-if "' + testdir + '" "' +
  79                                    patchname + '" "' + filename + '"')
  80             self.manifestv3.append('patch-if "' + testdir + '" "' +
  81                                    patchname + '" "' + filename + '"')
  82         else:
  83             print('      patch "' + patchname + '" "' + filename + '"')
  84             self.manifestv2.append('patch "' + patchname + '" "' + filename + '"')
  85             self.manifestv3.append('patch "' + patchname + '" "' + filename + '"')
  86
  87     def append_remove_instruction(self, filename):
  88         """ Appends an remove instruction for this patch.
  89             This was ported from
  90             mozilla/tools/update-packaging/common.sh/make_remove_instruction
  91         """
  92         if filename.endswith("/"):
  93             print('      rmdir "' + filename + '"')
  94             self.manifestv2.append('rmdir "' + filename + '"')
  95             self.manifestv3.append('rmdir "' + filename + '"')
  96         elif filename.endswith("/*"):
  97             filename = filename[:-1]
  98             print('    rmrfdir "' + filename + '"')
  99             self.manifestv2.append('rmrfdir "' + filename + '"')
 100             self.manifestv3.append('rmrfdir "' + filename + '"')
 101         else:
 102             print('     remove "' + filename + '"')
 103             self.manifestv2.append('remove "' + filename + '"')
 104             self.manifestv3.append('remove "' + filename + '"')
 105
 106     def create_manifest_files(self):
 107         """ Create the v2 manifest file in the root of the work_dir """
 108         manifest_file_path = os.path.join(self.work_dir, "updatev2.manifest")
 109         manifest_file = open(manifest_file_path, "wb")
 110         manifest_file.writelines(io.BytesIO(b"type \"partial\"\n"))
 111         manifest_file.writelines(io.BytesIO('\n'.join(self.manifestv2).encode('ascii')))
 112         manifest_file.writelines(io.BytesIO(b"\n"))
 113         manifest_file.close()
 114
 115         xz_file(manifest_file_path)
 116         self.archive_files.append('"updatev2.manifest"')
 117
 118         """ Create the v3 manifest file in the root of the work_dir """
 119         manifest_file_path = os.path.join(self.work_dir, "updatev3.manifest")
 120         manifest_file = open(manifest_file_path, "wb")
 121         manifest_file.writelines(io.BytesIO(b"type \"partial\"\n"))
 122         manifest_file.writelines(io.BytesIO('\n'.join(self.manifestv3).encode('ascii')))
 123         manifest_file.writelines(io.BytesIO(b"\n"))
 124         manifest_file.close()
 125
 126         xz_file(manifest_file_path)
 127         self.archive_files.append('"updatev3.manifest"')
 128
 129     def build_marfile_entry_hash(self, root_path):
 130         """ Iterates through the root_path, creating a MarFileEntry for each file
 131             and directory in that path.  Excludes any filenames in the file_exclusion_list
 132         """
 133         mar_entry_hash = {}
 134         filename_set = set()
 135         dirname_set = set()
 136         for root, dirs, files in os.walk(root_path):
 137             for name in files:
 138                 # filename is the relative path from root directory
 139                 partial_path = root[len(root_path) + 1:]
 140                 if name not in self.file_exclusion_list:
 141                     filename = os.path.join(partial_path, name)
 142                     if "/" + filename not in self.path_exclusion_list:
 143                         mar_entry_hash[filename] = MarFileEntry(root_path, filename)
 144                         filename_set.add(filename)
 145
 146             for name in dirs:
 147                 # dirname is the relative path from root directory
 148                 partial_path = root[len(root_path) + 1:]
 149                 if name not in self.file_exclusion_list:
 150                     dirname = os.path.join(partial_path, name)
 151                     if "/" + dirname not in self.path_exclusion_list:
 152                         dirname = dirname + "/"
 153                         mar_entry_hash[dirname] = MarFileEntry(root_path, dirname)
 154                         dirname_set.add(dirname)
 155
 156         return mar_entry_hash, filename_set, dirname_set
 157
 158
 159 class MarFileEntry:
 160     """Represents a file inside a Mozilla Archive Format (MAR)
 161         abs_path = abspath to the the file
 162         name =  relative path within the mar.  e.g.
 163           foo.mar/dir/bar.txt extracted into /tmp/foo:
 164             abs_path=/tmp/foo/dir/bar.txt
 165             name = dir/bar.txt
 166     """
 167
 168     def __init__(self, root, name):
 169         """root = path the the top of the mar
 170            name = relative path within the mar"""
 171         self.name = name.replace("\\", "/")
 172         self.abs_path = os.path.join(root, name)
 173         self.sha_cache = None
 174
 175     def __str__(self):
 176         return 'Name: %s FullPath: %s' % (self.name, self.abs_path)
 177
 178     def calc_file_sha_digest(self, filename):
 179         """ Returns sha digest of given filename"""
 180         file_content = open(filename, 'rb').read()
 181         return hashlib.sha1(file_content).digest()
 182
 183     def sha(self):
 184         """ Returns sha digest of file repreesnted by this _marfile_entry\x10"""
 185         if not self.sha_cache:
 186             self.sha_cache = self.calc_file_sha_digest(self.abs_path)
 187         return self.sha_cache
 188
 189
 190 def exec_shell_cmd(cmd):
 191     """Execs shell cmd and raises an exception if the cmd fails"""
 192     if (os.system(cmd)):
 193         raise Exception("cmd failed " + cmd)
 194
 195
 196 def copy_file(src_file_abs_path, dst_file_abs_path):
 197     """ Copies src to dst creating any parent dirs required in dst first """
 198     dst_file_dir = os.path.dirname(dst_file_abs_path)
 199     if not os.path.exists(dst_file_dir):
 200         os.makedirs(dst_file_dir)
 201     # Copy the file over
 202     shutil.copy2(src_file_abs_path, dst_file_abs_path)
 203
 204
 205 def xz_file(filename):
 206     """ XZ compresses the file in place.  The original file is replaced
 207         with the xz compressed version of itself assumes the path is absolute"""
 208     exec_shell_cmd('xz --compress --x86 --lzma2 --format=xz --check=crc64 "' + filename + '"')
 209     os.rename(filename + ".xz", filename)
 210
 211
 212 def xzunzip_file(filename):
 213     """ xz decompresses the file in palce.  The original file is replaced
 214         with a xz decompressed version of itself. doesn't matter if the
 215         filename ends in .xz or not"""
 216     if not filename.endswith(".xz"):
 217         os.rename(filename, filename + ".xz")
 218         filename = filename + ".xz"
 219     exec_shell_cmd('xz -d "' + filename + '"')
 220
 221
 222 def extract_mar(filename, work_dir):
 223     """ Extracts the marfile intot he work_dir
 224         assumes work_dir already exists otherwise will throw osError"""
 225     print("Extracting " + filename + " to " + work_dir)
 226     saved_path = os.getcwd()
 227     try:
 228         os.chdir(work_dir)
 229         exec_shell_cmd("mar -x " + filename)
 230     finally:
 231         os.chdir(saved_path)
 232
 233
 234 def create_partial_patch_for_file(from_marfile_entry, to_marfile_entry, shas, patch_info):
 235     """ Creates the partial patch file and manifest entry for the pair of files passed in
 236     """
 237     if not (from_marfile_entry.sha(), to_marfile_entry.sha()) in shas:
 238         print('diffing "' + from_marfile_entry.name + '\"')
 239         # bunzip to/from
 240         xzunzip_file(from_marfile_entry.abs_path)
 241         xzunzip_file(to_marfile_entry.abs_path)
 242
 243         # The patch file will be created in the working directory with the
 244         # name of the file in the mar + .patch
 245         patch_file_abs_path = os.path.join(patch_info.work_dir, from_marfile_entry.name + ".patch")
 246         patch_file_dir = os.path.dirname(patch_file_abs_path)
 247         if not os.path.exists(patch_file_dir):
 248             os.makedirs(patch_file_dir)
 249
 250         # Create xz compressed patch file
 251         exec_shell_cmd("mbsdiff " + from_marfile_entry.abs_path + " " +
 252                        to_marfile_entry.abs_path + " " + patch_file_abs_path)
 253         xz_file(patch_file_abs_path)
 254
 255         # Create xz compressed full file
 256         full_file_abs_path = os.path.join(patch_info.work_dir, to_marfile_entry.name)
 257         shutil.copy2(to_marfile_entry.abs_path, full_file_abs_path)
 258         xz_file(full_file_abs_path)
 259
 260         if os.path.getsize(patch_file_abs_path) < os.path.getsize(full_file_abs_path):
 261             # Patch is smaller than file.  Remove the file and add patch to manifest
 262             os.remove(full_file_abs_path)
 263             file_in_manifest_name = from_marfile_entry.name + ".patch"
 264             file_in_manifest_abspath = patch_file_abs_path
 265             patch_info.append_patch_instruction(to_marfile_entry.name, file_in_manifest_name)
 266         else:
 267             # File is smaller than patch.  Remove the patch and add file to manifest
 268             os.remove(patch_file_abs_path)
 269             file_in_manifest_name = from_marfile_entry.name
 270             file_in_manifest_abspath = full_file_abs_path
 271             patch_info.append_add_instruction(file_in_manifest_name)
 272
 273         shas[from_marfile_entry.sha(), to_marfile_entry.sha()] = (
 274             file_in_manifest_name, file_in_manifest_abspath)
 275         patch_info.archive_files.append('"' + file_in_manifest_name + '"')
 276     else:
 277         filename, src_file_abs_path = shas[from_marfile_entry.sha(), to_marfile_entry.sha()]
 278         # We've already calculated the patch for this pair of files.
 279         if (filename.endswith(".patch")):
 280             # print "skipping diff: "+from_marfile_entry.name
 281             # Patch was smaller than file - add patch instruction to manifest
 282             file_in_manifest_name = to_marfile_entry.name + '.patch'
 283             patch_info.append_patch_instruction(to_marfile_entry.name, file_in_manifest_name)
 284         else:
 285             # File was smaller than file - add file to manifest
 286             file_in_manifest_name = to_marfile_entry.name
 287             patch_info.append_add_instruction(file_in_manifest_name)
 288         # Copy the pre-calculated file into our new patch work aread
 289         copy_file(src_file_abs_path, os.path.join(patch_info.work_dir, file_in_manifest_name))
 290         patch_info.archive_files.append('"' + file_in_manifest_name + '"')
 291
 292
 293 def create_add_patch_for_file(to_marfile_entry, patch_info):
 294     """  Copy the file to the working dir, add the add instruction,
 295          and add it to the list of archive files """
 296     copy_file(to_marfile_entry.abs_path, os.path.join(patch_info.work_dir, to_marfile_entry.name))
 297     patch_info.append_add_instruction(to_marfile_entry.name)
 298     patch_info.archive_files.append('"' + to_marfile_entry.name + '"')
 299
 300
 301 def create_add_if_not_patch_for_file(to_marfile_entry, patch_info):
 302     """  Copy the file to the working dir, add the add-if-not instruction,
 303          and add it to the list of archive files """
 304     copy_file(to_marfile_entry.abs_path, os.path.join(patch_info.work_dir, to_marfile_entry.name))
 305     patch_info.append_add_if_not_instruction(to_marfile_entry.name)
 306     patch_info.archive_files.append('"' + to_marfile_entry.name + '"')
 307
 308
 309 def process_explicit_remove_files(dir_path, patch_info):
 310     """ Looks for a 'removed-files' file in the dir_path.  If the removed-files does not exist
 311     this will throw.  If found adds the removed-files
 312     found in that file to the patch_info"""
 313
 314     # Windows and linux have this file at the root of the dir
 315     list_file_path = os.path.join(dir_path, "removed-files")
 316     if not os.path.exists(list_file_path):
 317         list_file_path = os.path.join(dir_path, "Contents/Resources/removed-files")
 318
 319     if (os.path.exists(list_file_path)):
 320         fd, tmppath = tempfile.mkstemp('', 'tmp', os.getcwd())
 321         os.close(fd)
 322         exec_shell_cmd('xz -k -d --stdout "' + list_file_path + '" > "' + tmppath + '"')
 323         list_file = open(tmppath)
 324
 325         lines = []
 326         for line in list_file:
 327             lines.append(line.strip())
 328
 329         list_file.close()
 330         os.remove(tmppath)
 331         lines.sort(reverse=True)
 332         for line in lines:
 333             # Exclude any blank and comment lines.
 334             if line and not line.startswith("#"):
 335                 # Python on windows uses \ for path separators and the update
 336                 # manifests expects / for path separators on all platforms.
 337                 line = line.replace("\\", "/")
 338                 patch_info.append_remove_instruction(line)
 339
 340
 341 def create_partial_patch(from_dir_path, to_dir_path, patch_filename,
 342                          shas, patch_info, forced_updates, add_if_not_list):
 343     """ Builds a partial patch by comparing the files in from_dir_path to those of to_dir_path"""
 344     # Cannocolize the paths for safey
 345     from_dir_path = os.path.abspath(from_dir_path)
 346     to_dir_path = os.path.abspath(to_dir_path)
 347     # Create a hashtable of the from  and to directories
 348     from_dir_hash, from_file_set, from_dir_set = patch_info.build_marfile_entry_hash(from_dir_path)
 349     to_dir_hash, to_file_set, to_dir_set = patch_info.build_marfile_entry_hash(to_dir_path)
 350     # Create a list of the forced updates
 351     forced_list = forced_updates.strip().split('|')
 352     # Require that the precomplete file is included in the complete update
 353     if "precomplete" in to_file_set:
 354         forced_list.append("precomplete")
 355     elif "Contents/Resources/precomplete" in to_file_set:
 356         forced_list.append("Contents/Resources/precomplete")
 357     # The check with \ file separators allows tests for Mac to run on Windows
 358     elif "Contents\Resources\precomplete" in to_file_set:
 359         forced_list.append("Contents\Resources\precomplete")
 360     else:
 361         raise Exception("missing precomplete file in: " + to_dir_path)
 362
 363     if "removed-files" in to_file_set:
 364         forced_list.append("removed-files")
 365     elif "Contents/Resources/removed-files" in to_file_set:
 366         forced_list.append("Contents/Resources/removed-files")
 367     # The check with \ file separators allows tests for Mac to run on Windows
 368     elif "Contents\Resources\\removed-files" in to_file_set:
 369         forced_list.append("Contents\Resources\\removed-files")
 370     else:
 371         raise Exception("missing removed-files file in: " + to_dir_path)
 372
 373     # Files which exist in both sets need to be patched
 374     patch_filenames = list(from_file_set.intersection(to_file_set))
 375     patch_filenames.sort(reverse=True)
 376     for filename in patch_filenames:
 377         from_marfile_entry = from_dir_hash[filename]
 378         to_marfile_entry = to_dir_hash[filename]
 379         if os.path.basename(filename) in add_if_not_list:
 380             # This filename is in the add if not list, explicitly add-if-not
 381             create_add_if_not_patch_for_file(to_dir_hash[filename], patch_info)
 382         elif filename in forced_list:
 383             print('Forcing "' + filename + '"')
 384             # This filename is in the forced list, explicitly add
 385             create_add_patch_for_file(to_dir_hash[filename], patch_info)
 386         else:
 387             if from_marfile_entry.sha() != to_marfile_entry.sha():
 388                 # Not the same - calculate a patch
 389                 create_partial_patch_for_file(
 390                     from_marfile_entry, to_marfile_entry, shas, patch_info)
 391
 392     # files in to_dir not in from_dir need to added
 393     add_filenames = list(to_file_set - from_file_set)
 394     add_filenames.sort(reverse=True)
 395     for filename in add_filenames:
 396         if os.path.basename(filename) in add_if_not_list:
 397             create_add_if_not_patch_for_file(to_dir_hash[filename], patch_info)
 398         else:
 399             create_add_patch_for_file(to_dir_hash[filename], patch_info)
 400
 401     # files in from_dir not in to_dir need to be removed
 402     remove_filenames = list(from_file_set - to_file_set)
 403     remove_filenames.sort(reverse=True)
 404     for filename in remove_filenames:
 405         patch_info.append_remove_instruction(from_dir_hash[filename].name)
 406
 407     process_explicit_remove_files(to_dir_path, patch_info)
 408
 409     # directories in from_dir not in to_dir need to be removed
 410     remove_dirnames = list(from_dir_set - to_dir_set)
 411     remove_dirnames.sort(reverse=True)
 412     for dirname in remove_dirnames:
 413         patch_info.append_remove_instruction(from_dir_hash[dirname].name)
 414
 415     # Construct the Manifest files
 416     patch_info.create_manifest_files()
 417
 418     # And construct the mar
 419     mar_cmd = 'mar -C ' + patch_info.work_dir + \
 420         ' -c output.mar ' + ' '.join(patch_info.archive_files)
 421     exec_shell_cmd(mar_cmd)
 422
 423     # Copy mar to final destination
 424     patch_file_dir = os.path.split(patch_filename)[0]
 425     if not os.path.exists(patch_file_dir):
 426         os.makedirs(patch_file_dir)
 427     shutil.copy2(os.path.join(patch_info.work_dir, "output.mar"), patch_filename)
 428
 429     return patch_filename
 430
 431
 432 def usage():
 433     print("-h for help")
 434     print("-f for patchlist_file")
 435
 436
 437 def get_buildid(work_dir):
 438     """ extracts buildid from MAR
 439     """
 440     ini = '%s/application.ini' % work_dir
 441     if not os.path.exists(ini):
 442         ini = '%s/Contents/Resources/application.ini' % work_dir
 443         if not os.path.exists(ini):
 444             print('WARNING: application.ini not found, cannot find build ID')
 445             return ''
 446
 447     fd, tmppath = tempfile.mkstemp('', 'tmp', os.getcwd())
 448     os.close(fd)
 449     exec_shell_cmd('xz -k -d --stdout "' + ini + '" > "' + tmppath + '"')
 450     file = open(tmppath)
 451     for line in file:
 452         if line.find('BuildID') == 0:
 453             file.close()
 454             os.remove(tmppath)
 455             return line.strip().split('=')[1]
 456     print('WARNING: cannot find build ID in application.ini')
 457     file.close()
 458     os.remove(tmppath)
 459     return ''
 460
 461
 462 def decode_filename(filepath):
 463     """ Breaks filename/dir structure into component parts based on regex
 464         for example: firefox-3.0b3pre.en-US.linux-i686.complete.mar
 465         Or linux-i686/en-US/firefox-3.0b3.complete.mar
 466         Returns dict with keys product, version, locale, platform, type
 467     """
 468     try:
 469         m = re.search(
 470             '(?P<product>\w+)(-)(?P<version>\w+\.\w+(\.\w+){0,2})(\.)(?P<locale>.+?)(\.)(?P<platform>.+?)(\.)(?P<type>\w+)(.mar)',  # NOQA: E501
 471             os.path.basename(filepath))
 472         return m.groupdict()
 473     except Exception as exc:
 474         try:
 475             m = re.search(
 476                 '(?P<platform>.+?)\/(?P<locale>.+?)\/(?P<product>\w+)-(?P<version>\w+\.\w+)\.(?P<type>\w+).mar',  # NOQA: E501
 477                 filepath)
 478             return m.groupdict()
 479         except Exception:
 480             raise Exception("could not parse filepath %s: %s" % (filepath, exc))
 481
 482
 483 def create_partial_patches(patches):
 484     """ Given the patches generates a set of partial patches"""
 485     shas = {}
 486
 487     work_dir_root = None
 488     metadata = []
 489     try:
 490         work_dir_root = tempfile.mkdtemp('-fastmode', 'tmp', os.getcwd())
 491         print("Building patches using work dir: %s" % (work_dir_root))
 492
 493         # Iterate through every patch set in the patch file
 494         patch_num = 1
 495         for patch in patches:
 496             startTime = time.time()
 497
 498             from_filename, to_filename, patch_filename, forced_updates = patch.split(",")
 499             from_filename, to_filename, patch_filename = os.path.abspath(
 500                 from_filename), os.path.abspath(to_filename), os.path.abspath(patch_filename)
 501
 502             # Each patch iteration uses its own work dir
 503             work_dir = os.path.join(work_dir_root, str(patch_num))
 504             os.mkdir(work_dir)
 505
 506             # Extract from mar into from dir
 507             work_dir_from = os.path.join(work_dir, "from")
 508             os.mkdir(work_dir_from)
 509             extract_mar(from_filename, work_dir_from)
 510             from_decoded = decode_filename(from_filename)
 511             from_buildid = get_buildid(work_dir_from)
 512             from_shasum = hashlib.sha1(open(from_filename, "rb").read()).hexdigest()
 513             from_size = str(os.path.getsize(to_filename))
 514
 515             # Extract to mar into to dir
 516             work_dir_to = os.path.join(work_dir, "to")
 517             os.mkdir(work_dir_to)
 518             extract_mar(to_filename, work_dir_to)
 519             to_decoded = decode_filename(from_filename)
 520             to_buildid = get_buildid(work_dir_to)
 521             to_shasum = hashlib.sha1(open(to_filename, 'rb').read()).hexdigest()
 522             to_size = str(os.path.getsize(to_filename))
 523
 524             mar_extract_time = time.time()
 525
 526             partial_filename = create_partial_patch(work_dir_from, work_dir_to, patch_filename,
 527                                                     shas, PatchInfo(work_dir, [
 528                                                         'update.manifest',
 529                                                         'updatev2.manifest',
 530                                                         'updatev3.manifest'
 531                                                     ], []),
 532                                                     forced_updates,
 533                                                     ['channel-prefs.js', 'update-settings.ini'])
 534             partial_shasum = hashlib.sha1(open(partial_filename, "rb").read()).hexdigest()
 535             partial_size = str(os.path.getsize(partial_filename))
 536
 537             metadata.append({
 538                 'to_filename': os.path.basename(to_filename),
 539                 'from_filename': os.path.basename(from_filename),
 540                 'partial_filename': os.path.basename(partial_filename),
 541                 'to_buildid': to_buildid,
 542                 'from_buildid': from_buildid,
 543                 'to_sha1sum': to_shasum,
 544                 'from_sha1sum': from_shasum,
 545                 'partial_sha1sum': partial_shasum,
 546                 'to_size': to_size,
 547                 'from_size': from_size,
 548                 'partial_size': partial_size,
 549                 'to_version': to_decoded['version'],
 550                 'from_version': from_decoded['version'],
 551                 'locale': from_decoded['locale'],
 552                 'platform': from_decoded['platform'],
 553             })
 554             print("done with patch %s/%s time (%.2fs/%.2fs/%.2fs) (mar/patch/total)" % (str(patch_num),  # NOQA: E501
 555                                                                                         str(len(patches)), mar_extract_time - startTime, time.time() - mar_extract_time, time.time() - startTime))  # NOQA: E501
 556             patch_num += 1
 557         return metadata
 558     finally:
 559         # If we fail or get a ctrl-c during run be sure to clean up temp dir
 560         if (work_dir_root and os.path.exists(work_dir_root)):
 561             shutil.rmtree(work_dir_root)
 562
 563
 564 def main(argv):
 565     patchlist_file = None
 566     try:
 567         opts, args = getopt.getopt(argv, "hf:", ["help", "patchlist_file="])
 568         for opt, arg in opts:
 569             if opt in ("-h", "--help"):
 570                 usage()
 571                 sys.exit()
 572             elif opt in ("-f", "--patchlist_file"):
 573                 patchlist_file = arg
 574     except getopt.GetoptError:
 575         usage()
 576         sys.exit(2)
 577
 578     if not patchlist_file:
 579         usage()
 580         sys.exit(2)
 581
 582     patches = []
 583     f = open(patchlist_file, 'r')
 584     for line in f.readlines():
 585         patches.append(line)
 586     f.close()
 587     create_partial_patches(patches)
 588
 589
 590 if __name__ == "__main__":
 591     main(sys.argv[1:])