tools/update-packaging/make_incremental_updates.py

   1 # This Source Code Form is subject to the terms of the Mozilla Public
   2 # License, v. 2.0. If a copy of the MPL was not distributed with this
   3 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
   4
   5 import os
   6 import shutil
   7 import sha
   8 from os.path import join, getsize
   9 from stat import *
  10 import re
  11 import sys
  12 import getopt
  13 import time
  14 import datetime
  15 import bz2
  16 import string
  17 import tempfile
  18
  19 class PatchInfo:
  20     """ Represents the meta-data associated with a patch
  21         work_dir = working dir where files are stored for this patch
  22         archive_files = list of files to include in this patch
  23         manifestv1 = set of manifest version 1 patch instructions
  24         manifestv2 = set of manifest version 2 patch instructions
  25         file_exclusion_list =
  26         files to exclude from this patch. names without slashes will be
  27                               excluded anywhere in the directory hiearchy.   names with slashes
  28                               will only be excluded at that exact path
  29         """
  30     def __init__(self, work_dir, file_exclusion_list, path_exclusion_list):
  31         self.work_dir=work_dir
  32         self.archive_files=[]
  33         self.manifestv1=[]
  34         self.manifestv2=[]
  35         self.file_exclusion_list=file_exclusion_list
  36         self.path_exclusion_list=path_exclusion_list
  37
  38     def append_add_instruction(self, filename):
  39         """ Appends an add instruction for this patch.
  40             if the filename starts with distribution/extensions/ adds an add-if
  41             instruction to test the existence of the subdirectory.  This was
  42             ported from mozilla/tools/update-packaging/common.sh's
  43             make_add_instruction.
  44         """
  45         m = re.match("((?:|.*/)distribution/extensions)/", filename)
  46         if m:
  47             # Directory immediately following extensions is used for the test
  48             testdir = m.group(1)
  49             self.manifestv1.append('add-if "'+testdir+'" "'+filename+'"')
  50             self.manifestv2.append('add-if "'+testdir+'" "'+filename+'"')
  51         else:
  52             self.manifestv1.append('add "'+filename+'"')
  53             self.manifestv2.append('add "'+filename+'"')
  54
  55     def append_patch_instruction(self, filename, patchname):
  56         """ Appends an patch instruction for this patch.
  57
  58             filename = file to patch
  59             patchname = patchfile to apply to file
  60
  61             if the filename starts with distribution/extensions/ adds a
  62             patch-if instruction to test the existence of the subdirectory.
  63             This was ported from
  64             mozilla/tools/update-packaging/common.sh's make_patch_instruction.
  65         """
  66         m = re.match("((?:|.*/)distribution/extensions)/", filename)
  67         if m:
  68             testdir = m.group(1)
  69             self.manifestv1.append('patch-if "'+testdir+'" "'+patchname+'" "'+filename+'"')
  70             self.manifestv2.append('patch-if "'+testdir+'" "'+patchname+'" "'+filename+'"')
  71         else:
  72             self.manifestv1.append('patch "'+patchname+'" "'+filename+'"')
  73             self.manifestv2.append('patch "'+patchname+'" "'+filename+'"')
  74
  75     def append_remove_instruction(self, filename):
  76         """ Appends an remove instruction for this patch.
  77             This was ported from
  78             mozilla/tools/update-packaging/common.sh/make_remove_instruction
  79         """
  80         if filename.endswith("/"):
  81             self.manifestv2.append('rmdir "'+filename+'"')
  82         elif filename.endswith("/*"):
  83             filename = filename[:-1]
  84             self.manifestv2.append('rmrfdir "'+filename+'"')
  85         else:
  86             self.manifestv1.append('remove "'+filename+'"')
  87             self.manifestv2.append('remove "'+filename+'"')
  88
  89     def create_manifest_files(self):
  90         """ Createst the v1 manifest file in the root of the work_dir """
  91         manifest_file_path = os.path.join(self.work_dir,"update.manifest")
  92         manifest_file = open(manifest_file_path, "wb")
  93         manifest_file.writelines(string.join(self.manifestv1, '\n'))
  94         manifest_file.writelines("\n")
  95         manifest_file.close()
  96
  97         bzip_file(manifest_file_path)
  98         self.archive_files.append('"update.manifest"')
  99
 100         """ Createst the v2 manifest file in the root of the work_dir """
 101         manifest_file_path = os.path.join(self.work_dir,"updatev2.manifest")
 102         manifest_file = open(manifest_file_path, "wb")
 103         manifest_file.writelines("type \"partial\"\n")
 104         manifest_file.writelines(string.join(self.manifestv2, '\n'))
 105         manifest_file.writelines("\n")
 106         manifest_file.close()
 107
 108         bzip_file(manifest_file_path)
 109         self.archive_files.append('"updatev2.manifest"')
 110
 111     def build_marfile_entry_hash(self, root_path):
 112         """ Iterates through the root_path, creating a MarFileEntry for each file
 113             and directory in that path.  Excludes any filenames in the file_exclusion_list
 114         """
 115         mar_entry_hash = {}
 116         filename_set = set()
 117         dirname_set = set()
 118         for root, dirs, files in os.walk(root_path):
 119             for name in files:
 120                 # filename is the relative path from root directory
 121                 partial_path = root[len(root_path)+1:]
 122                 if name not in self.file_exclusion_list:
 123                     filename = os.path.join(partial_path, name)
 124                     if "/"+filename not in self.path_exclusion_list:
 125                         mar_entry_hash[filename]=MarFileEntry(root_path, filename)
 126                         filename_set.add(filename)
 127
 128             for name in dirs:
 129                 # dirname is the relative path from root directory
 130                 partial_path = root[len(root_path)+1:]
 131                 if name not in self.file_exclusion_list:
 132                     dirname = os.path.join(partial_path, name)
 133                     if "/"+dirname not in self.path_exclusion_list:
 134                         dirname = dirname+"/"
 135                         mar_entry_hash[dirname]=MarFileEntry(root_path, dirname)
 136                         dirname_set.add(dirname)
 137
 138         return mar_entry_hash, filename_set, dirname_set
 139
 140
 141 class MarFileEntry:
 142     """Represents a file inside a Mozilla Archive Format (MAR)
 143         abs_path = abspath to the the file
 144         name =  relative path within the mar.  e.g.
 145           foo.mar/dir/bar.txt extracted into /tmp/foo:
 146             abs_path=/tmp/foo/dir/bar.txt
 147             name = dir/bar.txt
 148     """
 149     def __init__(self, root, name):
 150         """root = path the the top of the mar
 151            name = relative path within the mar"""
 152         self.name=name.replace("\\", "/")
 153         self.abs_path=os.path.join(root,name)
 154         self.sha_cache=None
 155
 156     def __str__(self):
 157         return 'Name: %s FullPath: %s' %(self.name,self.abs_path)
 158
 159     def calc_file_sha_digest(self, filename):
 160         """ Returns sha digest of given filename"""
 161         file_content = open(filename, 'r').read()
 162         return sha.new(file_content).digest()
 163
 164     def sha(self):
 165         """ Returns sha digest of file repreesnted by this _marfile_entry\x10"""
 166         if not self.sha_cache:
 167             self.sha_cache=self.calc_file_sha_digest(self.abs_path)
 168         return self.sha_cache
 169
 170 def exec_shell_cmd(cmd):
 171     """Execs shell cmd and raises an exception if the cmd fails"""
 172     if (os.system(cmd)):
 173         raise Exception, "cmd failed "+cmd
 174
 175
 176 def copy_file(src_file_abs_path, dst_file_abs_path):
 177     """ Copies src to dst creating any parent dirs required in dst first """
 178     dst_file_dir=os.path.dirname(dst_file_abs_path)
 179     if not os.path.exists(dst_file_dir):
 180          os.makedirs(dst_file_dir)
 181     # Copy the file over
 182     shutil.copy2(src_file_abs_path, dst_file_abs_path)
 183
 184 def bzip_file(filename):
 185     """ Bzip's the file in place.  The original file is replaced with a bzip'd version of itself
 186         assumes the path is absolute"""
 187     exec_shell_cmd('bzip2 -z9 "' + filename+'"')
 188     os.rename(filename+".bz2",filename)
 189
 190 def bunzip_file(filename):
 191     """ Bzip's the file in palce.   The original file is replaced with a bunzip'd version of itself.
 192         doesn't matter if the filename ends in .bz2 or not"""
 193     if not filename.endswith(".bz2"):
 194         os.rename(filename, filename+".bz2")
 195         filename=filename+".bz2"
 196     exec_shell_cmd('bzip2 -d "' + filename+'"')
 197
 198
 199 def extract_mar(filename, work_dir):
 200     """ Extracts the marfile intot he work_dir
 201         assumes work_dir already exists otherwise will throw osError"""
 202     print "Extracting "+filename+" to "+work_dir
 203     saved_path = os.getcwd()
 204     try:
 205         os.chdir(work_dir)
 206         exec_shell_cmd("mar -x "+filename)
 207     finally:
 208         os.chdir(saved_path)
 209
 210 def create_partial_patch_for_file(from_marfile_entry, to_marfile_entry, shas, patch_info):
 211     """ Creates the partial patch file and manifest entry for the pair of files passed in
 212     """
 213     if not (from_marfile_entry.sha(),to_marfile_entry.sha()) in shas:
 214         print "diffing: " + from_marfile_entry.name
 215
 216         #bunzip to/from
 217         bunzip_file(from_marfile_entry.abs_path)
 218         bunzip_file(to_marfile_entry.abs_path)
 219
 220         # The patch file will be created in the working directory with the
 221         # name of the file in the mar + .patch
 222         patch_file_abs_path = os.path.join(patch_info.work_dir,from_marfile_entry.name+".patch")
 223         patch_file_dir=os.path.dirname(patch_file_abs_path)
 224         if not os.path.exists(patch_file_dir):
 225             os.makedirs(patch_file_dir)
 226
 227         # Create bzip'd patch file
 228         exec_shell_cmd("mbsdiff "+from_marfile_entry.abs_path+" "+to_marfile_entry.abs_path+" "+patch_file_abs_path)
 229         bzip_file(patch_file_abs_path)
 230
 231         # Create bzip's full file
 232         full_file_abs_path =  os.path.join(patch_info.work_dir, to_marfile_entry.name)
 233         shutil.copy2(to_marfile_entry.abs_path, full_file_abs_path)
 234         bzip_file(full_file_abs_path)
 235
 236         ## TOODO NEED TO ADD HANDLING FOR FORCED UPDATES
 237         if os.path.getsize(patch_file_abs_path) < os.path.getsize(full_file_abs_path):
 238             # Patch is smaller than file.  Remove the file and add patch to manifest
 239             os.remove(full_file_abs_path)
 240             file_in_manifest_name = from_marfile_entry.name+".patch"
 241             file_in_manifest_abspath = patch_file_abs_path
 242             patch_info.append_patch_instruction(to_marfile_entry.name, file_in_manifest_name)
 243         else:
 244             # File is smaller than patch.  Remove the patch and add file to manifest
 245             os.remove(patch_file_abs_path)
 246             file_in_manifest_name = from_marfile_entry.name
 247             file_in_manifest_abspath = full_file_abs_path
 248             patch_info.append_add_instruction(file_in_manifest_name)
 249
 250         shas[from_marfile_entry.sha(),to_marfile_entry.sha()] = (file_in_manifest_name,file_in_manifest_abspath)
 251         patch_info.archive_files.append('"'+file_in_manifest_name+'"')
 252     else:
 253         filename, src_file_abs_path = shas[from_marfile_entry.sha(),to_marfile_entry.sha()]
 254         # We've already calculated the patch for this pair of files.
 255         if (filename.endswith(".patch")):
 256             print "skipping diff: " + from_marfile_entry.name
 257             # Patch was smaller than file - add patch instruction to manifest
 258             file_in_manifest_name = to_marfile_entry.name+'.patch';
 259             patch_info.append_patch_instruction(to_marfile_entry.name, file_in_manifest_name)
 260         else:
 261             # File was smaller than file - add file to manifest
 262             file_in_manifest_name = to_marfile_entry.name
 263             patch_info.append_add_instruction(file_in_manifest_name)
 264         # Copy the pre-calculated file into our new patch work aread
 265         copy_file(src_file_abs_path, os.path.join(patch_info.work_dir, file_in_manifest_name))
 266         patch_info.archive_files.append('"'+file_in_manifest_name+'"')
 267
 268 def create_add_patch_for_file(to_marfile_entry, patch_info):
 269     """  Copy the file to the working dir, add the add instruction, and add it to the list of archive files """
 270     print "Adding New File " + to_marfile_entry.name
 271     copy_file(to_marfile_entry.abs_path, os.path.join(patch_info.work_dir, to_marfile_entry.name))
 272     patch_info.append_add_instruction(to_marfile_entry.name)
 273     patch_info.archive_files.append('"'+to_marfile_entry.name+'"')
 274
 275 def process_explicit_remove_files(dir_path, patch_info):
 276     """ Looks for a 'removed-files' file in the dir_path.  If the removed-files does not exist
 277     this will throw.  If found adds the removed-files
 278     found in that file to the patch_info"""
 279
 280     # Windows and linux have this file at the root of the dir
 281     list_file_path = os.path.join(dir_path, "removed-files")
 282     prefix=""
 283     if not os.path.exists(list_file_path):
 284         # On Mac removed-files contains relative paths from Contents/MacOS/
 285         prefix= "Contents/MacOS"
 286         list_file_path = os.path.join(dir_path, prefix+"/removed-files")
 287
 288     if (os.path.exists(list_file_path)):
 289         list_file = bz2.BZ2File(list_file_path,"r") # throws if doesn't exist
 290
 291         lines = []
 292         for line in list_file:
 293             lines.append(line.strip())
 294
 295         lines.sort(reverse=True)
 296         for line in lines:
 297             # Exclude any blank and comment lines.
 298             if line and not line.startswith("#"):
 299                 if prefix != "":
 300                     if line.startswith("../"):
 301                         line = line.replace("../../", "")
 302                         line = line.replace("../", "Contents/")
 303                     else:
 304                         line = os.path.join(prefix,line)
 305                 # Python on windows uses \ for path separators and the update
 306                 # manifests expects / for path separators on all platforms.
 307                 line = line.replace("\\", "/")
 308                 patch_info.append_remove_instruction(line)
 309
 310 def create_partial_patch(from_dir_path, to_dir_path, patch_filename, shas, patch_info, forced_updates):
 311     """ Builds a partial patch by comparing the files in from_dir_path to those of to_dir_path"""
 312     # Cannocolize the paths for safey
 313     from_dir_path = os.path.abspath(from_dir_path)
 314     to_dir_path = os.path.abspath(to_dir_path)
 315     # Create a hashtable of the from  and to directories
 316     from_dir_hash,from_file_set,from_dir_set = patch_info.build_marfile_entry_hash(from_dir_path)
 317     to_dir_hash,to_file_set,to_dir_set = patch_info.build_marfile_entry_hash(to_dir_path)
 318     # Require that the precomplete file is included in the to complete update
 319     if "precomplete" not in to_file_set:
 320         raise Exception, "missing precomplete file in: "+to_dir_path
 321     # Create a list of the forced updates
 322     forced_list = forced_updates.strip().split('|')
 323     forced_list.append("precomplete")
 324
 325     # Files which exist in both sets need to be patched
 326     patch_filenames = list(from_file_set.intersection(to_file_set))
 327     patch_filenames.sort(reverse=True)
 328     for filename in patch_filenames:
 329         from_marfile_entry = from_dir_hash[filename]
 330         to_marfile_entry = to_dir_hash[filename]
 331         if filename in forced_list:
 332             print "Forcing "+ filename
 333             # This filename is in the forced list, explicitly add
 334             create_add_patch_for_file(to_dir_hash[filename], patch_info)
 335         else:
 336           if from_marfile_entry.sha() != to_marfile_entry.sha():
 337               # Not the same - calculate a patch
 338               create_partial_patch_for_file(from_marfile_entry, to_marfile_entry, shas, patch_info)
 339
 340     # files in to_dir not in from_dir need to added
 341     add_filenames = list(to_file_set - from_file_set)
 342     add_filenames.sort(reverse=True)
 343     for filename in add_filenames:
 344         create_add_patch_for_file(to_dir_hash[filename], patch_info)
 345
 346     # files in from_dir not in to_dir need to be removed
 347     remove_filenames = list(from_file_set - to_file_set)
 348     remove_filenames.sort(reverse=True)
 349     for filename in remove_filenames:
 350         patch_info.append_remove_instruction(from_dir_hash[filename].name)
 351
 352     process_explicit_remove_files(to_dir_path, patch_info)
 353
 354     # directories in from_dir not in to_dir need to be removed
 355     remove_dirnames = list(from_dir_set - to_dir_set)
 356     remove_dirnames.sort(reverse=True)
 357     for dirname in remove_dirnames:
 358         patch_info.append_remove_instruction(from_dir_hash[dirname].name)
 359
 360     # Construct the Manifest files
 361     patch_info.create_manifest_files()
 362
 363     # And construct the mar
 364     mar_cmd = 'mar -C '+patch_info.work_dir+' -c output.mar '+string.join(patch_info.archive_files, ' ')
 365     exec_shell_cmd(mar_cmd)
 366
 367     # Copy mar to final destination
 368     patch_file_dir = os.path.split(patch_filename)[0]
 369     if not os.path.exists(patch_file_dir):
 370         os.makedirs(patch_file_dir)
 371     shutil.copy2(os.path.join(patch_info.work_dir,"output.mar"), patch_filename)
 372     return patch_filename
 373
 374 def usage():
 375     print "-h for help"
 376     print "-f for patchlist_file"
 377
 378 def get_buildid(work_dir, platform):
 379     """ extracts buildid from MAR
 380         TODO: this should handle 1.8 branch too
 381     """
 382     if platform == 'mac':
 383       ini = '%s/Contents/MacOS/application.ini' % work_dir
 384     else:
 385       ini = '%s/application.ini' % work_dir
 386     if not os.path.exists(ini):
 387         print 'WARNING: application.ini not found, cannot find build ID'
 388         return ''
 389     file = bz2.BZ2File(ini)
 390     for line in file:
 391       if line.find('BuildID') == 0:
 392         return line.strip().split('=')[1]
 393     print 'WARNING: cannot find build ID in application.ini'
 394     return ''
 395
 396 def decode_filename(filepath):
 397     """ Breaks filename/dir structure into component parts based on regex
 398         for example: firefox-3.0b3pre.en-US.linux-i686.complete.mar
 399         Or linux-i686/en-US/firefox-3.0b3.complete.mar
 400         Returns dict with keys product, version, locale, platform, type
 401     """
 402     try:
 403       m = re.search(
 404         '(?P<product>\w+)(-)(?P<version>\w+\.\w+(\.\w+){0,2})(\.)(?P<locale>.+?)(\.)(?P<platform>.+?)(\.)(?P<type>\w+)(.mar)',
 405       os.path.basename(filepath))
 406       return m.groupdict()
 407     except Exception, exc:
 408       try:
 409         m = re.search(
 410           '(?P<platform>.+?)\/(?P<locale>.+?)\/(?P<product>\w+)-(?P<version>\w+\.\w+)\.(?P<type>\w+).mar',
 411         filepath)
 412         return m.groupdict()
 413       except:
 414         raise Exception("could not parse filepath %s: %s" % (filepath, exc))
 415
 416 def create_partial_patches(patches):
 417     """ Given the patches generates a set of partial patches"""
 418     shas = {}
 419
 420     work_dir_root = None
 421     metadata = []
 422     try:
 423         work_dir_root = tempfile.mkdtemp('-fastmode', 'tmp', os.getcwd())
 424         print "Building patches using work dir: %s" % (work_dir_root)
 425
 426         # Iterate through every patch set in the patch file
 427         patch_num = 1
 428         for patch in patches:
 429             startTime = time.time()
 430
 431             from_filename,to_filename,patch_filename,forced_updates = patch.split(",")
 432             from_filename,to_filename,patch_filename = os.path.abspath(from_filename),os.path.abspath(to_filename),os.path.abspath(patch_filename)
 433
 434             # Each patch iteration uses its own work dir
 435             work_dir = os.path.join(work_dir_root,str(patch_num))
 436             os.mkdir(work_dir)
 437
 438             # Extract from mar into from dir
 439             work_dir_from =  os.path.join(work_dir,"from");
 440             os.mkdir(work_dir_from)
 441             extract_mar(from_filename,work_dir_from)
 442             from_decoded = decode_filename(from_filename)
 443             from_buildid = get_buildid(work_dir_from, from_decoded['platform'])
 444             from_shasum = sha.sha(open(from_filename).read()).hexdigest()
 445             from_size = str(os.path.getsize(to_filename))
 446
 447             # Extract to mar into to dir
 448             work_dir_to =  os.path.join(work_dir,"to")
 449             os.mkdir(work_dir_to)
 450             extract_mar(to_filename, work_dir_to)
 451             to_decoded = decode_filename(from_filename)
 452             to_buildid = get_buildid(work_dir_to, to_decoded['platform'])
 453             to_shasum = sha.sha(open(to_filename).read()).hexdigest()
 454             to_size = str(os.path.getsize(to_filename))
 455
 456             mar_extract_time = time.time()
 457
 458             partial_filename = create_partial_patch(work_dir_from, work_dir_to, patch_filename, shas, PatchInfo(work_dir, ['channel-prefs.js','update.manifest','updatev2.manifest','removed-files'],['/readme.txt']),forced_updates)
 459             partial_buildid = to_buildid
 460             partial_shasum = sha.sha(open(partial_filename).read()).hexdigest()
 461             partial_size = str(os.path.getsize(partial_filename))
 462
 463             metadata.append({
 464              'to_filename': os.path.basename(to_filename),
 465              'from_filename': os.path.basename(from_filename),
 466              'partial_filename': os.path.basename(partial_filename),
 467              'to_buildid':to_buildid,
 468              'from_buildid':from_buildid,
 469              'to_sha1sum':to_shasum,
 470              'from_sha1sum':from_shasum,
 471              'partial_sha1sum':partial_shasum,
 472              'to_size':to_size,
 473              'from_size':from_size,
 474              'partial_size':partial_size,
 475              'to_version':to_decoded['version'],
 476              'from_version':from_decoded['version'],
 477              'locale':from_decoded['locale'],
 478              'platform':from_decoded['platform'],
 479             })
 480             print "done with patch %s/%s time (%.2fs/%.2fs/%.2fs) (mar/patch/total)" % (str(patch_num),str(len(patches)),mar_extract_time-startTime,time.time()-mar_extract_time,time.time()-startTime)
 481             patch_num += 1
 482         return metadata
 483     finally:
 484         # If we fail or get a ctrl-c during run be sure to clean up temp dir
 485         if (work_dir_root and os.path.exists(work_dir_root)):
 486             shutil.rmtree(work_dir_root)
 487
 488 def main(argv):
 489     patchlist_file = None
 490     try:
 491          opts, args = getopt.getopt(argv, "hf:", ["help", "patchlist_file="])
 492          for opt, arg in opts:
 493             if opt in ("-h", "--help"):
 494                 usage()
 495                 sys.exit()
 496             elif opt in ("-f", "--patchlist_file"):
 497                 patchlist_file = arg
 498     except getopt.GetoptError:
 499           usage()
 500           sys.exit(2)
 501
 502     if not patchlist_file:
 503         usage()
 504         sys.exit(2)
 505
 506     patches = []
 507     f = open(patchlist_file, 'r')
 508     for line in f.readlines():
 509         patches.append(line)
 510     f.close()
 511     create_partial_patches(patches)
 512
 513 if __name__ == "__main__":
 514     main(sys.argv[1:])
 515