tools/update-packaging/make_incremental_update.sh

   1 #!/bin/bash
   2 # This Source Code Form is subject to the terms of the Mozilla Public
   3 # License, v. 2.0. If a copy of the MPL was not distributed with this
   4 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
   5
   6 #
   7 # This tool generates incremental update packages for the update system.
   8 # Author: Darin Fisher
   9 #
  10
  11 . $(dirname "$0")/common.sh
  12
  13 # -----------------------------------------------------------------------------
  14
  15 print_usage() {
  16   notice "Usage: $(basename $0) [OPTIONS] ARCHIVE FROMDIR TODIR"
  17   notice ""
  18   notice "The differences between FROMDIR and TODIR will be stored in ARCHIVE."
  19   notice ""
  20   notice "Options:"
  21   notice "  -h  show this help text"
  22   notice "  -f  clobber this file in the installation"
  23   notice "      Must be a path to a file to clobber in the partial update."
  24   notice "  -q  be less verbose"
  25   notice ""
  26 }
  27
  28 check_for_forced_update() {
  29   force_list="$1"
  30   forced_file_chk="$2"
  31
  32   local f
  33
  34   if [ "$forced_file_chk" = "precomplete" ]; then
  35     ## "true" *giggle*
  36     return 0;
  37   fi
  38
  39   if [ "$forced_file_chk" = "Contents/Resources/precomplete" ]; then
  40     ## "true" *giggle*
  41     return 0;
  42   fi
  43
  44   if [ "$forced_file_chk" = "removed-files" ]; then
  45     ## "true" *giggle*
  46     return 0;
  47   fi
  48
  49   if [ "$forced_file_chk" = "Contents/Resources/removed-files" ]; then
  50     ## "true" *giggle*
  51     return 0;
  52   fi
  53
  54   # notarization ticket
  55   if [ "$forced_file_chk" = "Contents/CodeResources" ]; then
  56     ## "true" *giggle*
  57     return 0;
  58   fi
  59
  60   if [ "${forced_file_chk##*.}" = "chk" ]; then
  61     ## "true" *giggle*
  62     return 0;
  63   fi
  64
  65   for f in $force_list; do
  66     #echo comparing $forced_file_chk to $f
  67     if [ "$forced_file_chk" = "$f" ]; then
  68       ## "true" *giggle*
  69       return 0;
  70     fi
  71   done
  72   ## 'false'... because this is bash. Oh yay!
  73   return 1;
  74 }
  75
  76 if [ $# = 0 ]; then
  77   print_usage
  78   exit 1
  79 fi
  80
  81 requested_forced_updates='Contents/MacOS/firefox'
  82
  83 while getopts "hqf:" flag
  84 do
  85    case "$flag" in
  86       h) print_usage; exit 0
  87       ;;
  88       q) QUIET=1
  89       ;;
  90       f) requested_forced_updates="$requested_forced_updates $OPTARG"
  91       ;;
  92       ?) print_usage; exit 1
  93       ;;
  94    esac
  95 done
  96
  97 # -----------------------------------------------------------------------------
  98
  99 mar_command="$MAR -V ${MOZ_PRODUCT_VERSION:?} -H ${MAR_CHANNEL_ID:?}"
 100
 101 let arg_start=$OPTIND-1
 102 shift $arg_start
 103
 104 archive="$1"
 105 olddir="$2"
 106 newdir="$3"
 107 # Prevent the workdir from being inside the targetdir so it isn't included in
 108 # the update mar.
 109 if [ $(echo "$newdir" | grep -c '\/$') = 1 ]; then
 110   # Remove the /
 111   newdir=$(echo "$newdir" | sed -e 's:\/$::')
 112 fi
 113 workdir="$(mktemp -d)"
 114 updatemanifestv3="$workdir/updatev3.manifest"
 115 archivefiles="updatev3.manifest"
 116
 117 mkdir -p "$workdir"
 118
 119 # Generate a list of all files in the target directory.
 120 pushd "$olddir"
 121 if test $? -ne 0 ; then
 122   exit 1
 123 fi
 124
 125 list_files oldfiles
 126 list_dirs olddirs
 127
 128 popd
 129
 130 pushd "$newdir"
 131 if test $? -ne 0 ; then
 132   exit 1
 133 fi
 134
 135 if [ ! -f "precomplete" ]; then
 136   if [ ! -f "Contents/Resources/precomplete" ]; then
 137     notice "precomplete file is missing!"
 138     exit 1
 139   fi
 140 fi
 141
 142 list_dirs newdirs
 143 list_files newfiles
 144
 145 popd
 146
 147 # Add the type of update to the beginning of the update manifests.
 148 notice ""
 149 notice "Adding type instruction to update manifests"
 150 > $updatemanifestv3
 151 notice "       type partial"
 152 echo "type \"partial\"" >> $updatemanifestv3
 153
 154 notice ""
 155 notice "Adding file patch and add instructions to update manifests"
 156
 157 num_oldfiles=${#oldfiles[*]}
 158 remove_array=
 159 num_removes=0
 160
 161 for ((i=0; $i<$num_oldfiles; i=$i+1)); do
 162   f="${oldfiles[$i]}"
 163
 164   # If this file exists in the new directory as well, then check if it differs.
 165   if [ -f "$newdir/$f" ]; then
 166
 167     if check_for_add_if_not_update "$f"; then
 168       # The full workdir may not exist yet, so create it if necessary.
 169       mkdir -p `dirname "$workdir/$f"`
 170       $XZ $XZ_OPT --compress $BCJ_OPTIONS --lzma2 --format=xz --check=crc64 --force --stdout "$newdir/$f" > "$workdir/$f"
 171       copy_perm "$newdir/$f" "$workdir/$f"
 172       make_add_if_not_instruction "$f" "$updatemanifestv3"
 173       archivefiles="$archivefiles \"$f\""
 174       continue 1
 175     fi
 176
 177     if check_for_forced_update "$requested_forced_updates" "$f"; then
 178       # The full workdir may not exist yet, so create it if necessary.
 179       mkdir -p `dirname "$workdir/$f"`
 180       $XZ $XZ_OPT --compress $BCJ_OPTIONS --lzma2 --format=xz --check=crc64 --force --stdout "$newdir/$f" > "$workdir/$f"
 181       copy_perm "$newdir/$f" "$workdir/$f"
 182       make_add_instruction "$f" "$updatemanifestv3" 1
 183       archivefiles="$archivefiles \"$f\""
 184       continue 1
 185     fi
 186
 187     if ! diff "$olddir/$f" "$newdir/$f" > /dev/null; then
 188       # Compute both the compressed binary diff and the compressed file, and
 189       # compare the sizes.  Then choose the smaller of the two to package.
 190       dir=$(dirname "$workdir/$f")
 191       mkdir -p "$dir"
 192       verbose_notice "diffing \"$f\""
 193       # MBSDIFF_HOOK represents the communication interface with funsize and,
 194       # if enabled, caches the intermediate patches for future use and
 195       # compute avoidance
 196       #
 197       # An example of MBSDIFF_HOOK env variable could look like this:
 198       # export MBSDIFF_HOOK="myscript.sh -A https://funsize/api -c /home/user"
 199       # where myscript.sh has the following usage:
 200       # myscript.sh -A SERVER-URL [-c LOCAL-CACHE-DIR-PATH] [-g] [-u] \
 201       #   PATH-FROM-URL PATH-TO-URL PATH-PATCH SERVER-URL
 202       #
 203       # Note: patches are bzipped or xz stashed in funsize to gain more speed
 204
 205       # if service is not enabled then default to old behavior
 206       if [ -z "$MBSDIFF_HOOK" ]; then
 207         $MBSDIFF "$olddir/$f" "$newdir/$f" "$workdir/$f.patch"
 208         $XZ $XZ_OPT --compress --lzma2 --format=xz --check=crc64 --force "$workdir/$f.patch"
 209       else
 210         # if service enabled then check patch existence for retrieval
 211         if $MBSDIFF_HOOK -g "$olddir/$f" "$newdir/$f" "$workdir/$f.patch.xz"; then
 212           verbose_notice "file \"$f\" found in funsize, diffing skipped"
 213         else
 214           # if not found already - compute it and cache it for future use
 215           $MBSDIFF "$olddir/$f" "$newdir/$f" "$workdir/$f.patch"
 216           $XZ $XZ_OPT --compress --lzma2 --format=xz --check=crc64 --force "$workdir/$f.patch"
 217           $MBSDIFF_HOOK -u "$olddir/$f" "$newdir/$f" "$workdir/$f.patch.xz"
 218         fi
 219       fi
 220       $XZ $XZ_OPT --compress $BCJ_OPTIONS --lzma2 --format=xz --check=crc64 --force --stdout "$newdir/$f" > "$workdir/$f"
 221       copy_perm "$newdir/$f" "$workdir/$f"
 222       patchfile="$workdir/$f.patch.xz"
 223       patchsize=$(get_file_size "$patchfile")
 224       fullsize=$(get_file_size "$workdir/$f")
 225
 226       if [ $patchsize -lt $fullsize ]; then
 227         make_patch_instruction "$f" "$updatemanifestv3"
 228         mv -f "$patchfile" "$workdir/$f.patch"
 229         rm -f "$workdir/$f"
 230         archivefiles="$archivefiles \"$f.patch\""
 231       else
 232         make_add_instruction "$f" "$updatemanifestv3"
 233         rm -f "$patchfile"
 234         archivefiles="$archivefiles \"$f\""
 235       fi
 236     fi
 237   else
 238     # remove instructions are added after add / patch instructions for
 239     # consistency with make_incremental_updates.py
 240     remove_array[$num_removes]=$f
 241     (( num_removes++ ))
 242   fi
 243 done
 244
 245 # Newly added files
 246 notice ""
 247 notice "Adding file add instructions to update manifests"
 248 num_newfiles=${#newfiles[*]}
 249
 250 for ((i=0; $i<$num_newfiles; i=$i+1)); do
 251   f="${newfiles[$i]}"
 252
 253   # If we've already tested this file, then skip it
 254   for ((j=0; $j<$num_oldfiles; j=$j+1)); do
 255     if [ "$f" = "${oldfiles[j]}" ]; then
 256       continue 2
 257     fi
 258   done
 259
 260   dir=$(dirname "$workdir/$f")
 261   mkdir -p "$dir"
 262
 263   $XZ $XZ_OPT --compress $BCJ_OPTIONS --lzma2 --format=xz --check=crc64 --force --stdout "$newdir/$f" > "$workdir/$f"
 264   copy_perm "$newdir/$f" "$workdir/$f"
 265
 266   if check_for_add_if_not_update "$f"; then
 267     make_add_if_not_instruction "$f" "$updatemanifestv3"
 268   else
 269     make_add_instruction "$f" "$updatemanifestv3"
 270   fi
 271
 272
 273   archivefiles="$archivefiles \"$f\""
 274 done
 275
 276 notice ""
 277 notice "Adding file remove instructions to update manifests"
 278 for ((i=0; $i<$num_removes; i=$i+1)); do
 279   f="${remove_array[$i]}"
 280   verbose_notice "     remove \"$f\""
 281   echo "remove \"$f\"" >> $updatemanifestv3
 282 done
 283
 284 # Add remove instructions for any dead files.
 285 notice ""
 286 notice "Adding file and directory remove instructions from file 'removed-files'"
 287 append_remove_instructions "$newdir" "$updatemanifestv3"
 288
 289 notice ""
 290 notice "Adding directory remove instructions for directories that no longer exist"
 291 num_olddirs=${#olddirs[*]}
 292
 293 for ((i=0; $i<$num_olddirs; i=$i+1)); do
 294   f="${olddirs[$i]}"
 295   # If this dir doesn't exist in the new directory remove it.
 296   if [ ! -d "$newdir/$f" ]; then
 297     verbose_notice "      rmdir $f/"
 298     echo "rmdir \"$f/\"" >> $updatemanifestv3
 299   fi
 300 done
 301
 302 $XZ $XZ_OPT --compress $BCJ_OPTIONS --lzma2 --format=xz --check=crc64 --force "$updatemanifestv3" && mv -f "$updatemanifestv3.xz" "$updatemanifestv3"
 303
 304 mar_command="$mar_command -C \"$workdir\" -c output.mar"
 305 eval "$mar_command $archivefiles"
 306 mv -f "$workdir/output.mar" "$archive"
 307
 308 # cleanup
 309 rm -fr "$workdir"
 310
 311 notice ""
 312 notice "Finished"
 313 notice ""