bin/update_pch_autotune.sh

   1 #! /bin/bash
   2 #
   3 # This file is part of the LibreOffice project.
   4 #
   5 # This Source Code Form is subject to the terms of the Mozilla Public
   6 # License, v. 2.0. If a copy of the MPL was not distributed with this
   7 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
   8 #
   9
  10 # Finds the optimal update_pch settings that results in,
  11 # per module and library, the fastest build time and
  12 # smallest intermediate files (.o/.obj) output.
  13
  14 # Usage: update_pch_autotune.sh [<module1> <module2>]
  15 # Invoke: /opt/lo/bin/make cmd cmd="./bin/update_pch_autotune.sh [..]"
  16
  17 # The resulting values may be entered in update_pch
  18 # to be use for generating PCH in the future.
  19 # Run this script after major header changes.
  20
  21 root=`dirname $0`
  22 root=`cd $root/.. && pwd`
  23 cd $root
  24
  25 if test -z "$1"; then
  26     modules=`ls ./*/inc/pch/precompiled_*.hxx | sed -e s%./%% -e s%/.*%% | uniq`
  27 else
  28     modules="$@"
  29 fi
  30
  31 if [[ "$OSTYPE" == "cygwin" ]]; then
  32     MAKE=/opt/lo/bin/make
  33 else
  34     MAKE=make
  35 fi
  36
  37 function build()
  38 {
  39     local START=$(date +%s.%N)
  40
  41     $MAKE -sr "$module" > /dev/null
  42     status=$?
  43     if [ $status -ne 0 ];
  44     then
  45         # Spurious failures happen.
  46         $MAKE "$module.build" > /dev/null
  47         status=$?
  48     fi
  49
  50     local END=$(date +%s.%N1)
  51     build_time=$(printf %.1f $(echo "$END - $START" | bc))
  52
  53     size="FAILED"
  54     score="FAILED"
  55     if [ $status -eq 0 ];
  56     then
  57         # The total size of the object files.
  58         size="$(du -s workdir/CxxObject/$module/ | awk '{print $1}')"
  59         # Add the pch file size.
  60         filename_rel="workdir/PrecompiledHeader/nodebug/$(basename $header)*"
  61         filename_dbg="workdir/PrecompiledHeader/debug/$(basename $header)*"
  62         if [[ $filename_rel -nt $filename_dbg ]]; then
  63             pch_size="$(du -s $filename_rel | awk '{print $1}' | paste -sd+ | bc)"
  64         else
  65             pch_size="$(du -s $filename_dbg | awk '{print $1}' | paste -sd+ | bc)"
  66         fi
  67         size="$(echo "$pch_size + $size" | bc)"
  68
  69         # Compute a score based on the build time and size.
  70         # The shorter the build time, and smaller disk usage, the higher the score.
  71         score=$(printf %.2f $(echo "10000 / ($build_time * e($size/1048576))" | bc -l))
  72     fi
  73 }
  74
  75 function run()
  76 {
  77     local msg="$module.$libname, ${@:3}, "
  78     printf "$msg"
  79     ./bin/update_pch "$module" "$libname" "${@:3}" --silent
  80     status=$?
  81
  82     if [ $status -eq 0 ];
  83     then
  84         build
  85
  86         summary="$build_time, $size, $score"
  87         if [ $status -eq 0 ];
  88         then
  89             new_best_for_cuttof=$(echo "$score > $best_score_for_cuttof" | bc -l)
  90             if [ $new_best_for_cuttof -eq 1 ];
  91             then
  92                 best_score_for_cuttof=$score
  93             fi
  94
  95             new_best=$(echo "$score > $best_score" | bc -l)
  96             if [ $new_best -eq 1 ];
  97             then
  98                 best_score=$score
  99                 best_args="${@:3}"
 100                 best_time=$build_time
 101                 best_cutoff=$cutoff
 102                 summary="$build_time, $size, $score,*"
 103             fi
 104         fi
 105     else
 106         # Skip if pch is not updated.
 107         summary="0, 0, 0"
 108     fi
 109
 110     echo "$summary"
 111 }
 112
 113 function args_to_table()
 114 {
 115     local sys="EXCLUDE"
 116     local mod="EXCLUDE"
 117     local loc="EXCLUDE"
 118     local cutoff=0
 119     IFS=' ' read -r -a aargs <<< $best_args
 120     for index in "${!aargs[@]}"
 121     do
 122         if [ "${aargs[index]}" = "--include:system" ];
 123         then
 124             sys="INCLUDE"
 125         elif [ "${aargs[index]}" = "--exclude:system" ];
 126         then
 127             sys="EXCLUDE"
 128         elif [ "${aargs[index]}" = "--include:module" ];
 129         then
 130             mod="INCLUDE"
 131         elif [ "${aargs[index]}" = "--exclude:module" ];
 132         then
 133             mod="EXCLUDE"
 134         elif [ "${aargs[index]}" = "--include:local" ];
 135         then
 136             loc="INCLUDE"
 137         elif [ "${aargs[index]}" = "--exclude:local" ];
 138         then
 139             loc="EXCLUDE"
 140         elif [[ "${aargs[index]}" == *"cutoff"* ]]
 141         then
 142             cutoff=$(echo "${aargs[index]}" | grep -Po '\-\-cutoff\=\K\d+')
 143         fi
 144     done
 145
 146     local key=$(printf "'%s.%s'" $module $libname)
 147     echo "$(printf "    %-36s: (%2d, %s, %s, %s), # %5.1f" $key $cutoff $sys $mod $loc $best_time)"
 148 }
 149
 150 for module in $modules; do
 151
 152     # Build without pch includes as sanity check.
 153     #run "$root" "$module" --cutoff=999
 154
 155     # Build before updating pch.
 156     $MAKE "$module.build" > /dev/null
 157     if [ $? -ne 0 ];
 158     then
 159         # Build with dependencies before updating pch.
 160         echo "Failed to build $module, building known state with dependencies..."
 161         ./bin/update_pch.sh "$module" > /dev/null
 162         $MAKE "$module.clean" > /dev/null
 163         $MAKE "$module.all" > /dev/null
 164         if [ $? -ne 0 ];
 165         then
 166             # Build all!
 167             echo "Failed to build $module with dependencies, building all..."
 168             $MAKE build-nocheck > /dev/null
 169             if [ $? -ne 0 ];
 170             then
 171                 >&2 echo "Broken build. Please revert changes and try again."
 172                 exit 1
 173             fi
 174         fi
 175     fi
 176
 177     # Find pch files in the module to update.
 178     headers=`find $root/$module/ -type f -iname "precompiled_*.hxx"`
 179
 180     # Each pch belongs to a library.
 181     for header in $headers; do
 182         libname=`echo $header | sed -e s/.*precompiled_// -e s/\.hxx//`
 183         #TODO: Backup the header and restore when last tune fails.
 184
 185         # Force update on first try below.
 186         echo "Autotuning $module.$libname..."
 187         ./bin/update_pch "$module" "$libname" --cutoff=999 --silent --force
 188
 189         best_score=0
 190         best_args=""
 191         best_time=0
 192         best_cutoff=0
 193         for i in {1..16}; do
 194             cutoff=$i
 195             best_score_for_cuttof=0
 196             #run "$root" "$module" "--cutoff=$i" --include:system --exclude:module --exclude:local
 197             run "$root" "$module" "--cutoff=$i" --exclude:system --exclude:module --exclude:local
 198             #run "$root" "$module" "--cutoff=$i" --include:system --include:module --exclude:local
 199             run "$root" "$module" "--cutoff=$i" --exclude:system --include:module --exclude:local
 200             #run "$root" "$module" "--cutoff=$i" --include:system --exclude:module --include:local
 201             run "$root" "$module" "--cutoff=$i" --exclude:system --exclude:module --include:local
 202             #run "$root" "$module" "--cutoff=$i" --include:system --include:module --include:local
 203             run "$root" "$module" "--cutoff=$i" --exclude:system --include:module --include:local
 204
 205             if [ $i -gt $((best_cutoff+2)) ];
 206             then
 207                 score_too_low=$(echo "$best_score_for_cuttof < $best_score / 1.10" | bc -l)
 208                 if [ $score_too_low -eq 1 ];
 209                 then
 210                     echo "Score hit low of $best_score_for_cuttof, well below overall best of $best_score. Stopping."
 211                     break;
 212                 fi
 213             fi
 214         done
 215
 216         ./bin/update_pch "$module" "$libname" $best_args --force --silent
 217         echo "> $module.$libname, $best_args, $best_time, $size, $score"
 218         echo
 219
 220         table+=$'\n'
 221         table+="$(args_to_table)"
 222     done
 223
 224 done
 225
 226 echo "Update the relevant lines in ./bin/update_pch script:"
 227 >&2 echo "$table"
 228
 229 exit 0