sources/functions-fwl.sh

   1 #!/bin/echo "This file is sourced, not run"
   2
   3 # Lots of reusable functions.  This file is sourced, not run.
   4
   5 function blank_tempdir()
   6 {
   7   # sanity test: never rm -rf something we don't own.
   8   [ -z "$1" ] && dienow
   9   touch -c "$1" || dienow
  10
  11   # Delete old directory, create new one.
  12   rm -rf "$1"
  13   mkdir -p "$1" || dienow
  14 }
  15
  16 # Figure out if we're using the stable or unstable versions of a package.
  17
  18 function unstable()
  19 {
  20   [ ! -z "$(echo ,"$USE_UNSTABLE", | grep ,"$1",)" ]
  21 }
  22
  23 # Find all files in $STAGE_DIR newer than $CURSRC.
  24
  25 function recent_binary_files()
  26 {
  27   PREVIOUS=
  28   (cd "$STAGE_DIR" || dienow
  29    # Note $WORK/$PACKAGE != $CURSRC here for renamed packages like gcc-core.
  30    find . -depth -newer "$WORK/$PACKAGE/FWL-TIMESTAMP" \
  31      | sed -e 's/^.//' -e 's/^.//' -e '/^$/d'
  32   ) | while read i
  33   do
  34     TEMP="${PREVIOUS##"$i"/}"
  35
  36     if [ $[${#PREVIOUS}-${#TEMP}] -ne $[${#i}+1] ]
  37     then
  38       # Because the expanded $i might have \ chars in it, that's why.
  39       echo -n "$i"
  40       echo -ne '\0'
  41     fi
  42     PREVIOUS="$i"
  43   done
  44 }
  45
  46 # Strip the version number off a tarball
  47
  48 function cleanup()
  49 {
  50   # If package build exited with an error, do not continue.
  51
  52   #[ $? -ne 0 ] && dienow
  53
  54   if [ ! -z "$BINARY_PACKAGE_TARBALLS" ]
  55   then
  56     TARNAME="$PACKAGE-$STAGE_NAME-${ARCH_NAME}".tar.bz2
  57     echo -n Creating "$TARNAME"
  58     { recent_binary_files | xargs -0 tar -cjvf \
  59         "$BUILD/${TARNAME}".tar.bz2 -C "$STAGE_DIR" || dienow
  60     } | dotprogress
  61   fi
  62
  63   if [ ! -z "$NO_CLEANUP" ]
  64   then
  65     echo "skip cleanup $PACKAGE $@"
  66     return
  67   fi
  68
  69   # Loop deleting directories
  70
  71   cd "$WORK" || dienow
  72   for i in "$PACKAGE" "$@"
  73   do
  74     [ -z "$i" ] && continue
  75     echo "cleanup $i"
  76     rm -rf "$i" || dienow
  77  done
  78 }
  79
  80 # Give filename.tar.ext minus the version number.
  81
  82 function noversion()
  83 {
  84   echo "$1" | sed -e 's/-*\(\([0-9\.]\)*\([_-]rc\)*\(-pre\)*\([0-9][a-zA-Z]\)*\)*\(\.tar\..z2*\)$/'"$2"'\6/'
  85 }
  86
  87 # Given a filename.tar.ext, return the versino number.
  88
  89 function getversion()
  90 {
  91   echo "$1" | sed -e 's/.*-\(\([0-9\.]\)*\([_-]rc\)*\(-pre\)*\([0-9][a-zA-Z]\)*\)*\(\.tar\..z2*\)$/'"$2"'\1/'
  92 }
  93
  94 # Give package name, minus file's version number and archive extension.
  95
  96 function basename()
  97 {
  98   noversion $1 | sed 's/\.tar\..z2*$//'
  99 }
 100
 101 # output the sha1sum of a file
 102 function sha1file()
 103 {
 104   sha1sum "$@" | awk '{print $1}'
 105 }
 106
 107 # Extract tarball named in $1 and apply all relevant patches into
 108 # "$BUILD/packages/$1".  Record sha1sum of tarball and patch files in
 109 # sha1-for-source.txt.  Re-extract if tarball or patches change.
 110
 111 function extract()
 112 {
 113   FILENAME="$1"
 114   SRCTREE="${BUILD}/packages"
 115   SHA1FILE="$(echo "${SRCTREE}/${PACKAGE}/sha1-for-source.txt")"
 116
 117   # Sanity check: don't ever "rm -rf /".  Just don't.
 118
 119   if [ -z "$PACKAGE" ] || [ -z "$SRCTREE" ]
 120   then
 121     dienow
 122   fi
 123
 124   # If the source tarball doesn't exist, but the extracted directory is there,
 125   # assume everything's ok.
 126
 127   [ ! -e "$FILENAME" ] && [ -e "$SHA1FILE" ] && return 0
 128
 129   SHA1TAR="$(sha1file "${SRCDIR}/${FILENAME}")"
 130
 131   # If it's already extracted and up to date (including patches), do nothing.
 132   SHALIST=$(cat "$SHA1FILE" 2> /dev/null)
 133   if [ ! -z "$SHALIST" ]
 134   then
 135     for i in "$SHA1TAR" $(sha1file "${SOURCES}/patches/${PACKAGE}"-* 2>/dev/null)
 136     do
 137       # Is this sha1 in the file?
 138       if [ -z "$(echo "$SHALIST" | sed -n "s/$i/$i/p" )" ]
 139       then
 140         SHALIST=missing
 141         break
 142       fi
 143       # Remove it
 144       SHALIST="$(echo "$SHALIST" | sed "s/$i//" )"
 145     done
 146     # If we matched all the sha1sums, nothing more to do.
 147     [ -z "$SHALIST" ] && return 0
 148   fi
 149
 150   echo -n "Extracting '${PACKAGE}'"
 151   # Delete the old tree (if any).  Create new empty working directories.
 152   rm -rf "${BUILD}/temp" "${SRCTREE}/${PACKAGE}" 2>/dev/null
 153   mkdir -p "${BUILD}"/{temp,packages} || dienow
 154
 155   # Is it a bzip2 or gzip tarball?
 156   DECOMPRESS=""
 157   [ "$FILENAME" != "${FILENAME/%\.tar\.bz2/}" ] && DECOMPRESS="j"
 158   [ "$FILENAME" != "${FILENAME/%\.tar\.gz/}" ] && DECOMPRESS="z"
 159
 160   { tar -xv${DECOMPRESS} -f "${SRCDIR}/${FILENAME}" -C "${BUILD}/temp" || dienow
 161   } | dotprogress
 162
 163   mv "${BUILD}/temp/"* "${SRCTREE}/${PACKAGE}" &&
 164   rmdir "${BUILD}/temp" &&
 165   echo "$SHA1TAR" > "$SHA1FILE"
 166
 167   [ $? -ne 0 ] && dienow
 168
 169   # Apply any patches to this package
 170
 171   ls "${SOURCES}/patches/${PACKAGE}"-* 2> /dev/null | sort | while read i
 172   do
 173     if [ -f "$i" ]
 174     then
 175       echo "Applying $i"
 176       (cd "${SRCTREE}/${PACKAGE}" && patch -p1 -i "$i") || dienow
 177       sha1file "$i" >> "$SHA1FILE"
 178     fi
 179   done
 180 }
 181
 182 function try_checksum()
 183 {
 184   SUM="$(sha1file "$SRCDIR/$FILENAME" 2>/dev/null)"
 185   if [ x"$SUM" == x"$SHA1" ] || [ -z "$SHA1" ] && [ -f "$SRCDIR/$FILENAME" ]
 186   then
 187     if [ -z "$SHA1" ]
 188     then
 189       echo "No SHA1 for $FILENAME ($SUM)"
 190     else
 191       echo "Confirmed $FILENAME"
 192     fi
 193
 194     # Preemptively extract source packages?
 195
 196     [ -z "$EXTRACT_ALL" ] && return 0
 197     EXTRACT_ONLY=1 setupfor "$(basename "$FILENAME")"
 198     return $?
 199   fi
 200
 201   return 1
 202 }
 203
 204
 205 function try_download()
 206 {
 207   # Return success if we have a valid copy of the file
 208
 209   try_checksum && return 0
 210
 211   # If there's a corrupted file, delete it.  In theory it would be nice
 212   # to resume downloads, but wget creates "*.1" files instead.
 213
 214   rm "$SRCDIR/$FILENAME" 2> /dev/null
 215
 216   # If we have another source, try to download file.
 217
 218   if [ -n "$1" ]
 219   then
 220     wget -t 2 -T 60 -O "$SRCDIR/$FILENAME" "$1" ||
 221       (rm "$SRCDIR/$FILENAME"; return 2)
 222     touch -c "$SRCDIR/$FILENAME"
 223   fi
 224
 225   try_checksum
 226 }
 227
 228 # Confirm a file matches sha1sum, else try to download it from mirror list.
 229
 230 function download()
 231 {
 232   FILENAME=`echo "$URL" | sed 's .*/  '`
 233   [ -z "$RENAME" ] || FILENAME="$(echo "$FILENAME" | sed -r "$RENAME")"
 234   ALTFILENAME=alt-"$(noversion "$FILENAME" -0)"
 235
 236   echo -ne "checking $FILENAME\r"
 237
 238   # Update timestamps on both stable and unstable tarballs (if any)
 239   # so cleanup_oldfiles doesn't delete them
 240   touch -c "$SRCDIR"/{"$FILENAME","$ALTFILENAME"} 2>/dev/null
 241
 242   # Is the unstable version selected?
 243   if unstable "$(basename "$FILENAME")"
 244   then
 245     # Download new one as alt-packagename.tar.ext
 246     FILENAME="$ALTFILENAME" SHA1= try_download "$UNSTABLE" ||
 247       ([ ! -z "$PREFERRED_MIRROR" ] && SHA1= FILENAME="$ALTFILENAME" try_download "$PREFERRED_MIRROR/$ALTFILENAME")
 248     return $?
 249   fi
 250
 251   # If environment variable specifies a preferred mirror, try that first.
 252
 253   if [ ! -z "$PREFERRED_MIRROR" ]
 254   then
 255     try_download "$PREFERRED_MIRROR/$FILENAME" && return 0
 256   fi
 257
 258   # Try standard locations
 259   # Note: the URLs in mirror list cannot contain whitespace.
 260
 261   try_download "$URL" && return 0
 262   for i in $MIRROR_LIST
 263   do
 264     try_download "$i/$FILENAME" && return 0
 265   done
 266
 267   # Return failure.
 268
 269   echo "Could not download $FILENAME"
 270   echo -en "\e[0m"
 271   return 1
 272 }
 273
 274 # Clean obsolete files out of the source directory
 275
 276 START_TIME=`date +%s`
 277
 278 function cleanup_oldfiles()
 279 {
 280   for i in "${SRCDIR}"/*
 281   do
 282     if [ -f "$i" ] && [ "$(date +%s -r "$i")" -lt "${START_TIME}" ]
 283     then
 284       echo Removing old file "$i"
 285       rm -rf "$i"
 286     fi
 287   done
 288 }
 289
 290 # An exit function that works properly even from a subshell.
 291
 292 function actually_dienow()
 293 {
 294   echo -e "\n\e[31mExiting due to errors ($PACKAGE)\e[0m"
 295   exit 1
 296 }
 297
 298 trap actually_dienow SIGUSR1
 299 TOPSHELL=$$
 300
 301 function dienow()
 302 {
 303   kill -USR1 $TOPSHELL
 304   exit 1
 305 }
 306
 307 # Turn a bunch of output lines into a much quieter series of periods.
 308
 309 function dotprogress()
 310 {
 311   x=0
 312   while read i
 313   do
 314     x=$[$x + 1]
 315     if [[ "$x" -eq 25 ]]
 316     then
 317       x=0
 318       echo -n .
 319     fi
 320   done
 321   echo
 322 }
 323
 324 function set_titlebar()
 325 {
 326   [ -z "$NO_TITLE_BAR" ] &&
 327     echo -en "\033]2;$1\007"
 328 }
 329
 330 # Extract package $1, use out-of-tree build directory $2 (or $1 if no $2)
 331 # Use link directory $3 (or $1 if no $3)
 332
 333 function setupfor()
 334 {
 335   export WRAPPY_LOGPATH="$WRAPPY_LOGDIR/cmdlines.${STAGE_NAME}.setupfor"
 336
 337   # Figure out whether we're using an unstable package.
 338
 339   PACKAGE="$1"
 340   unstable "$PACKAGE" && PACKAGE=alt-"$PACKAGE"
 341
 342   # Make sure the source is already extracted and up-to-date.
 343   cd "${SRCDIR}" &&
 344   extract "${PACKAGE}-"*.tar* || exit 1
 345
 346   # If all we want to do is extract source, bail out now.
 347   [ ! -z "$EXTRACT_ONLY" ] && return 0
 348
 349   # Set CURSRC
 350   CURSRC="$PACKAGE"
 351   if [ ! -z "$3" ]
 352   then
 353     CURSRC="$3"
 354     unstable "$CURSRC" && CURSRC=alt-"$CURSRC"
 355   fi
 356   export CURSRC="${WORK}/${CURSRC}"
 357
 358   [ -z "$SNAPSHOT_SYMLINK" ] && LINKTYPE="l" || LINKTYPE="s"
 359
 360   # Announce package, with easy-to-grep-for "===" marker.
 361
 362   echo "=== Building $PACKAGE ($ARCH_NAME $STAGE_NAME)"
 363   echo "Snapshot '$PACKAGE'..."
 364   cd "${WORK}" || dienow
 365   if [ $# -lt 3 ]
 366   then
 367     rm -rf "${CURSRC}" || dienow
 368   fi
 369   mkdir -p "${CURSRC}" &&
 370   cp -${LINKTYPE}fR "${SRCTREE}/$PACKAGE/"* "${CURSRC}"
 371
 372   [ $? -ne 0 ] && dienow
 373
 374   # Do we have a separate working directory?
 375
 376   if [ -z "$2" ]
 377   then
 378     cd "$PACKAGE"* || dienow
 379   else
 380     mkdir -p "$2" && cd "$2" || dienow
 381   fi
 382   export WRAPPY_LOGPATH="$WRAPPY_LOGDIR/cmdlines.${STAGE_NAME}.$1"
 383
 384   # Change window title bar to package now
 385   set_titlebar "$ARCH_NAME $STAGE_NAME $PACKAGE"
 386
 387   # Ugly bug workaround: timestamp granularity in a lot of filesystems is only
 388   # 1 second, so find -newer misses things installed in the same second, so we
 389   # make sure it's a new second before we start actually doing anything.
 390
 391   if [ ! -z "$BINARY_PACKAGE_TARBALLS" ]
 392   then
 393     touch "${CURSRC}/FWL-TIMESTAMP" || dienow
 394     TIME=$(date +%s)
 395     while true
 396     do
 397       [ $TIME != "$(date +%s)" ] && break
 398       sleep .1
 399     done
 400   fi
 401 }
 402
 403 # Figure out what version of a package we last built
 404
 405 function get_download_version()
 406 {
 407   getversion $(sed -n 's@URL=.*/\(.[^ ]*\).*@\1@p' "$TOP/download.sh" | grep ${1}-)
 408 }
 409
 410 # Filter out unnecessary noise
 411
 412 maybe_quiet()
 413 {
 414   [ -z "$QUIET" ] && cat || grep "^==="
 415 }
 416
 417 # Run a command either in foreground or background, depending on $FORK
 418
 419 maybe_fork()
 420 {
 421   if [ -z "$FORK" ]
 422   then
 423     eval "$*"
 424   else
 425     eval "$*" &
 426   fi
 427 }
 428
 429 # Kill a process and all its decendants
 430
 431 function killtree()
 432 {
 433   local KIDS=""
 434
 435   while [ $# -ne 0 ]
 436   do
 437     KIDS="$KIDS $(pgrep -P$1)"
 438     shift
 439   done
 440
 441   KIDS="$(echo -n $KIDS)"
 442   if [ ! -z "$KIDS" ]
 443   then
 444     # Depth first kill avoids reparent_to_init hiding stuff.
 445     killtree $KIDS
 446     kill $KIDS 2>/dev/null
 447   fi
 448 }