mk/scripts/binpkg-cache

   1 #!/bin/sh
   2 #
   3 # $NetBSD: binpkg-cache,v 1.20 2010/03/13 11:45:16 spz Exp $
   4 #
   5 # Script for generating a cache file with information about
   6 # all binary packages contained in a directory.
   7 #
   8 # Copyright (c) 2005, 2006 The NetBSD Foundation, Inc.
   9 # All rights reserved.
  10 #
  11 # This code is derived from software contributed to The NetBSD Foundation
  12 # by Dan McMahill.
  13 #
  14 # Redistribution and use in source and binary forms, with or without
  15 # modification, are permitted provided that the following conditions
  16 # are met:
  17 # 1. Redistributions of source code must retain the above copyright
  18 #    notice, this list of conditions and the following disclaimer.
  19 # 2. Redistributions in binary form must reproduce the above copyright
  20 #    notice, this list of conditions and the following disclaimer in the
  21 #    documentation and/or other materials provided with the distribution.
  22 # 3. All advertising materials mentioning features or use of this software
  23 #    must display the following acknowledgement:
  24 #        This product includes software developed by the NetBSD
  25 #        Foundation, Inc. and its contributors.
  26 # 4. Neither the name of The NetBSD Foundation nor the names of its
  27 #    contributors may be used to endorse or promote products derived
  28 #    from this software without specific prior written permission.
  29 #
  30 # THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
  31 # ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
  32 # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
  33 # PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
  34 # BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
  35 # CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
  36 # SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
  37 # INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
  38 # CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  39 # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  40 # POSSIBILITY OF SUCH DAMAGE.
  41 #
  42
  43
  44 TMPDIR=${TMPDIR:-/tmp}
  45 PACKAGES=${PACKAGES:-/usr/pkgsrc/packages/}
  46 AWK=${AWK:-awk}
  47 CMP=${CMP:-cmp}
  48 FIND=${FIND:-find}
  49 GREP=${GREP:-grep}
  50 GZIP_CMD=${GZIP_CMD:-gzip}
  51 BZIP2=${BZIP2:-bzip2}
  52 PKG_INFO=${PKG_INFO:-pkg_info}
  53 PKG_SUFX=${PKG_SUFX:-.tgz}
  54 SED=${SED:-sed}
  55 SORT=${SORT:-sort}
  56 STAT=${STAT:-stat}
  57
  58 cachefile=.pkgcache
  59 summaryfile=pkg_summary
  60 cacheversion=20050428
  61
  62 prompt="----> "
  63 tab="      "
  64
  65 tmpd=${TMPDIR}/pkg-cache.$$
  66 mkdir -m 0700 ${tmpd}
  67 if test $? -ne 0 ; then
  68         echo "ERROR:  Could not create temporary directory ${tmpd}"
  69         echo "Either you do not have write permission to ${tmpd} or"
  70         echo "${tmpd} already exists"
  71         exit 1
  72 fi
  73 all_dirs=${tmpd}/all_dirs
  74
  75 prog=$0
  76
  77 usage(){
  78         echo "$prog - Generates cache files for each directory containing binary"
  79         echo "        packages.  This cache file can then be used by the README.html"
  80         echo "        generation code to avoid having to call pkg_info(1) over and over"
  81         echo "        on the same binary package.  In addition, pkg_summary.gz files for"
  82         echo "        use by the pkgtools/pkg_chk pacakge may be generated."
  83         echo " "
  84         echo "Usage:      $prog [-d|--debug] [-v|--verbose] [-s|--summary] [-p|--packages <dir>]"
  85         echo " "
  86         echo "            $prog -h|--help"
  87         echo " "
  88         echo "            $prog -V|--version"
  89         echo " "
  90         echo "The options supported by $prog are: "
  91         echo " "
  92         echo "  -d|--debug            Enables debugging output"
  93         echo " "
  94         echo "  -F|--force            Forces a rebuild of the cache files even if they are"
  95         echo "                        otherwise up to date."
  96         echo " "
  97         echo "  -h|--help             Displays this help message"
  98         echo " "
  99         echo "  -p|--packages <dir>   Specifies the top level directory to be searched"
 100         echo "                        for binary packages."
 101         echo " "
 102         echo "  -s|--summary          Enables the creation of pkg_summary.gz files in each"
 103         echo "                        directory containing binary packages.  The pkg_summary.gz"
 104         echo "                        file is used by the pkgtools/pkg_chk package."
 105         echo " "
 106         echo "  -v|--verbose          Enables verbose output from the script."
 107         echo " "
 108         echo "  -V|--version          Displays the version of this script and exits."
 109         echo " "
 110         echo "Returns 0 on success, 1 on errors, 2 if the packages"
 111         echo "directory does not exist."
 112         echo " "
 113         echo "Example:    $prog -v --packages /usr/pkgsrc/packages"
 114         echo " "
 115 }
 116
 117 clean_and_exit0(){
 118         rm -fr ${tmpd}
 119         exit 0
 120 }
 121
 122 clean_and_exit1(){
 123         rm -fr ${tmpd}
 124         exit 1
 125 }
 126
 127 clean_and_exit2(){
 128         rm -fr ${tmpd}
 129         exit 2
 130 }
 131
 132 all_cache_files=""
 133
 134 #############################################################################
 135 #
 136 # process_binpkg_dir()
 137 #
 138 # Process a directory by checking to see if a cache file exists.  If the
 139 # cache file exists, make sure it is up to date.  If the file does not
 140 # exist, create one.
 141 #
 142 # also keep track of this directory so it can be added to the master
 143 # cache.
 144 #
 145
 146 process_binpkg_dir(){
 147         rdir=`${GREP} "^${d} " ${all_dirs} | ${AWK} '{print $2}'`
 148         need_update=no
 149         if test -f ${d}/${cachefile} ; then
 150                 stale_entries=`${FIND} ${d} -type f -name \*${PKG_SUFX} -newer ${d}/${cachefile} -print`
 151
 152                 # FIX_ME
 153                 #
 154                 # We also should find cache entries for files which no longer exist
 155                 # and nuke them.  Right now we simply declare the entire cache out
 156                 # of date.  Once we implement incremental updates to the cache,
 157                 # we need to remove the entries but not mark the entire cache as
 158                 # bad.
 159                 $if_debug echo "      Checking for cache entries with no corresponding pkg."
 160                 # get the list of what pkgs belong in the cache
 161                 rm -f ${tmpd}/pkg_list ${tmpd}/cache_pkg_list
 162                 ${FIND} ${d}/ -name \*${PKG_SUFX} -print | \
 163                         ${SED} -e "s;^${d}/*;${rdir}/;g" -e 's;//;/;g' | \
 164                         ${SORT} > ${tmpd}/pkg_list
 165
 166                 # and get the list of what is in the cache
 167                 ${AWK} '/pkgcache_begin/ {gsub(/pkgcache_begin[ \t]*/, ""); print}' \
 168                         ${d}/${cachefile} | ${SORT} > ${tmpd}/cache_pkg_list
 169
 170                 if ${CMP} -s ${tmpd}/pkg_list ${tmpd}/cache_pkg_list ; then
 171                         $if_debug echo "      No extra cache entries in ${d}/${cachefile}"
 172                 else
 173                         $if_debug echo "Package list:"
 174                         $if_debug cat  ${tmpd}/pkg_list
 175                         $if_debug echo "Cache list:"
 176                         $if_debug cat ${tmpd}/cache_pkg_list
 177                         echo "      Entries found in ${d}/${cachefile} but no packages found"
 178                         need_update=yes
 179                 fi
 180         else
 181                 stale_entries=""
 182         fi
 183
 184         if test "X${force}" = "Xyes" -o "X${need_update}" = "Xyes" ; then
 185                 need_update=yes
 186                 echo "${tab}Forcing rebuild of cache ${d}/${cachefile}."
 187         elif test ! -f ${d}/${cachefile} ; then
 188                 need_update=yes
 189                 echo "${tab}Missing cache file.  ${d}/${cachefile} will be generated."
 190         elif test -n "${stale_entries}" ; then
 191                 need_update=yes
 192                 echo "${tab}Stale cache file.  ${d}/${cachefile} will be regenerated."
 193         else
 194                 ${GREP} "pkgcache_version ${cacheversion}" ${d}/${cachefile} >/dev/null 2>&1
 195                 if test $? -ne 0 ; then
 196                         need_update=yes
 197                         echo "${tab}Invalid version cache file.  ${d}/${cachefile} will be regenerated."
 198                         echo "Need version ${cacheversion} but the file has"
 199                         ${GREP} "^pkgcache_version " ${d}/${cachefile}
 200                 else
 201                         $if_verbose echo "${tab}Cache file ${d}/${cachefile} is up to date."
 202                 fi
 203         fi
 204
 205         if test "${build_summary}" = "yes" -a ! -f ${d}/${summaryfile}.gz ; then
 206                 echo "${tab}Summary file ${d}/${summaryfile}.gz is missing and will be created."
 207                 need_update=yes
 208         fi
 209         if test "${build_summary}" = "yes" -a ${d}/${summaryfile}.gz -ot ${d}/${cachefile} ; then
 210                 echo "${tab}Summary file ${d}/${summaryfile}.gz is out of date and will be regenerated."
 211                 need_update=yes
 212         fi
 213
 214         # FIX_ME
 215         # We should use stale_entries in a way where we only update the
 216         # cache file entries corresponding to these if we're rebuilding
 217         # due to stale entries.  That should save a good bit of time.
 218         #
 219         if test "X${need_update}" = "Xyes" ; then
 220                 echo "pkgcache_version ${cacheversion}" > ${tmpd}/${cachefile}
 221                 rm -f ${tmpd}/${summaryfile}
 222                 touch ${tmpd}/${summaryfile}
 223                 for f in ${d}/*${PKG_SUFX} ; do
 224                         fn=`grep "^${d} " ${all_dirs} | ${AWK} '{print $2}'`"/"`basename ${f}`
 225                         $if_debug echo "     Adding ${fn} (${f}) to the cache"
 226                         echo " " >> ${tmpd}/${cachefile}
 227                         # stat(1) needs to be added to the bootstrap kit
 228                         # first if we want to use it here
 229                         #eval $(${STAT} -s ${f} 2>/dev/null)
 230                         echo "pkgcache_begin ${fn}" >> ${tmpd}/${cachefile}
 231                         #echo "pkgcache_mtime=${st_mtime}" >> ${tmpd}/${cachefile}
 232
 233                         $if_debug echo "${PKG_INFO} -q -B ${f}"
 234                         ${PKG_INFO} -q -B ${f} >> ${tmpd}/${cachefile}
 235
 236                         if test "${build_summary}" = "yes" ; then
 237                                 $if_debug echo "${PKG_INFO} -X ${f}"
 238                                 ${PKG_INFO} -X ${f} >> ${tmpd}/${summaryfile}
 239                         fi
 240                         echo "pkgcache_end ${fn}" >> ${tmpd}/${cachefile}
 241                 done
 242                 if test -f ${d}/${cachefile} ; then
 243                         rm -f ${d}/${cachefile}
 244                 fi
 245                 mv -f ${tmpd}/${cachefile} ${d}/${cachefile}
 246                 if test $? -ne 0 ; then
 247                         echo "********** WARNING **********"
 248                         echo "move of ${tmpd}/${cachefile} to ${d}/${cachefile} failed!"
 249                         echo "Perhaps you do not have write permissions to ${d}?"
 250                         echo "This directory will be dropped from the master cache file."
 251                         echo "********** WARNING **********"
 252                         return
 253                 fi
 254
 255                 if test "${build_summary}" = "yes" ; then
 256                         if test -f ${d}/${summaryfile}.gz ; then
 257                                 rm -f ${d}/${summaryfile}.gz
 258                         fi
 259                         cat ${tmpd}/${summaryfile} | ${GZIP_CMD} > ${d}/${summaryfile}.gz
 260                         if test $? -ne 0 ; then
 261                                 echo "********** WARNING **********"
 262                                 echo "${GZIP_CMD} of ${tmpd}/${summaryfile} to ${d}/${summaryfile}.gz failed!"
 263                                 echo "Perhaps you do not have write permissions to ${d}?"
 264                                 echo "********** WARNING **********"
 265                                 return
 266                         fi
 267                         # if it's there, update it, otherwise don't bother
 268                         if test -f ${d}/${summaryfile}.bz2 ; then
 269                                 rm -f ${d}/${summaryfile}.bz2
 270                                 cat ${tmpd}/${summaryfile} | ${BZIP2} > ${d}/${summaryfile}.bz2
 271                                 if test $? -ne 0 ; then
 272                                         echo "********** WARNING **********"
 273                                         echo "${BZIP2} of ${tmpd}/${summaryfile} to ${d}/${summaryfile}.bz2 failed!"
 274                                         echo "Perhaps you do not have write permissions to ${d}?"
 275                                         echo "********** WARNING **********"
 276                                         return
 277                                 fi
 278                         fi
 279                 fi
 280
 281         fi
 282
 283         # if we got here, then this directory should have a good cache file in
 284         # it and we should be able to add it to the master cache file
 285         all_cache_files="${all_cache_files} ${d}/${cachefile}"
 286
 287 }
 288
 289 process_cache_files(){
 290         echo "${prompt}Checking master cache file ${PACKAGES}/${cachefile}"
 291         echo "pkgcache_version ${cacheversion}" > ${tmpd}/${cachefile}
 292         if test -n "${all_cache_files}" ; then
 293                 for c in ${all_cache_files} ; do
 294                         echo "pkgcache_cachefile ${c}" >> ${tmpd}/${cachefile}
 295                 done
 296         fi
 297         if test ! -f ${PACKAGES}/${cachefile} ; then
 298                 echo "${tab}Creating master cache file ${PACKAGES}/${cachefile}"
 299                 mv -f ${tmpd}/${cachefile} ${PACKAGES}/${cachefile}
 300                 if test $? -ne 0 ; then
 301                         echo "********** ERROR **********"
 302                         echo "move of ${tmpd}/${cachefile} to ${PACKAGES}/${cachefile} failed!"
 303                         echo "Perhaps you do not have write permissions to ${PACKAGES}?"
 304                         echo "********** ERROR **********"
 305                         clean_and_exit1
 306                 fi
 307         elif ${CMP} -s ${tmpd}/${cachefile} ${PACKAGES}/${cachefile} ; then
 308                 echo "${tab}Master cache file ${PACKAGES}/${cachefile} is up to date"
 309         else
 310                 echo "${tab}Updating master cache file ${PACKAGES}/${cachefile}"
 311                 mv -f ${tmpd}/${cachefile} ${PACKAGES}/${cachefile}
 312                 if test $? -ne 0 ; then
 313                         echo "********** ERROR **********"
 314                         echo "move of ${tmpd}/${cachefile} to ${PACKAGES}/${cachefile} failed!"
 315                         echo "Perhaps you do not have write permissions to ${PACKAGES}?"
 316                         echo "********** ERROR **********"
 317                         clean_and_exit1
 318                 fi
 319         fi
 320 }
 321
 322 ######################################################################
 323 #
 324 #  Handle command line options
 325 #
 326 ######################################################################
 327
 328 if_debug=:              # either ":" or ""
 329 if_verbose=:            # either ":" or ""
 330 force=no
 331 build_summary=no
 332
 333 while
 334         test -n "$1"
 335 do
 336         case "$1" in
 337
 338                 # Turn on debugging
 339                 -d|--debug)
 340                         if_debug=""
 341                         if_verbose=""
 342                         shift
 343                         ;;
 344
 345                 # Force a rebuilde of the cache
 346                 -F|--force)
 347                         force=yes
 348                         shift
 349                         ;;
 350                 # Help
 351                 -h|--help)
 352                         usage
 353                         exit 0
 354                         ;;
 355
 356                 # Use the specified packages directory
 357                 -p|--packages)
 358                         PACKAGES=$2
 359                         shift 2
 360                         ;;
 361
 362                 # Build the pkg_summary.gz files
 363                 -s|--summary)
 364                         build_summary=yes
 365                         shift
 366                         ;;
 367
 368                 # Version
 369                 -V|--version)
 370                         ${AWK} '/^#[ \t]*\$NetBSD/ {gsub(/,v/,"",$3);printf("%s:  Version %s, %s\n",$3,$4,$5); exit 0;}' $prog
 371                         exit 0
 372                         ;;
 373
 374                 # Turn on verbose output, but not as noisy as debug
 375                 -v|--verbose)
 376                         if_verbose=""
 377                         shift
 378                         ;;
 379
 380                 -*) echo "$prog:  ERROR:  $1 is not a valid option"
 381                         usage
 382                         clean_and_exit1
 383                         ;;
 384
 385                 *)
 386                         break
 387                         ;;
 388
 389         esac
 390 done
 391
 392 if test $# -ne 0 ; then
 393         echo "$0:  $* is invalid"
 394         usage
 395         clean_and_exit1
 396 fi
 397
 398 if test ! -d ${PACKAGES} ; then
 399         echo "Packages directory ${PACKAGES} seems to be missing"
 400         clean_and_exit2
 401 fi
 402
 403 # put a trailing / after ${PACKAGES} in case ${PACKAGES} is
 404 # a link.
 405
 406 # pass 1, we find all directories under PACKAGES.  Note that this
 407 # may contain some directories more than once depending on what sort
 408 # of soft links may be in place
 409 rm -f ${all_dirs}.tmp
 410 for d in `${FIND} ${PACKAGES}/ -type d -follow -print` ; do
 411         cname=`(cd ${d} && pwd -P)`
 412         rname=`echo ${d} | ${SED} "s;^${PACKAGES}/*;;g"`
 413         echo "${cname} ${rname}" >> ${all_dirs}.tmp
 414 done
 415 ${SORT} -u -k1,1 ${all_dirs}.tmp > ${all_dirs}
 416 $if_debug echo "Full directory list:"
 417 $if_debug cat ${all_dirs}.tmp
 418 $if_debug echo "Unique directory list:"
 419 $if_debug cat ${all_dirs}
 420
 421 for d in `${AWK} '{print $1}' ${all_dirs}` ; do
 422         $if_debug echo "${prompt}Processing directory ${d}"
 423         is_pkg_dir=no
 424         for f in ${d}/*${PKG_SUFX} ; do
 425                 if test -f "${f}" -a ! -h "${f}" ; then
 426                         ${PKG_INFO} ${f} >/dev/null 2>&1
 427                         if test $? -eq 0 ; then
 428                                 is_pkg_dir=yes
 429                                 break
 430                         fi
 431                 fi
 432         done
 433         if test "X${is_pkg_dir}" = "Xyes" ; then
 434                 $if_verbose echo "${prompt}Checking cache in ${d}"
 435                 process_binpkg_dir
 436         else
 437                 $if_debug echo "${prompt}no binary packages in ${d}"
 438         fi
 439
 440 done
 441
 442 process_cache_files
 443
 444 clean_and_exit0