bin/sde-download

   1 #!/bin/bash
   2 # --- SDE-COPYRIGHT-NOTE-BEGIN ---
   3 # This copyright note is auto-generated by ./scripts/Create-CopyPatch.
   4 #
   5 # Filename: bin/sde-download
   6 # Copyright (C) 2006 - 2008 The OpenSDE Project
   7 # Copyright (C) 2004 - 2006 The T2 SDE Project
   8 # Copyright (C) 1998 - 2003 Clifford Wolf
   9 #
  10 # More information can be found in the files COPYING and README.
  11 #
  12 # This program is free software; you can redistribute it and/or modify
  13 # it under the terms of the GNU General Public License as published by
  14 # the Free Software Foundation; version 2 of the License. A copy of the
  15 # GNU General Public License can be found in the file COPYING.
  16 # --- SDE-COPYRIGHT-NOTE-END ---
  17
  18 #Description: Download sources
  19 #Alias: get
  20
  21 set -e
  22
  23 [ -n "$SDEROOT" ] ||
  24         export SDEROOT=$( cd "${0%/*}/.."; pwd -P )
  25
  26 . $SDEROOT/lib/libsde.in
  27 . $SDEROOT/lib/functions.in
  28
  29 download_usage() {
  30 cat <<EOT
  31 Usage:
  32         sde download <options> [ Package(s) ]
  33         sde download <options> [ Desc file(s) ]
  34         sde download <options> -repository Repositories
  35         sde download <options> [ -all | -required ]
  36
  37         Options is an alias for:
  38         [ -cfg <config> ] [ -nock ] [ -alt-dir <AlternativeDirectory> ]
  39         [ -mirror <URL> | -check ]  [ -try-questionable ]  [ -notimeout ]
  40         [ -longtimeout ]  [ -curl-opt <curl-option>[:<curl-option>[:..]] ]
  41         [ -copy ] [ -move ]
  42
  43         On default, this script auto-detects the best OpenSDE mirror.
  44
  45         Mirrors can also be a local directories in the form of 'file:///<dir>'.
  46
  47         sde download [ -list | -list-missing | -list-cksums ]
  48
  49         See '-mirror none' output for help on bypassing the official mirrors.
  50 EOT
  51 }
  52
  53 umask 022
  54
  55 cd "$SDEROOT"
  56
  57 # Handle options passed on the command line
  58 #
  59 mkdir -p tmp/ download/
  60
  61 # Load system wide configuration for this tool
  62 #
  63 config=default mirror= altdir= proxy= proxyauth=
  64
  65 if [ -s "$SDESETTINGS" ]; then
  66         eval $( $SDEROOT/bin/sde-config-ini -F "$SDESETTINGS" download )
  67         eval $( $SDEROOT/bin/sde-config-ini -F "$SDESETTINGS" download-$sdever )
  68 fi
  69
  70 this_is_the_2nd_run=0
  71 checkonly=0 tryques=0 nocheck=0 notimeout=0
  72 options='-this_is_the_2nd_run '
  73 curl_options='-A opensde-downloader --disable-epsv --location -f'
  74 altcopy=link ; verbose=1
  75 downloaderror=0
  76
  77 #
  78 while [ $# -gt 0 ]; do
  79         case "$1" in
  80
  81             -this_is_the_2nd_run)
  82                 this_is_the_2nd_run=1
  83                 ;;
  84
  85             -cfg)
  86                 options="$options -cfg $2"
  87                 config="$2" ; shift ;;
  88
  89             -q)
  90                 options="$options -q"
  91                 verbose=0 ;;
  92
  93             -nock)
  94                 # -nock skips checksum checking (don't use lightly)
  95                 options="$options -nock"
  96                 nocheck=1 ; shift ;;
  97
  98             -mirror)
  99                 # -mirror uses a mirror for finding source files
 100                 if [ "$2" = none ]; then
 101                         echo
 102                         echo "The option '-mirror none' is not supported anymore!"
 103                         echo
 104                         echo "You may edit \$HOME/.sde/settings if you really"
 105                         echo "want to use the original download resources. However, this"
 106                         echo "is not supported and if such a download fails, this is not"
 107                         echo "a bug in the OpenSDE and doesn't neccessarily needs fixing."
 108                         echo
 109                         exit 1;
 110                 elif [ "$2" = auto ]; then
 111                         mirror=
 112                 else
 113                         options="$options -mirror $2"
 114                         mirror="$2"
 115                         $SDEROOT/bin/sde-config-ini -F "$SDESETTINGS" "download-$sdever.mirror=$mirror"
 116                 fi
 117                 shift ;;
 118
 119             -check)
 120                 # -check just validates the file using the checksum
 121                 options="$options -check"
 122                 checkonly=1 ;;
 123
 124             -notimeout)
 125                 # don't add timeout curl options
 126                 options="$options -notimeout"
 127                 notimeout=2 ;;
 128
 129             -longtimeout)
 130                 # don't add timeout curl options
 131                 options="$options -longtimeout"
 132                 notimeout=1 ;;
 133
 134             -curl-opt)
 135                 # additional curl options
 136                 options="$options -curl-opt $2"
 137                 curl_options="$curl_options `echo $2 | tr : ' '`"
 138                 shift ;;
 139
 140             -alt-dir)
 141                 # check for an alternative directory where to search for
 142                 # package source tarballs
 143                 altdir=$( cd $2; pwd -P )
 144                 options="$options -alt-dir $2"
 145                 shift ;;
 146
 147             -try-questionable)
 148                 # also try to download questionable URLs
 149                 options="$options -try-questionable"
 150                 tryques=1 ;;
 151
 152             -move) altcopy=move ;;
 153             -copy) altcopy=copy ;;
 154
 155             *)  break   ;;
 156         esac
 157         shift
 158 done
 159
 160 if [ $notimeout -eq 0 ] ; then
 161         curl_options="$curl_options -y 10 -Y 10 --connect-timeout 60"
 162 fi
 163 if [ $notimeout -eq 1 ] ; then
 164         curl_options="$curl_options -y 60 -Y 1  --connect-timeout 300"
 165 fi
 166
 167 # proxy (server[:port])
 168 if [ -n "$proxy" ]; then
 169         curl_options="$curl_options --proxy $proxy"
 170
 171         # proxy needs auth (username[:password])
 172         [ -z "$proxyauth" ] || curl_options="$curl_options --proxy-user $proxyauth"
 173
 174         # only show once
 175         [ $this_is_the_2nd_run = 1 ] || echo_info "Using <$proxy> as ${proxyauth:+authenticated }http proxy."
 176 fi
 177
 178 #Disable checking for certificates on https downloads
 179 curl_options="$curl_options -k"
 180
 181 # Autodetect best Mirror and safe url in $mirror
 182 #
 183 detect_mirror() {
 184         if [ "$mirror" = "none" ] ; then
 185                 echo_info "Using original download locations only."
 186                 return
 187         elif [ "$mirror" = "broken" ]; then
 188                 echo_warning "Previous detection of the mirror failed, trying again."
 189         elif [ -n "$mirror" ]; then
 190                 echo_info "Using mirror <$mirror>."
 191                 return
 192         fi
 193
 194         echo_warning "Auto-detecting best mirror ..."
 195
 196         echo_info "Downloading mirror-list from opensde.net."
 197         curl -s -S $curl_options -o tmp/Download-Mirror-List \
 198                 "http://opensde.net/opensde-download-mirrors/$sdever"
 199
 200         if [ -r tmp/Download-Mirror-List ]; then
 201                 bash lib/sde-download/mirror-test.sh < tmp/Download-Mirror-List
 202         fi 2>&1 | echo_info
 203
 204         # read new mirror info
 205         mirror=
 206         eval $( $SDEROOT/bin/sde-config-ini -F "$SDESETTINGS" download-$sdever )
 207
 208         if [ -z "$mirror" ]; then
 209                 echo_error "Mirror detection loop hit a bug!"
 210         elif [ "$mirror" = "broken" ]; then
 211                 echo_warning "No Mirror Found!"
 212         else
 213                 echo_info "Using mirror <$mirror>."
 214         fi
 215 }
 216
 217 # download_file local-filename download-location cksum repo pkg
 218 #
 219 # This function decides if download directly or from a mirror,
 220 # validates checksum, etc.
 221 # Calls download_file_now to do the actual download.
 222 #
 223 download_file() {
 224
 225         # Init
 226         #
 227         local gzfile="$1" location="$2" cksum="$3" repo="$4" pkg="$5"
 228         # Make src directory for creating tar balls
 229         mkdir -p tmp/
 230         # Tarball file name:
 231         bzfile="`bz2filename "$gzfile"`"
 232         # Remove optional '-' prefix from $location
 233         [ "${location:0:1}" = '-' ] && location="${location:1}"
 234         # Lock file name:
 235         lkfile="tmp/down.lockfile.`echo $bzfile | tr / -`"
 236
 237         # Check if it's already there
 238         #
 239         [ -s "$bzfile" -a $checkonly != 1 ] && return 0
 240
 241         # Make locking
 242         #
 243         if [ -s "$lkfile" ]; then
 244                 echo "Found $lkfile -> skip download."
 245                 return 0
 246         fi
 247         trap 'rm -f "$lkfile"' INT
 248         echo $$ > "$lkfile"
 249
 250         # Check if we only like to test the cksum(s)
 251         #
 252         if [ $checkonly = 1 ] ; then
 253                 gzfile="$bzfile"
 254                 if [ ! -f "$bzfile" ] ; then
 255                         echo "File missing: $bzfile"
 256                         rm -f "$lkfile" ; trap INT ; return 1
 257                 fi
 258                 if [ -z "${cksum##X*}" ] ; then
 259                         echo "No checksum (ignore): $bzfile"
 260                         rm -f "$lkfile" ; trap INT ; return 1
 261                 fi
 262                 if [ "$cksum" -eq 0 ] ; then
 263                         echo "No checksum (missing): $bzfile"
 264                         rm -f "$lkfile" ; trap INT ; return 1
 265                 fi
 266
 267         elif [ -s "$gzfile" ] ; then
 268
 269                 echo ; echo "Already downloaded $pkg:$gzfile ..."
 270
 271         else
 272
 273                 echo ; echo "Downloading $pkg:$gzfile ..."
 274
 275                 # Existing *.cksum-err
 276                 #
 277                 if [ -s "$gzfile.cksum-err" ] ; then
 278                         # cksum-err file alread exists:
 279                         echo "ERROR: Found $gzfile.cksum-err."
 280                         echo "ERROR: That means that we downloaded the" \
 281                              "file already and it had an"
 282                         echo "ERROR: incorrect checksum. Remove the" \
 283                              "*.cksum-err file to force a"
 284                         echo "ERROR: new download of that file."
 285                         rm -f "$lkfile" ; trap INT ; return 1
 286                 fi
 287
 288                 # Existing *.extck-err
 289                 #
 290                 if [ -s "$gzfile.extck-err" ] ; then
 291                         # extck-err file alread exists:
 292                         echo "ERROR: Found $gzfile.extck-err."
 293                         echo "ERROR: That means that we downloaded the" \
 294                              "file already and it's content"
 295                         echo "ERROR: did not match it's filename extension." \
 296                              "Remove the *.extck-err file"
 297                         echo "ERROR: to force a new download of that file."
 298                         rm -f "$lkfile" ; trap INT ; return 1
 299                 fi
 300
 301                 # Questionable URL
 302                 #
 303                 if [ "$location" != "${location#\?}" ] ; then
 304                         if [ "$tryques" = 0 ] ; then
 305                                 echo "ERROR: URL is marked as questionable." \
 306                                         "Not downloading this file."
 307                                 rm -f "$lkfile" ; trap INT ; return 1
 308                         else
 309                                 echo "WARNING: URL is marked as questionable." \
 310                                         "Downloading it anyways."
 311                                 location="${location#\?}"
 312                         fi
 313                 fi
 314
 315                 # Make directory (if required)
 316                 #
 317                 if [ ! -d `dirname "$bzfile"` ] ; then
 318                         mkdir -p `dirname "$bzfile"`
 319                 fi
 320
 321                 # Alternative Directory
 322                 #
 323                 if [ "$altdir" ] ; then
 324                     altfile=$(find $altdir/ -name `basename $bzfile` | head -n 1)
 325                 else
 326                     altfile=
 327                 fi
 328
 329                 #FIXME: compatibility, can be removed sooner or later...
 330                 # Check old download dir layout
 331                 if [ -z "$altfile" ]; then
 332                         if [ -f "download/$repo${pkg:+/}$pkg/`basename $bzfile`" ]; then
 333                                 altfile="download/$repo${pkg:+/}$pkg/`basename $bzfile`"
 334                         fi
 335                 fi
 336
 337                 if [ "$altfile" ] ; then
 338
 339                         echo "Found `basename $bzfile` as $altfile."
 340                         if [ "$altcopy" = 'link' ]; then
 341                                 cp -lv $altfile $bzfile
 342                         elif [ "$altcopy" = 'copy' ]; then
 343                                 cp -v $altfile $bzfile
 344                         elif [ "$altcopy" = 'move' ]; then
 345                                 mv -v $altfile $bzfile
 346                         fi
 347                         gzfile="$bzfile"
 348
 349                 else
 350
 351                     # Mirroring
 352                     #
 353                     if [ -n "$mirror" -a "$mirror" != "none" -a "$mirror" != "broken" -a -z "${bzfile##download/mirror/*}" ] ; then
 354                         # try to use mirror
 355
 356                         if ! download_file_now "!$mirror/${bzfile#download/mirror/}" $bzfile $bzfile; then
 357                                 echo "INFO: download from mirror failed, trying original URL."
 358                                 download_file_now "$location" $gzfile $bzfile \
 359                                         || downloaderror=1
 360
 361                         else
 362                                 gzfile="$bzfile"
 363                         fi
 364                     else
 365                         # don't want to use mirror
 366                         download_file_now "$location" $gzfile $bzfile \
 367                                 downloaderror=1
 368                     fi
 369                 fi
 370
 371                 if [ ! -s "$gzfile" ]; then
 372                         rm -f "$lkfile" ; trap INT ; return 1
 373                 fi
 374         fi
 375
 376         # unsign .gpg file
 377         if expr "$gzfile" : "*.gpg" > /dev/null; then
 378                 gzfile=${gzfile%.gpg}
 379                 if [ -f $gzfile.gpg ]; then
 380                         echo "unsigning GnuPG file: $gzfile.gpg"
 381                         gpg $gzfile.gpg
 382                 fi
 383                 if [ ! -f $gzfile ]; then
 384                         echo "unsigning failed"
 385                         rm -f "$lkfile" ; trap INT ; return 1
 386                 fi
 387         fi
 388
 389         sh ./lib/sde-download/validate.sh "$gzfile" "$bzfile" "$cksum" || downloaderror=1
 390
 391         # Free Lock and finish
 392         #
 393         rm -f "$lkfile" ; trap INT ; return 0
 394 }
 395
 396 # download_file_now location remote_filename local_filename
 397 #
 398 # This function executes the actual download using curl.
 399 #
 400 download_file_now() {
 401         local location="$1" gzfile="$2" bzfile="$3" curlret=0
 402
 403         # Create URL
 404         #
 405         case "$location" in
 406                 manual://*) url="$location"     ;;
 407                 !*)     url="${location#!}"     ;;
 408                 *)      url="${location%/*}/${gzfile##*/}"      ;;
 409         esac
 410
 411         # Download
 412         #
 413         case "$url" in
 414         manual://*)
 415                 # Determine if the file has already been downloaded
 416                 # manually. For this we first look in $HOME then in
 417                 # download/manual.
 418                 downloadpath=${altdir:-$HOME}
 419                 downloadfile="${gzfile##*/}"
 420                 if [ -e $downloadpath/$downloadfile ]; then
 421                         location="file://$downloadpath/"
 422                 else
 423                         location="http://${url#manual://}"
 424                         # No manual download has taken place yet.
 425                         # So inform the user to do so.
 426                         cat <<-EOT
 427                         The file $downloadfile can not be fetched automatically
 428                         please visit: $location
 429                         and download it manually into $HOME or somewhere else using -alt-dir
 430                         EOT
 431                         return 1;
 432                 fi
 433
 434                 # I am to lazy to do the copy and conversion myself,
 435                 # so I use this function again with a modified
 436                 # download location.
 437                 download_file_now "$location" $gzfile $bzfile
 438                 return "$?"
 439                 ;;
 440         http://*|https://*|ftp://*|file://*)
 441                 if [ -s "$gzfile.incomplete" ] ; then
 442                     echo "INFO: Trying to resume previous download .."
 443                     resume="-C -"
 444                 else
 445                     resume=
 446                 fi
 447
 448                 [ -s download/translations.sed ] &&
 449                         trfile=download/translations.sh ||
 450                         trfile=etc/download.sed
 451                 trurl="$( echo "$url" | sed -f $trfile )"
 452                 if [ -n "$trurl" -a "$trurl" != "$url" ]; then
 453                         echo "INFO: url translated."
 454                         url="$trurl"
 455                 fi
 456                 unset trurl trfile
 457
 458                 curl -w '\rFinished downloading %{size_download} bytes in %{time_total} seconds (%{speed_download} bytes/sec). \n' --progress-bar $resume $curl_options "$url" -o "$gzfile.incomplete"
 459                 curlret="$?"
 460
 461                 if [ "$resume" ] && \
 462                    [ $curlret -eq 33 -o $curlret -eq 36 ] ; then
 463                     echo "INFO: Resuming download not possible. ->" \
 464                          "Overwriting old file."
 465                     rm -f "$gzfile.incomplete"
 466                     curl -w '\rFinished downloading %{size_download} bytes in %{time_total} seconds (%{speed_download} bytes/sec). \n' --progress-bar $curl_options "$url" -o "$gzfile.incomplete"
 467                     curlret="$?"
 468                 fi
 469
 470                 if [ $curlret -ne 0 ] ; then
 471                     case "$curlret" in
 472                       18)
 473                       echo "WARNING: Got only some of the" \
 474                            "file. A re-run of $0"
 475                       echo "WARNING: is required to complete" \
 476                            "the download." ;;
 477                       130)
 478                       echo -e '\rWARNING: CURL got a SIGINT' \
 479                            "(someone pressed Ctrl-C). A re-run of"
 480                       echo "WARNING: $0 is required to complete" \
 481                            "the download." ; sleep 1 ;;
 482                       *)
 483                       echo "$curlret $gzfile $url" \
 484                                         >> tmp/Download-Errors
 485                       echo -e '\rERROR: CURL Returned Error' \
 486                            "$curlret. Please read" \
 487                            "the curl manpage." ;;
 488                     esac
 489                     return 1
 490                 elif [ ! -s "$gzfile.incomplete" ] ; then
 491                     echo "0 $gzfile $url" >> tmp/Download-Errors
 492                     echo "ERROR: CURL returned success but" \
 493                          "we have no data!"
 494                     curlret=1
 495                 else
 496                     case "$gzfile" in
 497                       *.gz|*.tgz)
 498                           typeexpr="gzip compressed data" ;;
 499                       *.bz2|*.tbz2|*.tbz)
 500                           typeexpr="bzip2 compressed data" ;;
 501                       *.Z|*.tZ)
 502                           typeexpr="compress'd data" ;;
 503                       *.zip|*.jar)
 504                           typeexpr="Zip archive data" ;;
 505                       *.tar)
 506                           typeexpr="tar archive" ;;
 507                       *)
 508                           echo "WARNING: Unkown file extension: $gzfile"
 509                           typeexpr="." ;;
 510                     esac
 511                     if file "$gzfile.incomplete" | grep -v "$typeexpr"
 512                     then
 513                         echo "ERROR: File type does not match" \
 514                              "filename ($typeexpr)!"
 515                         mv "$gzfile.incomplete" "$gzfile.extck-err"
 516                     else
 517                         mv "$gzfile.incomplete" "$gzfile"
 518                     fi
 519                 fi
 520                 ;;
 521         *)
 522                 protocol="${url%%://*}"
 523
 524                 # we need to use $location - $url is already mangled above -ReneR
 525                 # $protocol://$url $options
 526                 url="`echo "$location" | sed "s,$protocol://\([^ ]*\).*,\1,"`"
 527                 options="`echo "$location" | cut -d' ' -f2-`"
 528
 529                 case "$protocol" in
 530                         cvs)
 531                                 # the first option is the module name
 532                                 module="${options%% *}"
 533                                 options="${options#* }"
 534                                 cmdline="cvs -z4 -Q -d $url co -P $options $module"
 535
 536                                 # sometimes cvs wants to read ~/.cvspass just for fun ..
 537                                 touch $HOME/.cvspass
 538                                 ;;
 539                         svn|svn\+http)
 540                                 if [ "$protocol" = "svn+http" ]; then
 541                                         url="http://$url"
 542                                 else
 543                                         url="svn://$url"
 544                                 fi
 545
 546                                 if [ "${options:0:1}" = "-" ]; then
 547                                         # the module is the last dir of $url
 548                                         module="${url##*/}"
 549                                 else
 550                                         # the first option is the module name
 551                                         module="${options%% *}"
 552                                         options="${options#* }"
 553                                 fi
 554                                 cmdline="svn co $options $url $module"
 555                                 ;;
 556                         *)
 557                                 echo "$cmdclient unrecognized!"
 558                                 return 1
 559                                 ;;
 560                 esac
 561
 562                 cvsdir="tmp/down.${protocol}dir.`echo $bzfile | tr / -`"
 563                 saved_pwd=$PWD ; mkdir -p $cvsdir ; cd $cvsdir
 564
 565                 echo "$cmdline"
 566                 {
 567                   $cmdline || touch .cvs_error
 568                 } &> .cvs_output &
 569
 570                 while fuser .cvs_output &> /dev/null ; do
 571                         echo -ne `nice du -sh 2> /dev/null | \
 572                                 cut -f1` 'downloaded from archive so far...\r'
 573                         sleep 3
 574                 done
 575
 576                 if [ -f .cvs_error ] ; then
 577                         cd $saved_pwd ; rm -rf $cvsdir
 578                         echo -e "\nError during checkout."
 579                         return 1
 580                 fi
 581
 582                 echo `du -sh 2> /dev/null | \
 583                         cut -f1` 'downloaded from archive (download finished).'
 584
 585
 586                 if [ `echo * | wc -w` -gt 1 ]; then
 587                         # multi-module module
 588                         echo "Multi-module package detected, relocating..."
 589                         mkdir t2-module.$$
 590                         for x in *; do
 591                                 [ "$x" != "t2-module.$$" ] && mv -f $x t2-module.$$/
 592                         done
 593                         mkdir -p "$module"
 594                         mv -f t2-module.$$/* "$module"
 595                         rm -f t2-module.$$
 596                 fi
 597
 598                 cd `dirname $module`
 599                 tarname="`basename $bzfile`"
 600                 echo "Preparing files for final tarball ..."
 601                 find -type d \( -name CVS -o -name .svn \) | xargs rm -rf
 602
 603                 if [ `find -type f | wc -l` -gt 4 ]; then
 604                         find `basename $module` | xargs touch -t 200001010000
 605                         tar --owner root --group root \
 606                             --use-compress-program=bzip2 \
 607                             -cf $tarname `basename $module`
 608                         mv $tarname $saved_pwd/$bzfile
 609                 else
 610                         echo "Too few files - assuming checkout failure."
 611                         curlret=1
 612                 fi
 613
 614                 cd $saved_pwd ; rm -rf $cvsdir
 615                 ;;
 616         esac
 617         return $curlret
 618 }
 619
 620 list_dtags() {
 621         {
 622                 grep -H '^\[D\] ' package/*/*/*.desc
 623                 grep -H '^[X0-9]' target/*/download.txt 2> /dev/null | sed 's,:,:[D] ,'
 624         } | column_clean
 625 }
 626
 627 list_cksums() {
 628         trap '' INT
 629
 630         # we know we only have single spaces due to list_dtags' column_clean
 631         list_dtags | sed -n \
 632           -e 's,[^ ]* \([X0-9]*\) \(.\)\([^ ]*\) -.*,\1 download/local/\2/\2\3,p' \
 633           -e 's,[^ ]* \([X0-9]*\) \(.\)\([^ ]*\) [^-].*,\1 download/mirror/\2/\2\3,p'
 634
 635         trap INT
 636 }
 637
 638 list() {
 639         trap '' INT
 640         list_cksums | cut -f2- -d' '
 641         trap INT
 642 }
 643
 644 list_missing() {
 645         trap '' INT
 646         list | bz2filename | \
 647         while read fn ; do
 648                 [ -f "$fn" ] || echo "$fn"
 649         done
 650         trap INT
 651 }
 652
 653 repository() {
 654         for repository ; do
 655                 packages `echo package/$repository/*/*.desc`
 656         done
 657 }
 658
 659 required() {
 660     # Choosen config must exist
 661     #
 662     if ! ./lib/sde-config/migrate.sh "$config"; then
 663         echo "ERROR: Config $config doesn't exist."
 664         echo "ERROR: try ./scripts/Config -cfg $config first."
 665         exit 1
 666     fi
 667
 668     while read on a b repo pkg c ; do
 669         if [ "$on" = "X" ] ; then
 670             grep -H '^\[D\] ' package/$repo/$pkg/$pkg.desc > tmp/down.$$.lst
 671             while read tag cksum file url ; do
 672                 download_file "`source_file cksum $file "$url"`" "$url" "$cksum" "$repo" "$pkg"
 673             done < tmp/down.$$.lst ; rm -f tmp/down.$$.lst
 674         fi
 675     done < config/$config/packages
 676
 677     target=`grep '^export SDECFG_TARGET=' config/$config/config | \
 678                                                 cut -f2 -d= | tr -d "'"`
 679     targetchain="$target"; x="$target"
 680     while [ -f "target/$x/extends" ]; do
 681         x="`cat target/$x/extends`"
 682         targetchain="$targetchain $x"
 683     done
 684
 685     for target in $targetchain; do
 686     if [ -f target/$target/download.txt ] ; then
 687         while read cksum file url ; do
 688             download_file "`source_file cksum $file "$url"`" "$url" "$cksum" "$target"
 689         done < target/$target/download.txt
 690     fi
 691     done
 692 }
 693
 694 all() {
 695     local each repo pkg
 696
 697     for repo in $( cd package; ls -1 ); do
 698         for each in package/$repo/*/*.desc; do
 699                 pkg="`echo $each | cut -f3 -d/`"
 700
 701                 while read tag cksum file url ; do
 702                     download_file "`source_file cksum $file "$url"`" "$url" "$cksum" "$repo" "$pkg"
 703                 done < <(grep -H '^\[D\] ' package/$repo/$pkg/$pkg.desc)
 704         done
 705     done
 706
 707     for each in $( ls -1 target/*/download.txt 2> /dev/null ); do
 708         target="`echo $each | cut -f2 -d/`"
 709
 710         while read cksum file url ; do
 711             download_file "`source_file cksum $file "$url"`" "$url" "$cksum" "$target"
 712         done < <(cat $each)
 713     done
 714 }
 715
 716 package() {
 717         descfile="`echo package/*/$1/$1.desc`"
 718
 719         if [ ! -f $descfile ]; then
 720                 echo "Skipping \"$1\" (not found)!"
 721                 return
 722         fi
 723
 724         pkg="`echo $descfile | cut -f3 -d/`"
 725         repo="`echo $descfile | cut -f2 -d/`"
 726
 727         while read tag cksum file url ; do
 728                 download_file "`source_file cksum $file "$url"`" "$url" "$cksum" "$repo" "$pkg"
 729         done < <(grep -H '^\[D\] ' package/$repo/$pkg/$pkg.desc)
 730 }
 731
 732 packages() {
 733         local descfile
 734         for arg; do
 735                 case "$arg" in
 736                 target/*)
 737                         if [ ! -f $arg ]; then
 738                             echo "Skipping \"$arg\" (not found)!"
 739                             continue
 740                         fi
 741
 742                         target="`echo $arg | cut -f2 -d/`"
 743
 744                         while read cksum file url ; do
 745                             download_file "`source_file cksum $file "$url"`" "$url" "$cksum" "$target"
 746                         done < <(cat $arg)
 747                         ;;
 748                 *)
 749                         if [ "${arg%.desc}" != "$arg" ]; then
 750                         arg="`echo $arg | cut -f3 -d/`"; fi
 751
 752
 753                         # active extensions
 754                         local extender=
 755
 756                         # pkg_*_{pre,post}.conf is only activated if extender
 757                         # is enabled on $config/packages, so we will only
 758                         # download files of those extenders
 759                         #
 760                         for extender in `ls -1 package/*/*/pkg_${arg}_{pre,post}.conf 2> /dev/null |
 761                                          cut -d/ -f3 | sort -u`; do
 762                                 if grep -q "^X .* $extender " \
 763                                    config/$config/packages; then
 764                                         echo_info "Also downloading $extender ..."
 765                                         package $extender
 766                                 fi
 767                         done
 768                         package $arg
 769                         ;;
 770                 esac
 771         done
 772 }
 773
 774 set +e
 775
 776 # Things to do only for downloading
 777 #
 778 if [ "${1:0:5}" != "-list" -a $checkonly = 0 ]; then
 779         # we need curl
 780         if [ -z "`type -p curl`" ]; then
 781                 echo_abort 2 "we need \`curl\` installed and available on \$PATH to proceed."
 782         fi
 783
 784         # do mirror detection, only once
 785         [ $this_is_the_2nd_run = 1 ] || detect_mirror
 786 fi
 787
 788 case "$1" in
 789         -list)          list ;;
 790         -list-missing)  list_missing ;;
 791         -list-cksums)   list_cksums ;;
 792
 793         -required)      required ;;
 794         -all)           all ;;
 795
 796         -repository)    shift ; repository "$@" ;;
 797
 798         -*|"")          download_usage
 799                         exit 1;;
 800
 801         *)              packages "$@" ;;
 802 esac
 803
 804 exit $downloaderr