lib/aur-sync

   1 #!/bin/bash
   2 # aur-sync - download and build AUR packages automatically
   3 [[ -v AUR_DEBUG ]] && set -o xtrace
   4 set -o errexit
   5 argv0=sync
   6 XDG_CACHE_HOME=${XDG_CACHE_HOME:-$HOME/.cache}
   7 XDG_CONFIG_HOME=${XDG_CONFIG_HOME:-$HOME/.config}
   8 XDG_STATE_HOME=${XDG_STATE_HOME:-$HOME/.local/state}
   9 AURDEST=${AURDEST:-$XDG_CACHE_HOME/aurutils/$argv0}
  10 AUR_SYNC_USE_NINJA=${AUR_SYNC_USE_NINJA:-0}
  11 PS4='+(${BASH_SOURCE}:${LINENO}): ${FUNCNAME[0]:+${FUNCNAME[0]}(): }'
  12
  13 # Avoid CDPATH screwing with cd (#1047)
  14 unset -v CDPATH
  15
  16 # default arguments
  17 build_args=(--clean --syncdeps)
  18 depends_args=() repo_args=() view_args=() filter_args=() fetch_args=() graph_args=()
  19
  20 # default options
  21 build=1 chkver_depth=2 download=1 view=1 provides=1 graph=1 keep_going=1
  22
  23 # default options (disabled)
  24 rotate=0 update=0 repo_targets=0 columns=0
  25
  26 args_csv() {
  27     # shellcheck disable=SC2155
  28     local str=$(printf '%s,' "$@")
  29     printf '%s' "${str%,}"
  30 }
  31
  32 lib32() {
  33     awk -v arch="$(uname -m)" '{
  34         if(arch == "i686") {
  35             gsub(/^lib32-/,"")
  36             gsub(/^gcc-multilib$/,"")
  37         }
  38         print
  39     }' "$@"
  40 }
  41
  42 # argv[1]: $1 pkgname $2 required_by $3 pkgbase
  43 # note: edges are not checked for duplicates
  44 select_pkgbase() {
  45     awk 'NR == FNR {
  46         map[$1] = $3
  47         next
  48     }
  49     $2 in map {
  50         printf("%s\t%s\n", $3, map[$2])
  51     }' "$@"
  52 }
  53
  54 # argv[1]: $1 pkgname (possibly empty, #910)
  55 # argv[2]: $1 pkgname
  56 select_ignores() {
  57     awk 'ARGV[1] == FILENAME {
  58         map[$1] = 1
  59         next
  60     }
  61     !($1 in map) {
  62         print
  63     }' "$@"
  64 }
  65
  66 select_ignores_pkgspec() {
  67     awk -F'/' -v needle="$1" '{
  68         if (NF == 2 && $1 == needle) {
  69             print $2
  70         } else if (NF == 1) {
  71             print $1
  72         }
  73     }' "${@:2}"
  74 }
  75
  76 swap() {
  77     awk '{print $2 "\t" $1}' "$@"
  78 }
  79
  80 complement() {
  81     # empty set should not return 1
  82     grep -Fxvf "$@" || return $(( $? - 1 ))
  83 }
  84
  85 trap_exit() {
  86     if [[ ! -v AUR_DEBUG ]]; then
  87         rm -rf -- "$tmp"
  88     else
  89         printf >&2 'AUR_DEBUG: %s: temporary files at %s\n' "$argv0" "$tmp"
  90     fi
  91 }
  92
  93 usage() {
  94     printf >&2 'usage: %s [-d repo] [--root path] [-cdfoSu] pkgname...\n' "$argv0"
  95     exit 1
  96 }
  97
  98 source /usr/share/makepkg/util/message.sh
  99
 100 if [[ ! -v NO_COLOR ]] && [[ ! -v AUR_DEBUG ]]; then
 101     [[ -t 2 ]] && colorize
 102 fi
 103
 104 # mollyguard for makepkg
 105 if (( UID == 0 )) && [[ ! -v AUR_ASROOT ]]; then
 106     printf >&2 'warning: aur-%s is not meant to be run as root.\n' "$argv0"
 107     printf >&2 'warning: To proceed anyway, set the %s variable.\n' 'AUR_ASROOT'
 108     exit 1
 109 fi
 110
 111 # option parsing
 112 opt_short='d:D:k:U:AcfLnorRSTuv'
 113 opt_long=('bind:' 'bind-rw:' 'database:' 'directory:' 'ignore:' 'root:'
 114           'makepkg-conf:' 'pacman-conf:' 'chroot' 'continue' 'force'
 115           'ignore-arch' 'log' 'no-confirm' 'no-ver' 'no-graph' 'no-sync' 'no-ver-argv'
 116           'no-view' 'no-provides' 'no-build' 'rmdeps' 'sign' 'temp' 'upgrades'
 117           'pkgver' 'rebuild' 'rebuild-tree' 'rebuild-all' 'ignore-file:'
 118           'remove' 'provides-from:' 'new' 'prevent-downgrade' 'verify'
 119           'makepkg-args:' 'format:' 'no-check' 'keep-going:' 'user:'
 120           'rebase' 'reset' 'ff' 'exclude:' 'columns' 'prefix' 'save:')
 121 opt_hidden=('dump-options' 'allan' 'ignorearch' 'ignorefile:' 'noconfirm'
 122             'nover' 'nograph' 'nosync' 'nover-argv' 'noview' 'noprovides' 'nobuild'
 123             'rebuildall' 'rebuildtree' 'rm-deps' 'gpg-sign' 'margs:' 'nocheck'
 124             'no-checkdepends' 'nocheckdepends' 'optdepends' 'repo:')
 125
 126 if opts=$(getopt -o "$opt_short" -l "$(args_csv "${opt_long[@]}" "${opt_hidden[@]}")" -n "$argv0" -- "$@"); then
 127     eval set -- "$opts"
 128 else
 129     usage
 130 fi
 131
 132 unset pkg pkg_i repo repo_p ignore_file stdout_file
 133 while true; do
 134     case "$1" in
 135         # sync options
 136         --allan)
 137             rotate=1 ;;
 138         --continue)
 139             download=0 ;;
 140         --ignore)
 141             shift; IFS=, read -a pkg -r <<< "$1"
 142             pkg_i+=("${pkg[@]}") ;;
 143         --ignorefile|--ignore-file)
 144             shift; ignore_file=$1 ;;
 145         -k|--keep-going)
 146             shift; keep_going=$1 ;;
 147         -o|--nobuild|--no-build)
 148             build=0 ;;
 149         --columns)
 150             build=0; columns=1 ;;
 151         --save)
 152             shift; stdout_file=$1 ;;
 153         --optdepends)
 154             depends_args+=(--optdepends)
 155             graph_args+=(-v OPTDEPENDS=1) ;;
 156         --nocheck|--no-check|--nocheckdepends|--no-checkdepends)
 157             depends_args+=(--no-checkdepends)
 158             build_args+=(--no-check)
 159             graph_args+=(-v CHECKDEPENDS=0) ;;
 160         --nograph|--no-graph)
 161             graph=0 ;;
 162         --nosync|--no-sync)
 163             build_args+=(--no-sync) ;;
 164         --nover|--no-ver)
 165             chkver_depth=0 ;;
 166         --nover-argv|--no-ver-argv)
 167             chkver_depth=1 ;;
 168         --noview|--no-view)
 169             view=0 ;;
 170         --noprovides|--no-provides)
 171             provides=0 ;;
 172         --provides-from)
 173             shift; IFS=, read -a repo -r <<< "$1"
 174             repo_p+=("${repo[@]}")
 175             provides=1 ;;
 176         --rebuild)
 177             # Command-line targets are excluded from `repo-filter`
 178             build_args+=(-f); chkver_depth=1 ;;
 179         --rebuildtree|--rebuild-tree)
 180             # Dependencies may be removed by `repo-filter` (#1066)
 181             build_args+=(-f); chkver_depth=0; provides=0 ;;
 182         --rebuildall|--rebuild-all)
 183             build_args+=(-f); chkver_depth=0; repo_targets=1 ;;
 184         -u|--upgrades)
 185             update=1 ;;
 186         # database options
 187         -d|--database|--repo)
 188             shift; repo_args+=(-d "$1") ;;
 189         --root)
 190             shift; repo_args+=(-r "$1") ;;
 191         # fetch options
 192         --ff)
 193             fetch_args+=(--ff) ;;
 194         --rebase)
 195             fetch_args+=(--rebase) ;;
 196         --reset)
 197             fetch_args+=(--reset) ;;
 198         # view options
 199         --format)
 200             shift; view_args+=(--format "$1") ;;
 201         --exclude)
 202             shift; view_args+=(--exclude "$1") ;;
 203         --prefix)
 204             view_args+=(--prefix) ;;  # experimental
 205         # build options
 206         -c|--chroot)
 207             build_args+=(--chroot) ;;
 208         -f|--force)
 209             build_args+=(--force) ;;
 210         --makepkg-args|--margs)
 211             shift; build_args+=(--margs "$1") ;;
 212         --makepkg-conf)
 213             shift; build_args+=(--makepkg-conf "$1") ;;
 214         --pacman-conf)
 215             shift; build_args+=(--pacman-conf "$1")
 216             repo_args+=(--config "$1")
 217             filter_args+=(--config "$1") ;;
 218         --pkgver)
 219             build_args+=(--pkgver) ;;
 220         -S|--sign|--gpg-sign)
 221             build_args+=(--sign) ;;
 222         -U|--user)
 223             shift; build_args+=(--user "$1") ;;
 224         # build options (devtools)
 225         -D|--directory)
 226             shift; build_args+=(--directory "$1") ;;
 227         --bind)
 228             shift; build_args+=(--bind "$1") ;;
 229         --bind-rw)
 230             shift; build_args+=(--bind-rw "$1") ;;
 231         -T|--temp)
 232             build_args+=(-T) ;;
 233         # build options (makepkg)
 234         -A|--ignorearch|--ignore-arch)
 235             build_args+=(--ignorearch) ;;
 236         -L|--log)
 237             build_args+=(--log) ;;
 238         -n|--noconfirm|--no-confirm)
 239             build_args+=(--noconfirm) ;;
 240         -r|--rmdeps|--rm-deps)
 241             build_args+=(--rmdeps) ;;
 242         # build options (repo-add)
 243         -R|--remove)
 244             build_args+=(--remove) ;;
 245         -v|--verify)
 246             build_args+=(--verify) ;;
 247         --prevent-downgrade)
 248             build_args+=(--prevent-downgrade) ;;
 249         --new)
 250             build_args+=(--new) ;;
 251         # other options
 252         --dump-options)
 253             printf -- '--%s\n' "${opt_long[@]}" ${AUR_DEBUG+"${opt_hidden[@]}"}
 254             printf -- '%s' "${opt_short}" | sed 's/.:\?/-&\n/g'
 255             exit ;;
 256         --) shift; break ;;
 257     esac
 258     shift
 259 done
 260
 261 # shellcheck disable=SC2174
 262 mkdir -pm 0700 -- "${TMPDIR:-/tmp}/aurutils-$UID"
 263 tmp=$(mktemp -d --tmpdir "aurutils-$UID/$argv0.XXXXXXXX")
 264 trap 'trap_exit' EXIT
 265
 266 # Default to showing PKGBUILD first in patch. (#399)
 267 orderfile=$XDG_CONFIG_HOME/aurutils/$argv0/orderfile
 268 mkdir -p -- "${orderfile%/*}"
 269
 270 if [[ ! -s $orderfile ]]; then
 271     printf 'PKGBUILD\n' > "$orderfile"
 272 fi
 273
 274 if (( rotate )); then
 275     if { hash caesar && target=$(aur pkglist | shuf -n 1); } 2>/dev/null; then
 276         exec bash -c "{ aur \"$argv0\" -c \"$target\" && repo-elephant | caesar 13; } 2>&1 | caesar 13"
 277     else
 278         echo '?'; exit 16 # EBUSY
 279     fi
 280 fi
 281 mkdir -p -- "$AURDEST"
 282
 283 if (( $# + update + repo_targets == 0 )); then
 284     printf >&2 '%s: no targets specified\n' "$argv0"
 285     exit 1
 286 fi
 287
 288 # Write --no-build / --columns results to a file (#1077)
 289 if [[ -v stdout_file ]]; then
 290     stdout_file=$(realpath -- "$stdout_file")
 291 fi
 292
 293 # Retrieve path to local repo (#448, #700)
 294 { IFS=: read -r _ db_name
 295   IFS=: read -r _ db_root
 296   IFS=: read -r _ db_path
 297 } < <(aur repo "${repo_args[@]}" --status)
 298 wait "$!"
 299
 300 msg >&2 'Using [%s] repository' "$db_name"
 301
 302 # Concatenate ignores from file and command-line arguments
 303 : "${ignore_file=$XDG_CONFIG_HOME/aurutils/sync/ignore}"
 304
 305 # Restrict ignores to local repository with a single pass (#880)
 306 { if [[ -r $ignore_file ]] && [[ ! -d $ignore_file ]]; then
 307       select_ignores_pkgspec "$db_name" "$ignore_file"
 308   fi
 309   if (( ${#pkg_i[@]} )); then
 310       printf '%s\n' "${pkg_i[@]}"
 311   fi
 312 } | complement <(printf '%s\n' "$@") >"$tmp"/igni
 313
 314 # Diagnostic on which packages are ignored
 315 if [[ -s $tmp/igni ]]; then
 316     mapfile -t pkg_i < "$tmp"/igni
 317     printf '%s: packages ignored: %s\n' "$argv0" "${pkg_i[*]}"
 318
 319     # Ignore dependencies from ignored targets (#592)
 320     depends_args+=(--assume-installed "$(args_csv "${pkg_i[@]}")")
 321 fi >&2
 322
 323 # Retrieve list of local repository packages ($1 pkgname $2 pkgver)
 324 aur repo-parse -p "$db_path" --list | select_ignores "$tmp"/igni - >"$tmp"/db_info
 325
 326 if (( PIPESTATUS[0] )); then
 327    exit 2
 328 fi
 329
 330 # Build list of AUR targets
 331 { if (( $# )); then
 332       # append command-line arguments
 333       printf '%s\n' "$@"
 334   fi
 335
 336   if (( repo_targets )); then
 337       # append repository packages (all)
 338       cut -f1 <"$tmp"/db_info
 339
 340   elif (( update )); then
 341       # append repository packages (updated)
 342       aur vercmp --quiet <"$tmp"/db_info
 343   fi
 344 } >"$tmp"/argv
 345
 346 # Build AUR dependency graph
 347 if [[ -s $tmp/argv ]]; then
 348     # shellcheck disable=SC2094
 349     # depends: $1 pkgname $2 required_by $3 pkgbase $4 pkgver $5 depends_type
 350     aur depends --table --reverse "${depends_args[@]}" - <"$tmp"/argv >"$tmp"/depends
 351 else
 352     printf >&2 '%s: there is nothing to do\n' "$argv0"
 353     exit
 354 fi
 355
 356 # pkginfo: $1 pkgname $2 pkgbase $3 pkgver
 357 cut -f2,5 --complement "$tmp"/depends | sort -u >"$tmp"/pkginfo
 358
 359 { cat "$tmp"/igni # ignored packages (already included by aur-depends --assume-installed)
 360
 361   # Packages with equal or newer versions are taken as complement
 362   # for the queue. If chkver_argv is enabled, packages on the
 363   # command-line are excluded from this complement.
 364   if (( chkver_depth )); then
 365       # note: AUR cannot be queried by pkgbase (FS#57230)
 366       cut -f1,3 "$tmp"/pkginfo | aur vercmp -p "$tmp"/db_info -c >"$tmp"/current
 367
 368       # shellcheck disable=SC2002
 369       case $chkver_depth in
 370           1) cat "$tmp"/current | complement "$tmp"/argv ;;
 371           2) cat "$tmp"/current ;;
 372       esac
 373   fi
 374
 375   if (( provides )); then
 376       if (( ${#repo_p[@]} )); then
 377           filter_args+=("${repo_p[@]/#/--repo=}")
 378       else
 379           filter_args+=(--sync)
 380       fi
 381
 382       # Note: this uses pacman's copy of the repo (as used by makepkg -s)
 383       cut -f1 "$tmp"/pkginfo | aur repo-filter "${filter_args[@]}" | complement "$tmp"/argv
 384   fi
 385 } >"$tmp"/filter
 386
 387 # $1 pkgname $2 required_by $3 pkgbase, filter by $1 (depends and self)
 388 cut -f1-3 "$tmp"/depends | select_ignores "$tmp"/filter - | lib32 - >"$tmp"/graph_0
 389
 390 # XXX a flat file is needed for aur-{graph,fetch,view}. `ninja` requires the
 391 # build files to be present before dependency resolution (with `ninja -n`) can
 392 # occur, and `ninja -t targets` sorts in alphabetical order. This implies
 393 # that dependency cycles cannot be resolved before retrieving files with
 394 # aur-fetch with `ninja` alone. `tsort` could either be used in this case (with
 395 # a less nice diagnostic on cycles), or fetches done in an arbitrary order
 396 # (e.g. sort -u) with checks for cycles done at build-time.
 397 if ! select_pkgbase "$tmp"/graph_0 "$tmp"/graph_0 | tee "$tmp"/graph | tsort >"$tmp"/queue; then
 398     printf >&2 '%s: dependency cycle detected\n' "$argv0"
 399     exit 22
 400 fi
 401
 402 if [[ -s $tmp/queue ]]; then
 403     cd -- "$AURDEST"
 404 else
 405     printf >&2 '%s: there is nothing to do\n' "$argv0"
 406     exit
 407 fi
 408
 409 if (( download )); then
 410     msg >&2 "Retrieving package files"
 411     aur fetch -S "${fetch_args[@]}" --discard --results "$tmp"/fetch_results - < "$tmp"/queue >&2
 412
 413     # shellcheck disable=SC2034
 414     while IFS=: read -r mode rev_old rev path; do
 415         path=${path#file://} name=${path##*/}
 416
 417         case $mode in
 418             clone)
 419                 git -C "$path" config diff.orderFile "$orderfile" ;;
 420             fetch|merge|rebase|reset)
 421                 ;;
 422         esac
 423     done < "$tmp"/fetch_results
 424 else
 425     xargs -a "$tmp"/queue stat >/dev/null || exit 2 # ensure all directories are available
 426 fi
 427
 428 # Verify dependency tree (#20)
 429 if (( graph )); then
 430     if ! { while read -r pkg; do
 431                [[ $pkg ]] && printf '%s\0' "$pkg/.SRCINFO"
 432            done
 433          } | xargs -0 cat -- | aur graph "${graph_args[@]}"
 434     then
 435         printf >&2 '%s: failed to verify dependency graph\n' "$argv0"
 436         exit 1
 437     fi <"$tmp"/queue | swap >"$tmp"/graph
 438
 439     # Recompute dependencies to include `provides` (#837)
 440     tsort < "$tmp"/graph >"$tmp"/queue || exit 22
 441 fi
 442
 443 { # Sort dependencies (`aur-sync--ninja`, `--columns`)
 444   swap "$tmp"/graph | sort -k1b,1 -k1 -u >"$tmp"/graph.ninja
 445
 446   # Resolve absolute paths (`--no-build`)
 447   while read -r pkg; do
 448       [[ $pkg ]] && printf '%s\n' "$AURDEST/$pkg"
 449   done <"$tmp"/queue >"$tmp"/queue.realpath
 450 }
 451
 452 # Inspect package files
 453 if (( view )); then
 454     aur view -a "$tmp"/queue "${view_args[@]}"
 455 fi
 456
 457 # `--columns` / `--no-build` output
 458 if (( columns )) && [[ -v stdout_file ]]; then
 459     cat "$tmp"/graph.ninja >"$stdout_file"
 460
 461 elif (( columns )); then
 462     cat "$tmp"/graph.ninja
 463
 464 elif ! (( build )) && [[ -v stdout_file ]]; then
 465     cat "$tmp"/queue.realpath >"$stdout_file"
 466
 467 elif ! (( build )); then
 468     cat "$tmp"/queue.realpath
 469
 470 # Build dependency tree with ninja (#908)
 471 elif (( AUR_SYNC_USE_NINJA )); then
 472     # Apply `--save` (#1091)
 473     if [[ -v stdout_file ]]; then
 474         cat "$tmp"/graph.ninja >"$stdout_file"
 475     fi
 476
 477     # Directory for stamp files (concurrent aur-sync processes)
 478     mkdir -p -- "$XDG_STATE_HOME"/aurutils/$argv0
 479     ninja_dir=$XDG_STATE_HOME/aurutils/$argv0/ninja-$USER-$$
 480     mkdir -- "$ninja_dir"
 481
 482     # Generate build.ninja
 483     # input:  $AURDEST/pkgbase/PKGBUILD
 484     # output: $ninja_dir/pkgbase.stamp
 485     aur sync--ninja "$AURDEST" <"$tmp"/graph.ninja >"$ninja_dir"/build.ninja \
 486         -- aur build "${build_args[@]}" -d "$db_name" --root "$db_root"
 487
 488     if NINJA_STATUS='[%s/%t] ' ninja -C "$ninja_dir" -k "$keep_going"; then
 489         # Remove ninja directory on successful build
 490         rm -rf "$ninja_dir"
 491     else
 492         # Print all targets in dependency order
 493         NINJA_STATUS='[%s/%t] ' ninja -nC /var/empty -f "$ninja_dir"/build.ninja | \
 494             # [\w@\.\-\+]: valid characters for pkgname
 495             # alternative: [^\s]+ from rule `env -C ... > pkgbase.stamp`
 496             pcregrep -o1 -o3 '(\[\d+/\d+\] )(.+?)([\w@\.\-\+]+)(\.stamp)' | while read -r status pkg
 497         do
 498             if [[ -f $ninja_dir/$pkg.stamp ]]; then
 499                 printf "${BOLD}${BLUE}%s${ALL_OFF} %s\t${BOLD}${GREEN}[OK]${ALL_OFF}\n" "$status" "$pkg"
 500             else
 501                 printf "${BOLD}${BLUE}%s${ALL_OFF} %s\t${BOLD}${RED}[FAIL]${ALL_OFF}\n" "$status" "$pkg"
 502             fi
 503         done | column -t
 504
 505         # Preserve ninja directory
 506         printf '%s: build files at %s\n' "$argv0" "$ninja_dir"
 507     fi
 508 else
 509     # Apply `--save` (#1091)
 510     if [[ -v stdout_file ]]; then
 511         cat "$tmp"/queue.realpath >"$stdout_file"
 512     fi
 513     aur build "${build_args[@]}" -a "$tmp"/queue.realpath -d "$db_name" --root "$db_root"
 514 fi
 515
 516 # vim: set et sw=4 sts=4 ft=sh: