lib/aur-sync

   1 #!/bin/bash
   2 # aur-sync - download and build AUR packages automatically
   3 [[ -v AUR_DEBUG ]] && set -o xtrace
   4 set -o errexit
   5 argv0=sync
   6 XDG_CACHE_HOME=${XDG_CACHE_HOME:-$HOME/.cache}
   7 XDG_CONFIG_HOME=${XDG_CONFIG_HOME:-$HOME/.config}
   8 XDG_STATE_HOME=${XDG_STATE_HOME:-$HOME/.local/state}
   9 AURDEST=${AURDEST:-$XDG_CACHE_HOME/aurutils/$argv0}
  10 AUR_SYNC_USE_NINJA=${AUR_SYNC_USE_NINJA:-0}
  11 PS4='+(${BASH_SOURCE}:${LINENO}): ${FUNCNAME[0]:+${FUNCNAME[0]}(): }'
  12
  13 # Avoid CDPATH screwing with cd (#1047)
  14 unset -v CDPATH
  15
  16 # default arguments
  17 build_args=(--clean --syncdeps)
  18 depends_args=() repo_args=() view_args=() filter_args=() fetch_args=() graph_args=()
  19
  20 # default options
  21 build=1 chkver_depth=2 download=1 view=1 provides=1 graph=1 keep_going=1 columns=0
  22
  23 # default options (disabled)
  24 rotate=0 update=0 repo_targets=0
  25
  26 args_csv() {
  27     # shellcheck disable=SC2155
  28     local str=$(printf '%s,' "$@")
  29     printf '%s' "${str%,}"
  30 }
  31
  32 lib32() {
  33     awk -v arch="$(uname -m)" '{
  34         if(arch == "i686") {
  35             gsub(/^lib32-/,"")
  36             gsub(/^gcc-multilib$/,"")
  37         }
  38         print
  39     }' "$@"
  40 }
  41
  42 # argv[1]: $1 pkgname $2 required_by $3 pkgbase
  43 # note: edges are not checked for duplicates
  44 select_pkgbase() {
  45     awk 'NR == FNR {
  46         map[$1] = $3
  47         next
  48     }
  49     $2 in map {
  50         printf("%s\t%s\n", $3, map[$2])
  51     }' "$@"
  52 }
  53
  54 # argv[1]: $1 pkgname (possibly empty, #910)
  55 # argv[2]: $1 pkgname
  56 select_ignores() {
  57     awk 'ARGV[1] == FILENAME {
  58         map[$1] = 1
  59         next
  60     }
  61     !($1 in map) {
  62         print
  63     }' "$@"
  64 }
  65
  66 select_ignores_pkgspec() {
  67     awk -F'/' -v needle="$1" '{
  68         if (NF == 2 && $1 == needle) {
  69             print $2
  70         } else if (NF == 1) {
  71             print $1
  72         }
  73     }' "${@:2}"
  74 }
  75
  76 swap() {
  77     awk '{print $2 "\t" $1}' "$@"
  78 }
  79
  80 complement() {
  81     # empty set should not return 1
  82     grep -Fxvf "$@" || return $(( $? - 1 ))
  83 }
  84
  85 trap_exit() {
  86     if [[ ! -v AUR_DEBUG ]]; then
  87         rm -rf -- "$tmp"
  88     else
  89         printf >&2 'AUR_DEBUG: %s: temporary files at %s\n' "$argv0" "$tmp"
  90     fi
  91 }
  92
  93 usage() {
  94     printf >&2 'usage: %s [-d repo] [--root path] [-cdfoSu] pkgname...\n' "$argv0"
  95     exit 1
  96 }
  97
  98 source /usr/share/makepkg/util/message.sh
  99
 100 if [[ ! -v NO_COLOR ]] && [[ ! -v AUR_DEBUG ]]; then
 101     [[ -t 2 ]] && colorize
 102 fi
 103
 104 # mollyguard for makepkg
 105 if (( UID == 0 )) && [[ ! -v AUR_ASROOT ]]; then
 106     printf >&2 'warning: aur-%s is not meant to be run as root.\n' "$argv0"
 107     printf >&2 'warning: To proceed anyway, set the %s variable.\n' 'AUR_ASROOT'
 108     exit 1
 109 fi
 110
 111 # option parsing
 112 opt_short='d:D:k:U:AcfLnorRSTuv'
 113 opt_long=('bind:' 'bind-rw:' 'database:' 'directory:' 'ignore:' 'root:'
 114           'makepkg-conf:' 'pacman-conf:' 'chroot' 'continue' 'force'
 115           'ignore-arch' 'log' 'no-confirm' 'no-ver' 'no-graph' 'no-ver-argv'
 116           'no-view' 'no-provides' 'no-build' 'rmdeps' 'sign' 'temp' 'upgrades'
 117           'pkgver' 'rebuild' 'rebuild-tree' 'rebuild-all' 'ignore-file:'
 118           'remove' 'provides-from:' 'new' 'prevent-downgrade' 'verify'
 119           'makepkg-args:' 'format:' 'no-check' 'keep-going:' 'user:'
 120           'rebase' 'reset' 'ff' 'exclude:' 'columns' 'prefix')
 121 opt_hidden=('dump-options' 'allan' 'ignorearch' 'ignorefile:' 'noconfirm'
 122             'nover' 'nograph' 'nover-argv' 'noview' 'noprovides' 'nobuild'
 123             'rebuildall' 'rebuildtree' 'rm-deps' 'gpg-sign' 'margs:' 'nocheck'
 124             'no-checkdepends' 'nocheckdepends' 'optdepends' 'repo:')
 125
 126 if opts=$(getopt -o "$opt_short" -l "$(args_csv "${opt_long[@]}" "${opt_hidden[@]}")" -n "$argv0" -- "$@"); then
 127     eval set -- "$opts"
 128 else
 129     usage
 130 fi
 131
 132 unset pkg pkg_i repo repo_p ignore_file
 133 while true; do
 134     case "$1" in
 135         # sync options
 136         --allan)
 137             rotate=1 ;;
 138         --continue)
 139             download=0 ;;
 140         --ignore)
 141             shift; IFS=, read -a pkg -r <<< "$1"
 142             pkg_i+=("${pkg[@]}") ;;
 143         --ignorefile|--ignore-file)
 144             shift; ignore_file=$1 ;;
 145         -k|--keep-going)
 146             shift; keep_going=$1 ;;
 147         -o|--nobuild|--no-build)
 148             build=0 ;;
 149         --columns)
 150             build=0; columns=1 ;;
 151         --optdepends)
 152             depends_args+=(--optdepends)
 153             graph_args+=(-v OPTDEPENDS=1) ;;
 154         --nocheck|--no-check|--nocheckdepends|--no-checkdepends)
 155             depends_args+=(--no-checkdepends)
 156             build_args+=(--no-check)
 157             graph_args+=(-v CHECKDEPENDS=0) ;;
 158         --nograph|--no-graph)
 159             graph=0 ;;
 160         --nover|--no-ver)
 161             chkver_depth=0 ;;
 162         --nover-argv|--no-ver-argv)
 163             chkver_depth=1 ;;
 164         --noview|--no-view)
 165             view=0 ;;
 166         --noprovides|--no-provides)
 167             provides=0 ;;
 168         --provides-from)
 169             shift; IFS=, read -a repo -r <<< "$1"
 170             repo_p+=("${repo[@]}")
 171             provides=1 ;;
 172         --rebuild)
 173             build_args+=(-f); chkver_depth=1 ;;
 174         --rebuildtree|--rebuild-tree)
 175             build_args+=(-f); chkver_depth=0 ;;
 176         --rebuildall|--rebuild-all)
 177             build_args+=(-f); chkver_depth=0; repo_targets=1 ;;
 178         -u|--upgrades)
 179             update=1 ;;
 180         # database options
 181         -d|--database|--repo)
 182             shift; repo_args+=(-d "$1") ;;
 183         --root)
 184             shift; repo_args+=(-r "$1") ;;
 185         # fetch options
 186         --ff)
 187             fetch_args+=(--ff) ;;
 188         --rebase)
 189             fetch_args+=(--rebase) ;;
 190         --reset)
 191             fetch_args+=(--reset) ;;
 192         # view options
 193         --format)
 194             shift; view_args+=(--format "$1") ;;
 195         --exclude)
 196             shift; view_args+=(--exclude "$1") ;;
 197         --prefix)
 198             view_args+=(--prefix) ;;  # experimental
 199         # build options
 200         -c|--chroot)
 201             build_args+=(--chroot) ;;
 202         -f|--force)
 203             build_args+=(--force) ;;
 204         --makepkg-args|--margs)
 205             shift; build_args+=(--margs "$1") ;;
 206         --makepkg-conf)
 207             shift; build_args+=(--makepkg-conf "$1") ;;
 208         --pacman-conf)
 209             shift; build_args+=(--pacman-conf "$1")
 210             repo_args+=(--config "$1")
 211             filter_args+=(--config "$1") ;;
 212         --pkgver)
 213             build_args+=(--pkgver) ;;
 214         -S|--sign|--gpg-sign)
 215             build_args+=(--sign) ;;
 216         -U|--user)
 217             shift; build_args+=(--user "$1") ;;
 218         # build options (devtools)
 219         -D|--directory)
 220             shift; build_args+=(--directory "$1") ;;
 221         --bind)
 222             shift; build_args+=(--bind "$1") ;;
 223         --bind-rw)
 224             shift; build_args+=(--bind-rw "$1") ;;
 225         -T|--temp)
 226             build_args+=(-T) ;;
 227         # build options (makepkg)
 228         -A|--ignorearch|--ignore-arch)
 229             build_args+=(--ignorearch) ;;
 230         -L|--log)
 231             build_args+=(--log) ;;
 232         -n|--noconfirm|--no-confirm)
 233             build_args+=(--noconfirm) ;;
 234         -r|--rmdeps|--rm-deps)
 235             build_args+=(--rmdeps) ;;
 236         # build options (repo-add)
 237         -R|--remove)
 238             build_args+=(--remove) ;;
 239         -v|--verify)
 240             build_args+=(--verify) ;;
 241         --prevent-downgrade)
 242             build_args+=(--prevent-downgrade) ;;
 243         --new)
 244             build_args+=(--new) ;;
 245         # other options
 246         --dump-options)
 247             printf -- '--%s\n' "${opt_long[@]}" ${AUR_DEBUG+"${opt_hidden[@]}"}
 248             printf -- '%s' "${opt_short}" | sed 's/.:\?/-&\n/g'
 249             exit ;;
 250         --) shift; break ;;
 251     esac
 252     shift
 253 done
 254
 255 # shellcheck disable=SC2174
 256 mkdir -pm 0700 -- "${TMPDIR:-/tmp}/aurutils-$UID"
 257 tmp=$(mktemp -d --tmpdir "aurutils-$UID/$argv0.XXXXXXXX")
 258 trap 'trap_exit' EXIT
 259
 260 # Default to showing PKGBUILD first in patch. (#399)
 261 orderfile=$XDG_CONFIG_HOME/aurutils/$argv0/orderfile
 262 mkdir -p -- "${orderfile%/*}"
 263
 264 if [[ ! -s $orderfile ]]; then
 265     printf 'PKGBUILD\n' > "$orderfile"
 266 fi
 267
 268 if (( rotate )); then
 269     if { hash caesar && target=$(aur pkglist | shuf -n 1); } 2>/dev/null; then
 270         exec bash -c "{ aur \"$argv0\" -c \"$target\" && repo-elephant | caesar 13; } 2>&1 | caesar 13"
 271     else
 272         echo '?'; exit 16 # EBUSY
 273     fi
 274 fi
 275
 276 mkdir -p -- "$AURDEST"
 277
 278 if (( $# + update + repo_targets == 0 )); then
 279     printf >&2 '%s: no targets specified\n' "$argv0"
 280     exit 1
 281 fi
 282
 283 # Retrieve path to local repo (#448, #700)
 284 { IFS=: read -r _ db_name
 285   IFS=: read -r _ db_root
 286   IFS=: read -r _ db_path
 287 } < <(aur repo "${repo_args[@]}" --status)
 288 wait "$!"
 289
 290 if [[ -w $db_path ]]; then
 291     msg >&2 'Using [%s] repository' "$db_name"
 292 else
 293     printf >&2 '%s: %s: permission denied\n' "$argv0" "$db_path"
 294     exit 13
 295 fi
 296
 297 # Concatenate ignores from file and command-line arguments
 298 : "${ignore_file=$XDG_CONFIG_HOME/aurutils/sync/ignore}"
 299
 300 # Restrict ignores to local repository with a single pass (#880)
 301 { if [[ -r $ignore_file ]] && [[ ! -d $ignore_file ]]; then
 302       select_ignores_pkgspec "$db_name" "$ignore_file"
 303   fi
 304   if (( ${#pkg_i[@]} )); then
 305       printf '%s\n' "${pkg_i[@]}"
 306   fi
 307 } | complement <(printf '%s\n' "$@") >"$tmp"/igni
 308
 309 # Diagnostic on which packages are ignored
 310 if [[ -s $tmp/igni ]]; then
 311     mapfile -t pkg_i < "$tmp"/igni
 312     printf '%s: packages ignored: %s\n' "$argv0" "${pkg_i[*]}"
 313
 314     # Ignore dependencies from ignored targets (#592)
 315     depends_args+=(--assume-installed "$(args_csv "${pkg_i[@]}")")
 316 fi >&2
 317
 318 # Retrieve list of local repository packages ($1 pkgname $2 pkgver)
 319 aur repo-parse -p "$db_path" --list | select_ignores "$tmp"/igni - >"$tmp"/db_info
 320 repo_err=${PIPESTATUS[0]}
 321
 322 if (( repo_err )); then
 323    exit "$repo_err"
 324 fi
 325
 326 # Build list of AUR targets
 327 { if (( $# )); then
 328       # append command-line arguments
 329       printf '%s\n' "$@"
 330   fi
 331
 332   if (( repo_targets )); then
 333       # append repository packages (all)
 334       cut -f1 <"$tmp"/db_info
 335
 336   elif (( update )); then
 337       # append repository packages (updated)
 338       aur vercmp --quiet <"$tmp"/db_info
 339   fi
 340 } >"$tmp"/argv
 341
 342 # Build AUR dependency graph
 343 if [[ -s $tmp/argv ]]; then
 344     # shellcheck disable=SC2094
 345     # depends: $1 pkgname $2 required_by $3 pkgbase $4 pkgver $5 depends_type
 346     aur depends --table --reverse "${depends_args[@]}" - <"$tmp"/argv >"$tmp"/depends
 347 else
 348     printf >&2 '%s: there is nothing to do\n' "$argv0"
 349     exit
 350 fi
 351
 352 # pkginfo: $1 pkgname $2 pkgbase $3 pkgver
 353 cut -f2,5 --complement "$tmp"/depends | sort -u >"$tmp"/pkginfo
 354
 355 { cat "$tmp"/igni # ignored packages (already included by aur-depends --assume-installed)
 356
 357   # Packages with equal or newer versions are taken as complement
 358   # for the queue. If chkver_argv is enabled, packages on the
 359   # command-line are excluded from this complement.
 360   if (( chkver_depth )); then
 361       # note: AUR cannot be queried by pkgbase (FS#57230)
 362       cut -f1,3 "$tmp"/pkginfo | aur vercmp -p "$tmp"/db_info -c >"$tmp"/current
 363
 364       # shellcheck disable=SC2002
 365       case $chkver_depth in
 366           1) cat "$tmp"/current | complement "$tmp"/argv ;;
 367           2) cat "$tmp"/current ;;
 368       esac
 369   fi
 370
 371   if (( provides )); then
 372       if (( ${#repo_p[@]} )); then
 373           filter_args+=("${repo_p[@]/#/--repo=}")
 374       else
 375           filter_args+=(--sync)
 376       fi
 377
 378       # Note: this uses pacman's copy of the repo (as used by makepkg -s)
 379       cut -f1 "$tmp"/pkginfo | aur repo-filter "${filter_args[@]}" | complement "$tmp"/argv
 380   fi
 381 } >"$tmp"/filter
 382
 383 # $1 pkgname $2 required_by $3 pkgbase, filter by $1 (depends and self)
 384 cut -f1-3 "$tmp"/depends | select_ignores "$tmp"/filter - | lib32 - >"$tmp"/graph_0
 385
 386 # XXX a flat file is needed for aur-{graph,fetch,view}. `ninja` requires the
 387 # build files to be present before dependency resolution (with `ninja -n`) can
 388 # occur, and `ninja -t targets` sorts in alphabetical order. This implies
 389 # that dependency cycles cannot be resolved before retrieving files with
 390 # aur-fetch with `ninja` alone. `tsort` could either be used in this case (with
 391 # a less nice diagnostic on cycles), or fetches done in an arbitrary order
 392 # (e.g. sort -u) with checks for cycles done at build-time.
 393 if ! select_pkgbase "$tmp"/graph_0 "$tmp"/graph_0 | tee "$tmp"/graph | tsort >"$tmp"/queue; then
 394     printf >&2 '%s: dependency cycle detected\n' "$argv0"
 395     exit 22
 396 fi
 397
 398 if [[ -s $tmp/queue ]]; then
 399     cd -- "$AURDEST"
 400 else
 401     printf >&2 '%s: there is nothing to do\n' "$argv0"
 402     exit
 403 fi
 404
 405 if (( download )); then
 406     msg >&2 "Retrieving package files"
 407     aur fetch -S "${fetch_args[@]}" --discard --results "$tmp"/fetch_results - < "$tmp"/queue >&2
 408
 409     # shellcheck disable=SC2034
 410     while IFS=: read -r mode rev_old rev path; do
 411         path=${path#file://} name=${path##*/}
 412
 413         case $mode in
 414             clone)
 415                 git -C "$path" config diff.orderFile "$orderfile" ;;
 416             fetch|merge|rebase|reset)
 417                 ;;
 418         esac
 419     done < "$tmp"/fetch_results
 420 else
 421     xargs -a "$tmp"/queue stat >/dev/null || exit 2 # ensure all directories are available
 422 fi
 423
 424 # Verify dependency tree (#20)
 425 if (( graph )); then
 426     if ! { while read -r pkg; do
 427                [[ $pkg ]] && printf '%s\0' "$pkg/.SRCINFO"
 428            done
 429          } | xargs -0 cat -- | aur graph "${graph_args[@]}"
 430     then
 431         printf >&2 '%s: failed to verify dependency graph\n' "$argv0"
 432         exit 1
 433     fi < "$tmp"/queue | swap >"$tmp"/graph
 434
 435     # Recompute dependencies to include `provides` (#837)
 436     tsort < "$tmp"/graph >"$tmp"/queue
 437 fi
 438
 439 # Inspect package files
 440 if (( view )); then
 441     aur view -a "$tmp"/queue "${view_args[@]}"
 442 fi
 443
 444 # Input for aur-sync--ninja
 445 if (( columns )); then
 446     swap "$tmp"/graph | sort -k1b,1 -k1 -u
 447
 448 # Build dependency tree with ninja (#908)
 449 elif (( AUR_SYNC_USE_NINJA )); then
 450     # directory for stamp files (concurrent aur-sync processes)
 451     mkdir -p -- "$XDG_STATE_HOME"/aurutils/$argv0
 452     ninja_dir=$XDG_STATE_HOME/aurutils/$argv0/ninja-$USER-$$
 453     mkdir -- "$ninja_dir"
 454
 455     # generate build.ninja
 456     # input:  $AURDEST/pkgbase/PKGBUILD
 457     # output: $ninja_dir/pkgbase.stamp
 458     cd -- "$ninja_dir"
 459     swap "$tmp"/graph | sort -k1b,1 -k1 -u | aur sync--ninja "$AURDEST" >build.ninja \
 460         -- aur build "${build_args[@]}" -d "$db_name" --root "$db_root"
 461
 462     if (( build )) && NINJA_STATUS='[%s/%t] ' ninja -k "$keep_going"; then
 463         # remove ninja directory on successful build
 464         rm -rf "$ninja_dir"
 465
 466     elif (( build )); then
 467         # print all targets in dependency order
 468         NINJA_STATUS='[%s/%t] ' ninja -nC /var/empty -f "$PWD"/build.ninja | \
 469             # [\w@\.\-\+]: valid characters for pkgname
 470             # alternative: [^\s]+ from rule `env -C ... > pkgbase.stamp`
 471             pcregrep -o1 -o3 '(\[\d+/\d+\] )(.+?)([\w@\.\-\+]+)(\.stamp)' | while read -r status pkg
 472         do
 473             if [[ -f $pkg.stamp ]]; then
 474                 printf "${BOLD}${BLUE}%s${ALL_OFF} %s\t${BOLD}${GREEN}[OK]${ALL_OFF}\n" "$status" "$pkg"
 475             else
 476                 printf "${BOLD}${BLUE}%s${ALL_OFF} %s\t${BOLD}${RED}[FAIL]${ALL_OFF}\n" "$status" "$pkg"
 477             fi
 478         done | column -t
 479
 480         # preserve ninja directory
 481         printf '%s: build files at %s\n' "$argv0" "$(pwd -P)"
 482     else
 483         printf '%s\n' "$(pwd -P)"
 484     fi
 485
 486 elif (( build )); then
 487     aur build "${build_args[@]}" -a "$tmp"/queue -d "$db_name" --root "$db_root"
 488 else
 489     while read -r pkg; do
 490         [[ $pkg ]] && printf '%s\n' "$(pwd -P)/$pkg"
 491     done < "$tmp"/queue
 492 fi
 493
 494 # vim: set et sw=4 sts=4 ft=sh: