lib/aur-sync

   1 #!/bin/bash
   2 # aur-sync - download and build AUR packages automatically
   3 [[ -v AUR_DEBUG ]] && set -o xtrace
   4 set -o errexit
   5 argv0=sync
   6 XDG_CACHE_HOME=${XDG_CACHE_HOME:-$HOME/.cache}
   7 XDG_CONFIG_HOME=${XDG_CONFIG_HOME:-$HOME/.config}
   8 XDG_STATE_HOME=${XDG_STATE_HOME:-$HOME/.local/state}
   9 AURDEST=${AURDEST:-$XDG_CACHE_HOME/aurutils/$argv0}
  10 AUR_SYNC_USE_NINJA=${AUR_SYNC_USE_NINJA:-0}
  11 PS4='+(${BASH_SOURCE}:${LINENO}): ${FUNCNAME[0]:+${FUNCNAME[0]}(): }'
  12
  13 # Avoid CDPATH screwing with cd (#1047)
  14 unset -v CDPATH
  15
  16 # default arguments
  17 build_args=(--syncdeps) build_repo_args=()
  18 depends_args=() view_args=() filter_args=() fetch_args=() graph_args=() reparse_args=()
  19
  20 # default options
  21 build=1 chkver_depth=2 download=1 view=1 provides=1 graph=1 keep_going=1
  22
  23 # default options (disabled)
  24 rotate=0 update=0 repo_targets=0 columns=0
  25
  26 args_csv() {
  27     # shellcheck disable=SC2155
  28     local str=$(printf '%s,' "$@")
  29     printf '%s' "${str%,}"
  30 }
  31
  32 lib32() {
  33     awk -v arch="$(uname -m)" '{
  34         if(arch == "i686") {
  35             gsub(/^lib32-/,"")
  36             gsub(/^gcc-multilib$/,"")
  37         }
  38         print
  39     }' "$@"
  40 }
  41
  42 select_pkgspec() {
  43     awk -F'/' -v needle="$1" '{
  44         if (NF == 2 && $1 == needle) {
  45             print $2
  46         }
  47         else if (NF == 1) {
  48             print $1
  49         }
  50     }' "${@:2}"
  51 }
  52
  53 swap() {
  54     awk '{print $2 "\t" $1}' "$@"
  55 }
  56
  57 complement() {
  58     # empty set should not return 1
  59     grep -Fxvf "$@" || return $(( $? - 1 ))
  60 }
  61
  62 trap_exit() {
  63     if [[ ! -v AUR_DEBUG ]]; then
  64         rm -rf -- "$tmp"
  65     else
  66         printf >&2 'AUR_DEBUG: %s: temporary files at %s\n' "$argv0" "$tmp"
  67     fi
  68 }
  69
  70 usage() {
  71     printf >&2 'usage: %s [-d repo] [--root path] [-cdfoSu] pkgname...\n' "$argv0"
  72     exit 1
  73 }
  74
  75 source /usr/share/makepkg/util/message.sh
  76
  77 if [[ ! -v NO_COLOR ]] && [[ ! -v AUR_DEBUG ]]; then
  78     [[ -t 2 ]] && colorize
  79 fi
  80
  81 # mollyguard for makepkg
  82 if (( UID == 0 )) && [[ ! -v AUR_ASROOT ]]; then
  83     printf >&2 'warning: aur-%s is not meant to be run as root.\n' "$argv0"
  84     printf >&2 'warning: To proceed anyway, set the %s variable.\n' 'AUR_ASROOT'
  85     exit 1
  86 fi
  87
  88 # option parsing
  89 opt_short='d:D:k:U:ACcfLnorRSTuv'
  90 opt_long=('bind:' 'bind-rw:' 'database:' 'directory:' 'ignore:' 'root:'
  91           'makepkg-conf:' 'pacman-conf:' 'chroot' 'continue' 'force' 'ignore-arch'
  92           'log' 'no-confirm' 'no-ver' 'no-graph' 'no-sync' 'no-ver-argv' 'no-view'
  93           'no-provides' 'no-build' 'rmdeps' 'sign' 'temp' 'upgrades' 'pkgver'
  94           'rebuild' 'rebuild-tree' 'rebuild-all' 'ignore-file:' 'remove'
  95           'provides-from:' 'new' 'prevent-downgrade' 'verify' 'makepkg-args:'
  96           'format:' 'no-check' 'keep-going:' 'user:' 'rebase' 'reset' 'ff' 'exclude:'
  97           'columns' 'prefix' 'save:' 'clean' 'cleanbuild')
  98 opt_hidden=('dump-options' 'allan' 'ignorearch' 'ignorefile:' 'noconfirm'
  99             'nover' 'nograph' 'nosync' 'nover-argv' 'noview' 'noprovides' 'nobuild'
 100             'rebuildall' 'rebuildtree' 'rm-deps' 'gpg-sign' 'margs:' 'nocheck'
 101             'no-checkdepends' 'nocheckdepends' 'optdepends' 'repo:')
 102
 103 if opts=$(getopt -o "$opt_short" -l "$(args_csv "${opt_long[@]}" "${opt_hidden[@]}")" -n "$argv0" -- "$@"); then
 104     eval set -- "$opts"
 105 else
 106     usage
 107 fi
 108
 109 unset pkg pkg_i repo repo_p ignore_file stdout_file
 110 while true; do
 111     case "$1" in
 112         # sync options
 113         --allan)
 114             rotate=1 ;;
 115         --continue)
 116             download=0 ;;
 117         --ignore)
 118             shift; IFS=, read -a pkg -r <<< "$1"
 119             pkg_i+=("${pkg[@]}") ;;
 120         --ignorefile|--ignore-file)
 121             shift; ignore_file=$1 ;;
 122         -k|--keep-going)
 123             shift; keep_going=$1 ;;
 124         -o|--nobuild|--no-build)
 125             build=0 ;;
 126         --columns)
 127             build=0; columns=1 ;;
 128         --save)
 129             shift; stdout_file=$1 ;;
 130         --optdepends)
 131             depends_args+=(--optdepends)
 132             graph_args+=(-v OPTDEPENDS=1) ;;
 133         --nocheck|--no-check|--nocheckdepends|--no-checkdepends)
 134             depends_args+=(--no-checkdepends)
 135             build_args+=(--no-check)
 136             graph_args+=(-v CHECKDEPENDS=0) ;;
 137         --nograph|--no-graph)
 138             graph=0 ;;
 139         --nosync|--no-sync)
 140             build_args+=(--no-sync) ;;
 141         --nover|--no-ver)
 142             chkver_depth=0 ;;
 143         --nover-argv|--no-ver-argv)
 144             chkver_depth=1 ;;
 145         --noview|--no-view)
 146             view=0 ;;
 147         --noprovides|--no-provides)
 148             provides=0 ;;
 149         --provides-from)
 150             shift; IFS=, read -a repo -r <<< "$1"
 151             repo_p+=("${repo[@]}")
 152             provides=1 ;;
 153         --rebuild)
 154             # Command-line targets are excluded from `repo-filter`
 155             build_args+=(-f); chkver_depth=1 ;;
 156         --rebuildtree|--rebuild-tree)
 157             # Dependencies may be removed by `repo-filter` (#1066)
 158             build_args+=(-f); chkver_depth=0; provides=0 ;;
 159         --rebuildall|--rebuild-all)
 160             build_args+=(-f); chkver_depth=0; repo_targets=1 ;;
 161         -u|--upgrades)
 162             update=1 ;;
 163         # database options
 164         -d|--database|--repo)
 165             shift; build_repo_args+=(-d "$1") ;;
 166         --root)
 167             shift; build_repo_args+=(-r "$1") ;;
 168         # fetch options
 169         --ff)
 170             fetch_args+=(--ff) ;;
 171         --rebase)
 172             fetch_args+=(--rebase) ;;
 173         --reset)
 174             fetch_args+=(--reset) ;;
 175         # view options
 176         --format)
 177             shift; view_args+=(--format "$1") ;;
 178         --exclude)
 179             shift; view_args+=(--exclude "$1") ;;
 180         --prefix)
 181             view_args+=(--prefix) ;;  # experimental
 182         # build options
 183         -c|--chroot)
 184             build_args+=(--chroot) ;;
 185         -f|--force)
 186             build_args+=(--force) ;;
 187         -C|--clean)
 188             build_args+=(--clean) ;;
 189         --cleanbuild)
 190             build_args+=(--cleanbuild) ;;
 191         --makepkg-args|--margs)
 192             shift; build_args+=(--margs "$1") ;;
 193         --makepkg-conf)
 194             shift; build_args+=(--makepkg-conf "$1") ;;
 195         --pacman-conf)
 196             shift; build_args+=(--pacman-conf "$1")
 197             filter_args+=(--config "$1") ;;
 198         --pkgver)
 199             build_args+=(--pkgver) ;;
 200         -S|--sign|--gpg-sign)
 201             build_args+=(--sign) ;;
 202         -U|--user)
 203             shift; build_args+=(--user "$1") ;;
 204         # build options (devtools)
 205         -D|--directory)
 206             shift; build_args+=(--directory "$1") ;;
 207         --bind)
 208             shift; build_args+=(--bind "$1") ;;
 209         --bind-rw)
 210             shift; build_args+=(--bind-rw "$1") ;;
 211         -T|--temp)
 212             build_args+=(-T) ;;
 213         # build options (makepkg)
 214         -A|--ignorearch|--ignore-arch)
 215             build_args+=(--ignorearch) ;;
 216         -L|--log)
 217             build_args+=(--log) ;;
 218         -n|--noconfirm|--no-confirm)
 219             build_args+=(--noconfirm) ;;
 220         -r|--rmdeps|--rm-deps)
 221             build_args+=(--rmdeps) ;;
 222         # build options (repo-add)
 223         -R|--remove)
 224             build_args+=(--remove) ;;
 225         -v|--verify)
 226             build_args+=(--verify) ;;
 227         --prevent-downgrade)
 228             build_args+=(--prevent-downgrade) ;;
 229         --new)
 230             build_args+=(--new) ;;
 231         # other options
 232         --dump-options)
 233             printf -- '--%s\n' "${opt_long[@]}" ${AUR_DEBUG+"${opt_hidden[@]}"}
 234             printf -- '%s' "${opt_short}" | sed 's/.:\?/-&\n/g'
 235             exit ;;
 236         --) shift; break ;;
 237     esac
 238     shift
 239 done
 240
 241 # shellcheck disable=SC2174
 242 mkdir -pm 0700 -- "${TMPDIR:-/tmp}/aurutils-$UID"
 243 tmp=$(mktemp -d --tmpdir "aurutils-$UID/$argv0.XXXXXXXX")
 244 trap 'trap_exit' EXIT
 245
 246 # Default to showing PKGBUILD first in patch. (#399)
 247 orderfile=$XDG_CONFIG_HOME/aurutils/$argv0/orderfile
 248 mkdir -p -- "${orderfile%/*}"
 249
 250 if [[ ! -s $orderfile ]]; then
 251     printf 'PKGBUILD\n' > "$orderfile"
 252 fi
 253
 254 if (( rotate )); then
 255     if { hash caesar && target=$(aur pkglist | shuf -n 1); } 2>/dev/null; then
 256         exec bash -c "{ aur \"$argv0\" -c \"$target\" && repo-elephant | caesar 13; } 2>&1 | caesar 13"
 257     else
 258         echo '?'; exit 16 # EBUSY
 259     fi
 260 fi
 261 mkdir -p -- "$AURDEST"
 262
 263 if (( $# + update + repo_targets == 0 )); then
 264     printf >&2 '%s: no targets specified\n' "$argv0"
 265     exit 1
 266 fi
 267
 268 # Write --no-build / --columns results to a file (#1077)
 269 if [[ -v stdout_file ]]; then
 270     stdout_file=$(realpath -- "$stdout_file")
 271 fi
 272
 273 # Retrieve path to local repo (#448, #700, #1135)
 274 { IFS=: read -r _ db_name
 275   IFS=: read -r _ db_root
 276   IFS=: read -r _ db_path
 277   IFS=: read -r _ _
 278   IFS=: read -r _ _
 279 } < <(aur build "${build_args[@]}" "${build_repo_args[@]}" --status)
 280 wait "$!"
 281
 282 msg >&2 'Using [%s] repository' "$db_name"
 283
 284 # Ignores file can be the result of process substitution (#880)
 285 : "${ignore_file=$XDG_CONFIG_HOME/aurutils/sync/ignore}"
 286
 287 if [[ -r $ignore_file ]] && [[ ! -d $ignore_file ]]; then
 288     # Append file ignores to command-line ignores
 289     while read -r i; do
 290         pkg_i+=("$i")
 291     done < <(select_pkgspec "$db_name" "$ignore_file")
 292 fi
 293
 294 # Diagnostic on which packages are ignored
 295 if (( ${#pkg_i[@]} )); then
 296     # Do not include command-line arguments as ignores (#952)
 297     mapfile -t pkg_i < <(complement <(printf '%s\n' "$@") <(printf '%s\n' "${pkg_i[@]}"))
 298
 299     printf '%s: packages ignored: %s\n' "$argv0" "${pkg_i[*]}"
 300
 301     # Ignore local repository targets (#1146)
 302     reparse_args+=(--ignore "$(args_csv "${pkg_i[@]}")")
 303 fi >&2
 304
 305 # Retrieve list of local repository packages ($1 pkgname $2 pkgver)
 306 aur repo-parse "${reparse_args[@]}" -p "$db_path" --list >"$tmp"/db_info
 307
 308 # Build list of AUR targets
 309 { if (( $# )); then
 310       # append command-line arguments
 311       printf '%s\n' "$@"
 312   fi
 313
 314   if (( repo_targets )); then
 315       # append repository packages (all)
 316       cut -f1 <"$tmp"/db_info
 317
 318   elif (( update )); then
 319       # append repository packages (updated)
 320       aur vercmp --quiet <"$tmp"/db_info
 321   fi
 322 } >"$tmp"/argv
 323
 324 # Build AUR dependency graph
 325 if [[ -s $tmp/argv ]]; then
 326     # shellcheck disable=SC2094
 327     aur depends --jsonl "${depends_args[@]}" - <"$tmp"/argv >"$tmp"/depends.jsonl
 328 else
 329     printf >&2 '%s: there is nothing to do\n' "$argv0"
 330     exit
 331 fi
 332
 333 # pkginfo: $1 pkgname $2 pkgbase $3 pkgver
 334 aur format -f '%n\t%b\t%v\n' "$tmp"/depends.jsonl | sort -u >"$tmp"/pkginfo
 335
 336 { if (( ${#pkg_i[@]} )); then
 337       printf '%s\n' "${pkg_i[@]}"  # Ignored packages
 338   fi
 339
 340   # Packages with equal or newer versions are taken as complement
 341   # for the queue. If chkver_argv is enabled, packages on the
 342   # command-line are excluded from this complement.
 343   if (( chkver_depth )); then
 344       # note: AUR cannot be queried by pkgbase (FS#57230)
 345       cut -f1,3 "$tmp"/pkginfo | aur vercmp -p "$tmp"/db_info -c >"$tmp"/current
 346
 347       # shellcheck disable=SC2002
 348       case $chkver_depth in
 349           1) cat "$tmp"/current | complement "$tmp"/argv ;;
 350           2) cat "$tmp"/current ;;
 351       esac
 352   fi
 353
 354   if (( provides )); then
 355       if (( ${#repo_p[@]} )); then
 356           filter_args+=("${repo_p[@]/#/--repo=}")
 357       else
 358           filter_args+=(--sync)
 359       fi
 360
 361       # Note: this uses pacman's copy of the repo (as used by makepkg -s)
 362       cut -f1 "$tmp"/pkginfo | aur repo-filter "${filter_args[@]}" | complement "$tmp"/argv
 363   fi
 364 } >"$tmp"/filter
 365
 366 # Filter out targets determined in the steps above recursively (#1136, #1140)
 367 aur sync--filter -p "$tmp"/depends.jsonl -f "$tmp"/filter >"$tmp"/graph
 368
 369 # XXX: a flat file is needed for aur-{graph,fetch,view}. `ninja` requires the
 370 # build files to be present before dependency resolution (with `ninja -n`) can
 371 # occur, and `ninja -t targets` sorts in alphabetical order. This implies
 372 # that dependency cycles cannot be resolved before retrieving files with
 373 # aur-fetch with `ninja` alone. `tsort` could either be used in this case (with
 374 # a less nice diagnostic on cycles), or fetches done in an arbitrary order
 375 # (e.g. sort -u) with checks for cycles done at build-time.
 376 if ! tsort <"$tmp"/graph >"$tmp"/queue; then
 377     printf >&2 '%s: dependency cycle detected\n' "$argv0"
 378     exit 22
 379 fi
 380
 381 if [[ -s $tmp/queue ]]; then
 382     cd -- "$AURDEST"
 383 else
 384     printf >&2 '%s: there is nothing to do\n' "$argv0"
 385     exit
 386 fi
 387
 388 if (( download )); then
 389     msg >&2 "Retrieving package files"
 390     aur fetch -S "${fetch_args[@]}" --discard --results "$tmp"/fetch_results - < "$tmp"/queue >&2
 391
 392     # shellcheck disable=SC2034
 393     while IFS=: read -r mode rev_old rev path; do
 394         path=${path#file://} name=${path##*/}
 395
 396         case $mode in
 397             clone)
 398                 git -C "$path" config diff.orderFile "$orderfile" ;;
 399             fetch|merge|rebase|reset)
 400                 ;;
 401         esac
 402     done < "$tmp"/fetch_results
 403 else
 404     xargs -a "$tmp"/queue stat >/dev/null || exit 2 # ensure all directories are available
 405 fi
 406
 407 # Verify dependency tree (#20)
 408 if (( graph )); then
 409     if ! { while read -r pkg; do
 410                [[ $pkg ]] && printf '%s\0' "$pkg/.SRCINFO"
 411            done
 412          } | xargs -0 cat -- | aur graph "${graph_args[@]}"
 413     then
 414         printf >&2 '%s: failed to verify dependency graph\n' "$argv0"
 415         exit 1
 416     fi <"$tmp"/queue | swap >"$tmp"/graph
 417
 418     # Recompute dependencies to include `provides` (#837)
 419     tsort < "$tmp"/graph >"$tmp"/queue || exit 22
 420 fi
 421
 422 { # Sort dependencies (`aur-sync--ninja`, `--columns`)
 423   swap "$tmp"/graph | sort -k1b,1 -k1 -u >"$tmp"/graph.ninja
 424
 425   # Resolve absolute paths (`--no-build`)
 426   while read -r pkg; do
 427       [[ $pkg ]] && printf '%s\n' "$AURDEST/$pkg"
 428   done <"$tmp"/queue >"$tmp"/queue.realpath
 429 }
 430
 431 # Inspect package files
 432 if (( view )); then
 433     aur view -a "$tmp"/queue "${view_args[@]}"
 434 fi
 435
 436 # `--columns` / `--no-build` output
 437 if (( columns )) && [[ -v stdout_file ]]; then
 438     cat "$tmp"/graph.ninja >"$stdout_file"
 439
 440 elif (( columns )); then
 441     cat "$tmp"/graph.ninja
 442
 443 elif ! (( build )) && [[ -v stdout_file ]]; then
 444     cat "$tmp"/queue.realpath >"$stdout_file"
 445
 446 elif ! (( build )); then
 447     cat "$tmp"/queue.realpath
 448
 449 # Build dependency tree with ninja (#908)
 450 elif (( AUR_SYNC_USE_NINJA )); then
 451     # Apply `--save` (#1091)
 452     if [[ -v stdout_file ]]; then
 453         cat "$tmp"/graph.ninja >"$stdout_file"
 454     fi
 455
 456     # Directory for stamp files (concurrent aur-sync processes)
 457     mkdir -p -- "$XDG_STATE_HOME"/aurutils/$argv0
 458     ninja_dir=$XDG_STATE_HOME/aurutils/$argv0/ninja-$USER-$$
 459     mkdir -- "$ninja_dir"
 460
 461     # Generate build.ninja
 462     # input:  $AURDEST/pkgbase/PKGBUILD
 463     # output: $ninja_dir/pkgbase.stamp
 464     aur sync--ninja "$AURDEST" <"$tmp"/graph.ninja >"$ninja_dir"/build.ninja \
 465         -- aur build "${build_args[@]}" -d "$db_name" --root "$db_root"
 466
 467     if NINJA_STATUS='[%s/%t] ' ninja -C "$ninja_dir" -k "$keep_going"; then
 468         # Remove ninja directory on successful build
 469         rm -rf "$ninja_dir"
 470     else
 471         # Print all targets in dependency order
 472         NINJA_STATUS='[%s/%t] ' ninja -nC /var/empty -f "$ninja_dir"/build.ninja | \
 473             # [\w@\.\-\+]: valid characters for pkgname
 474             # alternative: [^\s]+ from rule `env -C ... > pkgbase.stamp`
 475             pcregrep -o1 -o3 '(\[\d+/\d+\] )(.+?)([\w@\.\-\+]+)(\.stamp)' | while read -r status pkg
 476         do
 477             if [[ -f $ninja_dir/$pkg.stamp ]]; then
 478                 printf "${BOLD}${BLUE}%s${ALL_OFF} %s\t${BOLD}${GREEN}[OK]${ALL_OFF}\n" "$status" "$pkg"
 479             else
 480                 printf "${BOLD}${BLUE}%s${ALL_OFF} %s\t${BOLD}${RED}[FAIL]${ALL_OFF}\n" "$status" "$pkg"
 481             fi
 482         done | column -t
 483
 484         # Preserve ninja directory
 485         printf '%s: build files at %s\n' "$argv0" "$ninja_dir"
 486     fi
 487 else
 488     # Apply `--save` (#1091)
 489     if [[ -v stdout_file ]]; then
 490         cat "$tmp"/queue.realpath >"$stdout_file"
 491     fi
 492     aur build "${build_args[@]}" -a "$tmp"/queue.realpath -d "$db_name" --root "$db_root"
 493 fi
 494
 495 # vim: set et sw=4 sts=4 ft=sh: