validate.sh

   1 #!/bin/sh
   2 # shellcheck disable=SC2086
   3
   4 # default config
   5 #######################################################################
   6
   7 # We use the default ghc in PATH as default
   8 # Use the ghc-x.y.z trigger several errors in windows:
   9 # * It triggers the max path length issue:
  10 #   See https://github.com/haskell/cabal/issues/6271#issuecomment-1065102255
  11 # * It triggers a `createProcess: does not exist` error in units tests
  12 #   See https://github.com/haskell/cabal/issues/8049
  13 HC=ghc
  14 CABAL=cabal
  15 JOBS=4
  16 LIBTESTS=true
  17 CLITESTS=true
  18 CABALSUITETESTS=true
  19 LIBONLY=false
  20 DEPSONLY=false
  21 DOCTEST=false
  22 BENCHMARKS=false
  23 VERBOSE=false
  24 HACKAGETESTSALL=false
  25
  26 TARGETS=""
  27 STEPS=""
  28 EXTRAHCS=""
  29
  30 LISTSTEPS=false
  31
  32 # Help
  33 #######################################################################
  34
  35 show_usage() {
  36 cat <<EOF
  37 ./validate.sh - build & test
  38
  39 Usage: ./validate.sh [options]
  40   A script which runs all the tests.
  41
  42 Available options:
  43   -j, --jobs JOBS                   cabal build -j argument (default:  $JOBS)
  44       --libonly                     Test only Cabal-the-library
  45       --cli                         Test both Cabal-the-library and cabal-install
  46       --(no-)run-lib-tests          Run library tests
  47       --(no-)run-cli-tests          Run client tests
  48       --(no-)run-lib-suite          Run cabal-testsuite with library
  49       --(no-)run-cli-suite          Run cabal-testsuite with client
  50   -w, --with-compiler HC            With compiler
  51       --with-cabal CABAL            With cabal-install
  52       --extra-hc HC                 Extra compiler to run test-suite with
  53       --(no-)doctest                Run doctest on library
  54       --(no-)solver-benchmarks      Build and trial run solver-benchmarks
  55       --complete-hackage-tests      Run hackage-tests on complete Hackage data
  56       --partial-hackage-tests       Run hackage-tests on parts of Hackage data
  57   -v, --verbose                     Verbose output
  58   -q, --quiet                       Less output
  59   -s, --step STEP                   Run only specific step (can be specified multiple times)
  60       --list-steps                  List steps and build-targets and exit
  61       --help                        Print this message and exit
  62 EOF
  63 }
  64
  65 # "library"
  66 #######################################################################
  67
  68 OUTPUT=$(mktemp)
  69
  70 # `red` and `green` are used to output also the spent time in white at the end.
  71 # `blue` and `cyan` are used to print the spawned command, so they only have one
  72 # argument.
  73 red () {
  74   printf "\033[0;31m%s\033[0m %s \n" "$1" "$2"
  75 }
  76 green () {
  77   printf "\033[0;32m%s\033[0m %s \n" "$1" "$2"
  78 }
  79 blue () {
  80   printf "\033[0;34m%s\033[0m\n" "$1"
  81 }
  82 cyan () {
  83   printf "\033[0;96m%s\033[0m\n" "$1"
  84 }
  85
  86 JOB_START_TIME=$(date +%s)
  87
  88 timed() {
  89     PRETTYCMD=$(echo "$@" | sed -E 's/\/home[^ ]*\/([^\/])/**\/\1/g')
  90     blue "$PRETTYCMD"
  91     start_time=$(date +%s)
  92
  93     if $VERBOSE; then
  94         "$@" 2>&1
  95     else
  96         "$@" > "$OUTPUT" 2>&1
  97     fi
  98     # echo "MOCK" > "$OUTPUT"
  99     RET=$?
 100
 101     end_time=$(date +%s)
 102     duration=$((end_time - start_time))
 103     tduration=$((end_time - JOB_START_TIME))
 104
 105     if [ $RET -eq 0 ]; then
 106         if ! $VERBOSE; then
 107             # if output is relatively short, show everything
 108             if [ "$(wc -l < "$OUTPUT")" -le 50 ]; then
 109                 cat "$OUTPUT"
 110             else
 111                 echo "..."
 112                 tail -n 20 "$OUTPUT"
 113             fi
 114
 115             rm -f "$OUTPUT"
 116         fi
 117
 118         green "<<< $PRETTYCMD" "($duration/$tduration sec)"
 119
 120         # bottom-margin
 121         echo ""
 122     else
 123         if ! $VERBOSE; then
 124             cat "$OUTPUT"
 125         fi
 126
 127         red "<<< $PRETTYCMD" "($duration/$tduration sec, $RET)"
 128         red "<<< $*" "($duration/$tduration sec, $RET)"
 129         rm -f "$OUTPUT"
 130         exit 1
 131     fi
 132 }
 133
 134 print_header() {
 135     TITLE=$1
 136     TITLEPAT="$(echo "$TITLE"|sed 's:.:=:g')"
 137     cyan "===X========================================================================== $(date +%T) ===" \
 138       | sed "s#X$TITLEPAT=# $TITLE #"
 139
 140 }
 141
 142 # getopt
 143 #######################################################################
 144
 145 while [ $# -gt 0 ]; do
 146     arg=$1
 147     case $arg in
 148         --help)
 149             show_usage
 150             exit
 151             ;;
 152         -j|--jobs)
 153             JOBS="$2"
 154             shift
 155             shift
 156             ;;
 157         --lib-only)
 158             LIBONLY=true
 159             shift
 160             ;;
 161         --cli)
 162             LIBONLY=false
 163             shift
 164             ;;
 165         --run-lib-tests)
 166             LIBTESTS=true
 167             shift
 168             ;;
 169         --no-run-lib-tests)
 170             LIBTESTS=false
 171             shift
 172             ;;
 173         --run-cli-tests)
 174             CLITESTS=true
 175             shift
 176             ;;
 177         --no-run-cli-tests)
 178             CLITESTS=false
 179             shift
 180             ;;
 181         --run-lib-suite)
 182             LIBSUITE=true
 183             shift
 184             ;;
 185         --no-run-lib-suite)
 186             LIBSUITE=false
 187             shift
 188             ;;
 189         --run-cli-suite)
 190             CLISUITE=true
 191             shift
 192             ;;
 193         --no-run-cli-suite)
 194             CLISUITE=false
 195             shift
 196             ;;
 197         -w|--with-compiler)
 198             HC=$2
 199             shift
 200             shift
 201             ;;
 202         --with-cabal)
 203             CABAL=$2
 204             shift
 205             shift
 206             ;;
 207         --extra-hc)
 208             EXTRAHCS="$EXTRAHCS $2"
 209             shift
 210             shift
 211             ;;
 212         --doctest)
 213             DOCTEST=true
 214             shift
 215             ;;
 216         --no-doctest)
 217             DOCTEST=false
 218             shift
 219             ;;
 220         --solver-benchmarks)
 221             BENCHMARKS=true
 222             shift
 223             ;;
 224         --no-solver-benchmarks)
 225             BENCHMARKS=false
 226             shift
 227             ;;
 228         --complete-hackage-tests)
 229             HACKAGETESTSALL=true
 230             shift
 231             ;;
 232         --partial-hackage-tests)
 233             HACKAGETESTSALL=false
 234             shift
 235             ;;
 236         -v|--verbose)
 237             VERBOSE=true
 238             shift
 239             ;;
 240         -q|--quiet)
 241             VERBOSE=false
 242             shift
 243             ;;
 244         -s|--step)
 245             STEPS="$STEPS $2"
 246             shift
 247             shift
 248             ;;
 249         --list-steps)
 250             LISTSTEPS=true
 251             shift
 252             ;;
 253         *)
 254             echo "Unknown option $arg"
 255             exit 1
 256     esac
 257 done
 258
 259 # calculate steps and build targets
 260 #######################################################################
 261
 262 # If there are no explicit steps given calculate them
 263 if $LIBONLY; then
 264     CLITESTS=false
 265     CLISUITE=false
 266     BENCHMARKS=false
 267 fi
 268
 269 if [ -z "$STEPS" ]; then
 270     STEPS="print-config print-tool-versions"
 271     STEPS="$STEPS build"
 272     if $DOCTEST;    then STEPS="$STEPS doctest";   fi
 273     if $LIBTESTS;   then STEPS="$STEPS lib-tests"; fi
 274     if $LIBSUITE;   then STEPS="$STEPS lib-suite"; fi
 275     if $LIBSUITE && [ -n "$EXTRAHCS" ];
 276                     then STEPS="$STEPS lib-suite-extras"; fi
 277     if $CLITESTS;   then STEPS="$STEPS cli-tests"; fi
 278     if $CLISUITE;   then STEPS="$STEPS cli-suite"; fi
 279     if $BENCHMARKS; then STEPS="$STEPS solver-benchmarks-tests solver-benchmarks-run"; fi
 280     STEPS="$STEPS time-summary"
 281 fi
 282
 283 TARGETS="Cabal cabal-testsuite Cabal-tests Cabal-QuickCheck Cabal-tree-diff Cabal-described"
 284 if ! $LIBONLY;  then TARGETS="$TARGETS cabal-install cabal-install-solver cabal-benchmarks"; fi
 285 if $BENCHMARKS; then TARGETS="$TARGETS solver-benchmarks"; fi
 286
 287 if $LISTSTEPS; then
 288   echo "Targets: $TARGETS"
 289   echo "Steps:   $STEPS"
 290   exit
 291 fi
 292
 293 # Adjust runtime configuration
 294 #######################################################################
 295
 296 TESTSUITEJOBS="-j$JOBS"
 297 JOBS="-j$JOBS"
 298
 299 # assume compiler is GHC
 300 RUNHASKELL=$(echo "$HC" | sed -E 's/ghc(-[0-9.]*)$/runghc\1/')
 301
 302 case "$(uname)" in
 303     MINGW64*)
 304         ARCH="x86_64-windows"
 305         ;;
 306     Linux   )
 307         ARCH="x86_64-linux"
 308         ;;
 309     *)
 310         ARCH="x86_64-osx"
 311         ;;
 312 esac
 313
 314 if $LIBONLY; then
 315     PROJECTFILE=cabal.project.validate.libonly
 316 else
 317     PROJECTFILE=cabal.project.validate
 318 fi
 319
 320 BASEHC=ghc-$($HC --numeric-version)
 321 BUILDDIR=dist-newstyle-validate-$BASEHC
 322 CABAL_TESTSUITE_BDIR="$(pwd)/$BUILDDIR/build/$ARCH/$BASEHC/cabal-testsuite-3"
 323
 324 CABALNEWBUILD="${CABAL} build $JOBS -w $HC --builddir=$BUILDDIR --project-file=$PROJECTFILE"
 325 CABALLISTBIN="${CABAL} list-bin --builddir=$BUILDDIR --project-file=$PROJECTFILE"
 326
 327 # header
 328 #######################################################################
 329
 330 step_print_config() {
 331 print_header print-config
 332
 333 cat <<EOF
 334 compiler:            $HC
 335 runhaskell:          $RUNHASKELL
 336 cabal-install:       $CABAL
 337 jobs:                $JOBS
 338 Cabal tests:         $LIBTESTS
 339 cabal-install tests: $CLITESTS
 340 cabal-testsuite:     $CABALSUITETESTS
 341 library only:        $LIBONLY
 342 dependencies only:   $DEPSONLY
 343 doctest:             $DOCTEST
 344 benchmarks:          $BENCHMARKS
 345 verbose:             $VERBOSE
 346 extra compilers:     $EXTRAHCS
 347
 348 EOF
 349 }
 350
 351 step_print_tool_versions() {
 352 print_header print-tool-versions
 353
 354 timed "$HC" --version
 355 timed "$CABAL" --version
 356
 357 for EXTRAHC in $EXTRAHCS; do
 358     timed "$EXTRAHC" --version
 359 done
 360 }
 361
 362 step_time_summary() {
 363     print_header END
 364
 365     JOB_END_TIME=$(date +%s)
 366     tduration=$((JOB_END_TIME - JOB_START_TIME))
 367
 368     cyan "!!! Validation took $tduration seconds."
 369 }
 370
 371 # build
 372 #######################################################################
 373
 374 step_build() {
 375 print_header "build"
 376 print_header "Step Build: dry run"
 377 timed $CABALNEWBUILD $TARGETS --dry-run || exit 1
 378 print_header "Step Build: full build plan (cached and to-be-built dependencies):"
 379 jq -r '."install-plan" | map(."pkg-name" + "-" + ."pkg-version" + " " + ."component-name") | join("\n")' "$BUILDDIR/cache/plan.json"
 380 print_header "Step Build: actual build"
 381 timed $CABALNEWBUILD $TARGETS || exit 1
 382 }
 383
 384 # Cabal lib
 385 #######################################################################
 386
 387 step_doctest() {
 388 print_header "Cabal: doctest"
 389 cabal-env --name doctest-Cabal --transitive QuickCheck
 390 cabal-env --name doctest-Cabal array bytestring containers deepseq directory filepath pretty process time binary unix text parsec mtl
 391 timed doctest -package-env=doctest-Cabal --fast Cabal/Distribution Cabal/Language
 392 }
 393
 394 step_lib_tests() {
 395 print_header "Cabal: tests"
 396
 397 CMD="$($CABALLISTBIN Cabal-tests:test:unit-tests) $TESTSUITEJOBS --hide-successes --with-ghc=$HC"
 398 (cd Cabal-tests && timed $CMD) || exit 1
 399
 400 CMD="$($CABALLISTBIN Cabal-tests:test:check-tests) $TESTSUITEJOBS --hide-successes"
 401 (cd Cabal-tests && timed $CMD) || exit 1
 402
 403 CMD="$($CABALLISTBIN Cabal-tests:test:parser-tests) $TESTSUITEJOBS --hide-successes"
 404 (cd Cabal-tests && timed $CMD) || exit 1
 405
 406 CMD="$($CABALLISTBIN Cabal-tests:test:rpmvercmp) $TESTSUITEJOBS --hide-successes"
 407 (cd Cabal-tests && timed $CMD) || exit 1
 408
 409 CMD="$($CABALLISTBIN Cabal-tests:test:no-thunks-test) $TESTSUITEJOBS --hide-successes"
 410 (cd Cabal-tests && timed $CMD) || exit 1
 411
 412 CMD=$($CABALLISTBIN Cabal-tests:test:hackage-tests)
 413 (cd Cabal-tests && timed $CMD read-fields) || exit 1
 414 if $HACKAGETESTSALL; then
 415     (cd Cabal-tests && timed $CMD parsec)    || exit 1
 416     (cd Cabal-tests && timed $CMD roundtrip) || exit 1
 417 else
 418     (cd Cabal-tests && timed $CMD parsec d)    || exit 1
 419     (cd Cabal-tests && timed $CMD roundtrip k) || exit 1
 420 fi
 421 }
 422
 423 # Cabal cabal-testsuite
 424 #######################################################################
 425
 426 step_lib_suite() {
 427 print_header "Cabal: cabal-testsuite"
 428
 429 CMD="$($CABALLISTBIN cabal-testsuite:exe:cabal-tests) --builddir=$CABAL_TESTSUITE_BDIR $TESTSUITEJOBS --with-ghc=$HC --hide-successes"
 430 (cd cabal-testsuite && timed $CMD) || exit 1
 431 }
 432
 433 step_lib_suite_extras() {
 434 for EXTRAHC in $EXTRAHCS; do
 435
 436 CMD="$($CABALLISTBIN cabal-testsuite:exe:cabal-tests) --builddir=$CABAL_TESTSUITE_BDIR $TESTSUITEJOBS --with-ghc=$EXTRAHC --hide-successes"
 437 (cd cabal-testsuite && timed $CMD) || exit 1
 438
 439 done
 440 }
 441
 442 # cabal-install
 443 #######################################################################
 444
 445 step_cli_tests() {
 446 print_header "cabal-install: tests"
 447
 448 # this are sorted in asc time used, quicker tests first.
 449 CMD="$($CABALLISTBIN cabal-install:test:long-tests) $TESTSUITEJOBS --hide-successes"
 450 (cd cabal-install && timed $CMD) || exit 1
 451
 452 # This doesn't work in parallel either
 453 CMD="$($CABALLISTBIN cabal-install:test:unit-tests) -j1 --hide-successes"
 454 (cd cabal-install && timed $CMD) || exit 1
 455
 456 # Only single job, otherwise we fail with "Heap exhausted"
 457 CMD="$($CABALLISTBIN cabal-install:test:mem-use-tests) -j1 --hide-successes"
 458 (cd cabal-install && timed $CMD) || exit 1
 459
 460 # This test-suite doesn't like concurrency
 461 CMD="$($CABALLISTBIN cabal-install:test:integration-tests2) -j1 --hide-successes --with-ghc=$HC"
 462 (cd cabal-install && timed $CMD) || exit 1
 463 }
 464
 465 # cabal-install cabal-testsuite
 466 #######################################################################
 467
 468 step_cli_suite() {
 469 print_header "cabal-install: cabal-testsuite"
 470
 471 CMD="$($CABALLISTBIN cabal-testsuite:exe:cabal-tests) --builddir=$CABAL_TESTSUITE_BDIR --with-cabal=$($CABALLISTBIN cabal-install:exe:cabal) $TESTSUITEJOBS  --with-ghc=$HC --hide-successes --intree-cabal-lib=$PWD --test-tmp=$PWD/testdb"
 472 (cd cabal-testsuite && timed $CMD) || exit 1
 473 }
 474
 475 # solver-benchmarks
 476 #######################################################################
 477
 478 step_solver_benchmarks_tests() {
 479 print_header "solver-benchmarks: test"
 480
 481 CMD="$($CABALLISTBIN solver-benchmarks:test:unit-tests)"
 482 (cd Cabal && timed $CMD) || exit 1
 483 }
 484
 485 step_solver_benchmarks_run() {
 486 print_header "solver-benchmarks: run"
 487
 488 SOLVEPKG=Chart-diagrams
 489 CMD="$($CABALLISTBIN solver-benchmarks:exe:hackage-benchmark) --cabal1=$CABAL --cabal2=$($CABALLISTBIN cabal-install:exe:cabal) --trials=5 --packages=$SOLVEPKG --print-trials"
 490 (cd Cabal && timed $CMD) || exit 1
 491 }
 492
 493 # Steps dispatcher
 494 #######################################################################
 495
 496 for step in $STEPS; do
 497     case $step in
 498         print-config)             step_print_config            ;;
 499         print-tool-versions)      step_print_tool_versions     ;;
 500         build)                    step_build                   ;;
 501         doctest)                  step_doctest                 ;;
 502         lib-tests)                step_lib_tests               ;;
 503         cli-tests)                step_cli_tests               ;;
 504         lib-suite)                step_lib_suite               ;;
 505         lib-suite-extras)         step_lib_suite_extras        ;;
 506         cli-suite)                step_cli_suite               ;;
 507         solver-benchmarks-tests)  step_solver_benchmarks_tests ;;
 508         solver-benchmarks-run)    step_solver_benchmarks_run   ;;
 509         time-summary)             step_time_summary            ;;
 510         *)
 511             echo "Invalid step $step"
 512             exit 1
 513             ;;
 514     esac
 515 done
 516
 517 #######################################################################