validate.sh

   1 #!/bin/sh
   2 # shellcheck disable=SC2086
   3
   4 # default config
   5 #######################################################################
   6
   7 # We use the default ghc in PATH as default
   8 # Use the ghc-x.y.z trigger several errors in windows:
   9 # * It triggers the max path length issue:
  10 #   See https://github.com/haskell/cabal/issues/6271#issuecomment-1065102255
  11 # * It triggers a `createProcess: does not exist` error in units tests
  12 #   See https://github.com/haskell/cabal/issues/8049
  13 HC=ghc
  14 CABAL=cabal
  15 CABALPLAN=cabal-plan
  16 JOBS=4
  17 LIBTESTS=true
  18 CLITESTS=true
  19 CABALSUITETESTS=true
  20 LIBONLY=false
  21 DEPSONLY=false
  22 DOCTEST=false
  23 BENCHMARKS=false
  24 VERBOSE=false
  25 HACKAGETESTSALL=false
  26
  27 TARGETS=""
  28 STEPS=""
  29 EXTRAHCS=""
  30
  31 LISTSTEPS=false
  32
  33 # Help
  34 #######################################################################
  35
  36 show_usage() {
  37 cat <<EOF
  38 ./validate.sh - build & test
  39
  40 Usage: ./validate.sh [options]
  41   A script which runs all the tests.
  42
  43 Available options:
  44   -j, --jobs JOBS                   cabal v2-build -j argument (default:  $JOBS)
  45       --libonly                     Test only Cabal-the-library
  46       --cli                         Test both Cabal-the-library and cabal-install
  47       --(no-)run-lib-tests          Run library tests
  48       --(no-)run-cli-tests          Run client tests
  49       --(no-)run-lib-suite          Run cabal-testsuite with library
  50       --(no-)run-cli-suite          Run cabal-testsuite with client
  51   -w, --with-compiler HC            With compiler
  52       --with-cabal CABAL            With cabal-install
  53       --with-cabal-plan CABALPLAN   With cabal-plan
  54       --extra-hc HC                 Extra compiler to run test-suite with
  55       --(no-)doctest                Run doctest on library
  56       --(no-)solver-benchmarks      Build and trial run solver-benchmarks
  57       --complete-hackage-tests      Run hackage-tests on complete Hackage data
  58       --partial-hackage-tests       Run hackage-tests on parts of Hackage data
  59   -v, --verbose                     Verbose output
  60   -q, --quiet                       Less output
  61   -s, --step STEP                   Run only specific step (can be specified multiple times)
  62       --list-steps                  List steps and build-targets and exit
  63       --help                        Print this message and exit
  64 EOF
  65 }
  66
  67 # "library"
  68 #######################################################################
  69
  70 OUTPUT=$(mktemp)
  71
  72 RED='\033[0;31m'
  73 GREEN='\033[0;32m'
  74 BLUE='\033[0;34m'
  75 CYAN='\033[0;96m'
  76 RESET='\033[0m' # No Color
  77
  78 JOB_START_TIME=$(date +%s)
  79
  80 timed() {
  81     PRETTYCMD=$(echo "$@" | sed -E 's/\/home[^ ]*\/([^\/])/**\/\1/g')
  82     echo "$BLUE>>> $PRETTYCMD $RESET"
  83     start_time=$(date +%s)
  84
  85     if $VERBOSE; then
  86         "$@" 2>&1
  87     else
  88         "$@" > "$OUTPUT" 2>&1
  89     fi
  90     # echo "MOCK" > "$OUTPUT"
  91     RET=$?
  92
  93     end_time=$(date +%s)
  94     duration=$((end_time - start_time))
  95     tduration=$((end_time - JOB_START_TIME))
  96
  97     if [ $RET -eq 0 ]; then
  98         if ! $VERBOSE; then
  99             # if output is relatively short, show everything
 100             if [ "$(wc -l < "$OUTPUT")" -le 50 ]; then
 101                 cat "$OUTPUT"
 102             else
 103                 echo "..."
 104                 tail -n 20 "$OUTPUT"
 105             fi
 106
 107             rm -f "$OUTPUT"
 108         fi
 109
 110         echo "$GREEN<<< $PRETTYCMD $RESET ($duration/$tduration sec)"
 111
 112         # bottom-margin
 113         echo ""
 114     else
 115         if ! $VERBOSE; then
 116             cat "$OUTPUT"
 117         fi
 118
 119         echo "$RED<<< $PRETTYCMD $RESET ($duration/$tduration sec, $RET)"
 120         echo "$RED<<< $* $RESET ($duration/$tduration sec, $RET)"
 121         rm -f "$OUTPUT"
 122         exit 1
 123     fi
 124 }
 125
 126 print_header() {
 127     TITLE=$1
 128     TITLEPAT="$(echo "$TITLE"|sed 's:.:=:g')"
 129     echo "$CYAN===X============================================================ $(date +%T) ===$RESET" \
 130       | sed "s#X$TITLEPAT=# $TITLE #"
 131
 132 }
 133
 134 # getopt
 135 #######################################################################
 136
 137 while [ $# -gt 0 ]; do
 138     arg=$1
 139     case $arg in
 140         --help)
 141             show_usage
 142             exit
 143             ;;
 144         -j|--jobs)
 145             JOBS="$2"
 146             shift
 147             shift
 148             ;;
 149         --lib-only)
 150             LIBONLY=true
 151             shift
 152             ;;
 153         --cli)
 154             LIBONLY=false
 155             shift
 156             ;;
 157         --run-lib-tests)
 158             LIBTESTS=true
 159             shift
 160             ;;
 161         --no-run-lib-tests)
 162             LIBTESTS=false
 163             shift
 164             ;;
 165         --run-cli-tests)
 166             CLITESTS=true
 167             shift
 168             ;;
 169         --no-run-cli-tests)
 170             CLITESTS=false
 171             shift
 172             ;;
 173         --run-lib-suite)
 174             LIBSUITE=true
 175             shift
 176             ;;
 177         --no-run-lib-suite)
 178             LIBSUITE=false
 179             shift
 180             ;;
 181         --run-cli-suite)
 182             CLISUITE=true
 183             shift
 184             ;;
 185         --no-run-cli-suite)
 186             CLISUITE=false
 187             shift
 188             ;;
 189         -w|--with-compiler)
 190             HC=$2
 191             shift
 192             shift
 193             ;;
 194         --with-cabal)
 195             CABAL=$2
 196             shift
 197             shift
 198             ;;
 199         --with-cabal-plan)
 200             CABALPLAN=$2
 201             shift
 202             shift
 203             ;;
 204         --extra-hc)
 205             EXTRAHCS="$EXTRAHCS $2"
 206             shift
 207             shift
 208             ;;
 209         --doctest)
 210             DOCTEST=true
 211             shift
 212             ;;
 213         --no-doctest)
 214             DOCTEST=false
 215             shift
 216             ;;
 217         --solver-benchmarks)
 218             BENCHMARKS=true
 219             shift
 220             ;;
 221         --no-solver-benchmarks)
 222             BENCHMARKS=false
 223             shift
 224             ;;
 225         --complete-hackage-tests)
 226             HACKAGETESTSALL=true
 227             shift
 228             ;;
 229         --partial-hackage-tests)
 230             HACKAGETESTSALL=false
 231             shift
 232             ;;
 233         -v|--verbose)
 234             VERBOSE=true
 235             shift
 236             ;;
 237         -q|--quiet)
 238             VERBOSE=false
 239             shift
 240             ;;
 241         -s|--step)
 242             STEPS="$STEPS $2"
 243             shift
 244             shift
 245             ;;
 246         --list-steps)
 247             LISTSTEPS=true
 248             shift
 249             ;;
 250         *)
 251             echo "Unknown option $arg"
 252             exit 1
 253     esac
 254 done
 255
 256 # calculate steps and build targets
 257 #######################################################################
 258
 259 # If there are no explicit steps given calculate them
 260 if $LIBONLY; then
 261     CLITESTS=false
 262     CLISUITE=false
 263     BENCHMARKS=false
 264 fi
 265
 266 if [ -z "$STEPS" ]; then
 267     STEPS="print-config print-tool-versions"
 268     STEPS="$STEPS build"
 269     if $DOCTEST;    then STEPS="$STEPS doctest";   fi
 270     if $LIBTESTS;   then STEPS="$STEPS lib-tests"; fi
 271     if $LIBSUITE;   then STEPS="$STEPS lib-suite"; fi
 272     if $LIBSUITE && [ -n "$EXTRAHCS" ];
 273                     then STEPS="$STEPS lib-suite-extras"; fi
 274     if $CLITESTS;   then STEPS="$STEPS cli-tests"; fi
 275     if $CLISUITE;   then STEPS="$STEPS cli-suite"; fi
 276     if $BENCHMARKS; then STEPS="$STEPS solver-benchmarks-tests solver-benchmarks-run"; fi
 277     STEPS="$STEPS time-summary"
 278 fi
 279
 280 TARGETS="Cabal cabal-testsuite Cabal-tests Cabal-QuickCheck Cabal-tree-diff Cabal-described"
 281 if ! $LIBONLY;  then TARGETS="$TARGETS cabal-install cabal-install-solver cabal-benchmarks"; fi
 282 if $BENCHMARKS; then TARGETS="$TARGETS solver-benchmarks"; fi
 283
 284 if $LISTSTEPS; then
 285   echo "Targets: $TARGETS"
 286   echo "Steps:   $STEPS"
 287   exit
 288 fi
 289
 290 # Adjust runtime configuration
 291 #######################################################################
 292
 293 TESTSUITEJOBS="-j$JOBS"
 294 JOBS="-j$JOBS"
 295
 296 # assume compiler is GHC
 297 RUNHASKELL=$(echo $HC | sed -E 's/ghc(-[0-9.]*)$/runghc\1/')
 298
 299 if [ "$OSTYPE" = "msys" ]; then
 300     ARCH="x86_64-windows"
 301 elif [ "$(uname)" = "Linux" ]; then
 302     ARCH="x86_64-linux"
 303 else
 304     ARCH="x86_64-osx"
 305 fi
 306
 307 if $LIBONLY; then
 308     PROJECTFILE=cabal.project.validate.libonly
 309 else
 310     PROJECTFILE=cabal.project.validate
 311 fi
 312
 313 BASEHC=ghc-$($HC --numeric-version)
 314 BUILDDIR=dist-newstyle-validate-$BASEHC
 315 CABAL_TESTSUITE_BDIR="$(pwd)/$BUILDDIR/build/$ARCH/$BASEHC/cabal-testsuite-3"
 316
 317 CABALNEWBUILD="${CABAL} v2-build $JOBS -w $HC --builddir=$BUILDDIR --project-file=$PROJECTFILE"
 318 CABALPLANLISTBIN="${CABALPLAN} list-bin --builddir=$BUILDDIR"
 319
 320 # header
 321 #######################################################################
 322
 323 step_print_config() {
 324 print_header print-config
 325
 326 cat <<EOF
 327 compiler:            $HC
 328 runhaskell:          $RUNHASKELL
 329 cabal-install:       $CABAL
 330 cabal-plan:          $CABALPLAN
 331 jobs:                $JOBS
 332 Cabal tests:         $LIBTESTS
 333 cabal-install tests: $CLITESTS
 334 cabal-testsuite:     $CABALSUITETESTS
 335 library only:        $LIBONLY
 336 dependencies only:   $DEPSONLY
 337 doctest:             $DOCTEST
 338 benchmarks:          $BENCHMARKS
 339 verbose:             $VERBOSE
 340 extra compilers:     $EXTRAHCS
 341
 342 EOF
 343 }
 344
 345 step_print_tool_versions() {
 346 print_header print-tool-versions
 347
 348 timed $HC --version
 349 timed $CABAL --version
 350 timed $CABALPLAN --version
 351
 352 for EXTRAHC in $EXTRAHCS; do
 353     timed $EXTRAHC --version
 354 done
 355 }
 356
 357 step_time_summary() {
 358     print_header END
 359
 360     JOB_END_TIME=$(date +%s)
 361     tduration=$((JOB_END_TIME - JOB_START_TIME))
 362
 363     echo "$CYAN!!! Validation took $tduration seconds. $RESET"
 364 }
 365
 366 # build
 367 #######################################################################
 368
 369 step_build() {
 370 print_header "build"
 371 timed $CABALNEWBUILD $TARGETS --dry-run || exit 1
 372 timed $CABALNEWBUILD $TARGETS || exit 1
 373 }
 374
 375 # Cabal lib
 376 #######################################################################
 377
 378 step_doctest() {
 379 print_header "Cabal: doctest"
 380 cabal-env --name doctest-Cabal --transitive QuickCheck
 381 cabal-env --name doctest-Cabal array bytestring containers deepseq directory filepath pretty process time binary unix text parsec mtl
 382 timed doctest -package-env=doctest-Cabal --fast Cabal/Distribution Cabal/Language
 383 }
 384
 385 step_lib_tests() {
 386 print_header "Cabal: tests"
 387
 388 CMD="$($CABALPLANLISTBIN Cabal-tests:test:unit-tests) $TESTSUITEJOBS --hide-successes --with-ghc=$HC"
 389 (cd Cabal-tests && timed $CMD) || exit 1
 390
 391 CMD="$($CABALPLANLISTBIN Cabal-tests:test:check-tests) $TESTSUITEJOBS --hide-successes"
 392 (cd Cabal-tests && timed $CMD) || exit 1
 393
 394 CMD="$($CABALPLANLISTBIN Cabal-tests:test:parser-tests) $TESTSUITEJOBS --hide-successes"
 395 (cd Cabal-tests && timed $CMD) || exit 1
 396
 397 CMD="$($CABALPLANLISTBIN Cabal-tests:test:rpmvercmp) $TESTSUITEJOBS --hide-successes"
 398 (cd Cabal-tests && timed $CMD) || exit 1
 399
 400 CMD="$($CABALPLANLISTBIN Cabal-tests:test:no-thunks-test) $TESTSUITEJOBS --hide-successes"
 401 (cd Cabal-tests && timed $CMD) || exit 1
 402
 403 CMD=$($CABALPLANLISTBIN Cabal-tests:test:hackage-tests)
 404 (cd Cabal-tests && timed $CMD read-fields) || exit 1
 405 if $HACKAGETESTSALL; then
 406     (cd Cabal-tests && timed $CMD parsec)    || exit 1
 407     (cd Cabal-tests && timed $CMD roundtrip) || exit 1
 408 else
 409     (cd Cabal-tests && timed $CMD parsec d)    || exit 1
 410     (cd Cabal-tests && timed $CMD roundtrip k) || exit 1
 411 fi
 412 }
 413
 414 # Cabal cabal-testsuite
 415 #######################################################################
 416
 417 step_lib_suite() {
 418 print_header "Cabal: cabal-testsuite"
 419
 420 CMD="$($CABALPLANLISTBIN cabal-testsuite:exe:cabal-tests) --builddir=$CABAL_TESTSUITE_BDIR $TESTSUITEJOBS --with-ghc=$HC --hide-successes"
 421 (cd cabal-testsuite && timed $CMD) || exit 1
 422 }
 423
 424 step_lib_suite_extras() {
 425 for EXTRAHC in $EXTRAHCS; do
 426
 427 CMD="$($CABALPLANLISTBIN cabal-testsuite:exe:cabal-tests) --builddir=$CABAL_TESTSUITE_BDIR $TESTSUITEJOBS --with-ghc=$EXTRAHC --hide-successes"
 428 (cd cabal-testsuite && timed $CMD) || exit 1
 429
 430 done
 431 }
 432
 433 # cabal-install
 434 #######################################################################
 435
 436 step_cli_tests() {
 437 print_header "cabal-install: tests"
 438
 439 # this are sorted in asc time used, quicker tests first.
 440 CMD="$($CABALPLANLISTBIN cabal-install:test:long-tests) $TESTSUITEJOBS --hide-successes"
 441 (cd cabal-install && timed $CMD) || exit 1
 442
 443 # This doesn't work in parallel either
 444 CMD="$($CABALPLANLISTBIN cabal-install:test:unit-tests) -j1 --hide-successes"
 445 (cd cabal-install && timed $CMD) || exit 1
 446
 447 # Only single job, otherwise we fail with "Heap exhausted"
 448 CMD="$($CABALPLANLISTBIN cabal-install:test:mem-use-tests) -j1 --hide-successes"
 449 (cd cabal-install && timed $CMD) || exit 1
 450
 451 # This test-suite doesn't like concurrency
 452 CMD="$($CABALPLANLISTBIN cabal-install:test:integration-tests2) -j1 --hide-successes --with-ghc=$HC"
 453 (cd cabal-install && timed $CMD) || exit 1
 454 }
 455
 456 # cabal-install cabal-testsuite
 457 #######################################################################
 458
 459 step_cli_suite() {
 460 print_header "cabal-install: cabal-testsuite"
 461
 462 CMD="$($CABALPLANLISTBIN cabal-testsuite:exe:cabal-tests) --builddir=$CABAL_TESTSUITE_BDIR --with-cabal=$($CABALPLANLISTBIN cabal-install:exe:cabal) $TESTSUITEJOBS  --with-ghc=$HC --hide-successes"
 463 (cd cabal-testsuite && timed $CMD) || exit 1
 464 }
 465
 466 # solver-benchmarks
 467 #######################################################################
 468
 469 step_solver_benchmarks_tests() {
 470 print_header "solver-benchmarks: test"
 471
 472 CMD="$($CABALPLANLISTBIN solver-benchmarks:test:unit-tests)"
 473 (cd Cabal && timed $CMD) || exit 1
 474 }
 475
 476 step_solver_benchmarks_run() {
 477 print_header "solver-benchmarks: run"
 478
 479 SOLVEPKG=Chart-diagrams
 480 CMD="$($CABALPLANLISTBIN solver-benchmarks:exe:hackage-benchmark) --cabal1=$CABAL --cabal2=$($CABALPLANLISTBIN cabal-install:exe:cabal) --trials=5 --packages=$SOLVEPKG --print-trials"
 481 (cd Cabal && timed $CMD) || exit 1
 482 }
 483
 484 # Steps dispatcher
 485 #######################################################################
 486
 487 for step in $STEPS; do
 488     case $step in
 489         print-config)             step_print_config            ;;
 490         print-tool-versions)      step_print_tool_versions     ;;
 491         build)                    step_build                   ;;
 492         doctest)                  step_doctest                 ;;
 493         lib-tests)                step_lib_tests               ;;
 494         cli-tests)                step_cli_tests               ;;
 495         lib-suite)                step_lib_suite               ;;
 496         lib-suite-extras)         step_lib_suite_extras        ;;
 497         cli-suite)                step_cli_suite               ;;
 498         solver-benchmarks-tests)  step_solver_benchmarks_tests ;;
 499         solver-benchmarks-run)    step_solver_benchmarks_run   ;;
 500         time-summary)             step_time_summary            ;;
 501         *)
 502             echo "Invalid step $step"
 503             exit 1
 504             ;;
 505     esac
 506 done
 507
 508 #######################################################################