validate.sh

   1 #!/bin/sh
   2 # shellcheck disable=SC2086
   3
   4 # default config
   5 #######################################################################
   6
   7 # We use the default ghc in PATH as default
   8 # Use the ghc-x.y.z trigger several errors in windows:
   9 # * It triggers the max path length issue:
  10 #   See https://github.com/haskell/cabal/issues/6271#issuecomment-1065102255
  11 # * It triggers a `createProcess: does not exist` error in units tests
  12 #   See https://github.com/haskell/cabal/issues/8049
  13 HC=ghc
  14 CABAL=cabal
  15 JOBS=4
  16 LIBTESTS=true
  17 CLITESTS=true
  18 CABALSUITETESTS=true
  19 LIBONLY=false
  20 DEPSONLY=false
  21 DOCTEST=false
  22 BENCHMARKS=false
  23 VERBOSE=false
  24 HACKAGETESTSALL=false
  25
  26 TARGETS=""
  27 STEPS=""
  28 EXTRAHCS=""
  29
  30 LISTSTEPS=false
  31
  32 # Help
  33 #######################################################################
  34
  35 show_usage() {
  36 cat <<EOF
  37 ./validate.sh - build & test
  38
  39 Usage: ./validate.sh [options]
  40   A script which runs all the tests.
  41
  42 Available options:
  43   -j, --jobs JOBS                   cabal build -j argument (default:  $JOBS)
  44       --libonly                     Test only Cabal-the-library
  45       --cli                         Test both Cabal-the-library and cabal-install
  46       --(no-)run-lib-tests          Run library tests
  47       --(no-)run-cli-tests          Run client tests
  48       --(no-)run-lib-suite          Run cabal-testsuite with library
  49       --(no-)run-cli-suite          Run cabal-testsuite with client
  50   -w, --with-compiler HC            With compiler
  51       --with-cabal CABAL            With cabal-install
  52       --extra-hc HC                 Extra compiler to run test-suite with
  53       --(no-)doctest                Run doctest on library
  54       --(no-)solver-benchmarks      Build and trial run solver-benchmarks
  55       --complete-hackage-tests      Run hackage-tests on complete Hackage data
  56       --partial-hackage-tests       Run hackage-tests on parts of Hackage data
  57   -v, --verbose                     Verbose output
  58   -q, --quiet                       Less output
  59   -s, --step STEP                   Run only specific step (can be specified multiple times)
  60       --list-steps                  List steps and build-targets and exit
  61       --help                        Print this message and exit
  62 EOF
  63 }
  64
  65 # "library"
  66 #######################################################################
  67
  68 OUTPUT=$(mktemp)
  69
  70 RED='\033[0;31m'
  71 GREEN='\033[0;32m'
  72 BLUE='\033[0;34m'
  73 CYAN='\033[0;96m'
  74 RESET='\033[0m' # No Color
  75
  76 JOB_START_TIME=$(date +%s)
  77
  78 timed() {
  79     PRETTYCMD=$(echo "$@" | sed -E 's/\/home[^ ]*\/([^\/])/**\/\1/g')
  80     echo "$BLUE>>> $PRETTYCMD $RESET"
  81     start_time=$(date +%s)
  82
  83     if $VERBOSE; then
  84         "$@" 2>&1
  85     else
  86         "$@" > "$OUTPUT" 2>&1
  87     fi
  88     # echo "MOCK" > "$OUTPUT"
  89     RET=$?
  90
  91     end_time=$(date +%s)
  92     duration=$((end_time - start_time))
  93     tduration=$((end_time - JOB_START_TIME))
  94
  95     if [ $RET -eq 0 ]; then
  96         if ! $VERBOSE; then
  97             # if output is relatively short, show everything
  98             if [ "$(wc -l < "$OUTPUT")" -le 50 ]; then
  99                 cat "$OUTPUT"
 100             else
 101                 echo "..."
 102                 tail -n 20 "$OUTPUT"
 103             fi
 104
 105             rm -f "$OUTPUT"
 106         fi
 107
 108         echo "$GREEN<<< $PRETTYCMD $RESET ($duration/$tduration sec)"
 109
 110         # bottom-margin
 111         echo ""
 112     else
 113         if ! $VERBOSE; then
 114             cat "$OUTPUT"
 115         fi
 116
 117         echo "$RED<<< $PRETTYCMD $RESET ($duration/$tduration sec, $RET)"
 118         echo "$RED<<< $* $RESET ($duration/$tduration sec, $RET)"
 119         rm -f "$OUTPUT"
 120         exit 1
 121     fi
 122 }
 123
 124 print_header() {
 125     TITLE=$1
 126     TITLEPAT="$(echo "$TITLE"|sed 's:.:=:g')"
 127     echo "$CYAN===X============================================================ $(date +%T) ===$RESET" \
 128       | sed "s#X$TITLEPAT=# $TITLE #"
 129
 130 }
 131
 132 # getopt
 133 #######################################################################
 134
 135 while [ $# -gt 0 ]; do
 136     arg=$1
 137     case $arg in
 138         --help)
 139             show_usage
 140             exit
 141             ;;
 142         -j|--jobs)
 143             JOBS="$2"
 144             shift
 145             shift
 146             ;;
 147         --lib-only)
 148             LIBONLY=true
 149             shift
 150             ;;
 151         --cli)
 152             LIBONLY=false
 153             shift
 154             ;;
 155         --run-lib-tests)
 156             LIBTESTS=true
 157             shift
 158             ;;
 159         --no-run-lib-tests)
 160             LIBTESTS=false
 161             shift
 162             ;;
 163         --run-cli-tests)
 164             CLITESTS=true
 165             shift
 166             ;;
 167         --no-run-cli-tests)
 168             CLITESTS=false
 169             shift
 170             ;;
 171         --run-lib-suite)
 172             LIBSUITE=true
 173             shift
 174             ;;
 175         --no-run-lib-suite)
 176             LIBSUITE=false
 177             shift
 178             ;;
 179         --run-cli-suite)
 180             CLISUITE=true
 181             shift
 182             ;;
 183         --no-run-cli-suite)
 184             CLISUITE=false
 185             shift
 186             ;;
 187         -w|--with-compiler)
 188             HC=$2
 189             shift
 190             shift
 191             ;;
 192         --with-cabal)
 193             CABAL=$2
 194             shift
 195             shift
 196             ;;
 197         --extra-hc)
 198             EXTRAHCS="$EXTRAHCS $2"
 199             shift
 200             shift
 201             ;;
 202         --doctest)
 203             DOCTEST=true
 204             shift
 205             ;;
 206         --no-doctest)
 207             DOCTEST=false
 208             shift
 209             ;;
 210         --solver-benchmarks)
 211             BENCHMARKS=true
 212             shift
 213             ;;
 214         --no-solver-benchmarks)
 215             BENCHMARKS=false
 216             shift
 217             ;;
 218         --complete-hackage-tests)
 219             HACKAGETESTSALL=true
 220             shift
 221             ;;
 222         --partial-hackage-tests)
 223             HACKAGETESTSALL=false
 224             shift
 225             ;;
 226         -v|--verbose)
 227             VERBOSE=true
 228             shift
 229             ;;
 230         -q|--quiet)
 231             VERBOSE=false
 232             shift
 233             ;;
 234         -s|--step)
 235             STEPS="$STEPS $2"
 236             shift
 237             shift
 238             ;;
 239         --list-steps)
 240             LISTSTEPS=true
 241             shift
 242             ;;
 243         *)
 244             echo "Unknown option $arg"
 245             exit 1
 246     esac
 247 done
 248
 249 # calculate steps and build targets
 250 #######################################################################
 251
 252 # If there are no explicit steps given calculate them
 253 if $LIBONLY; then
 254     CLITESTS=false
 255     CLISUITE=false
 256     BENCHMARKS=false
 257 fi
 258
 259 if [ -z "$STEPS" ]; then
 260     STEPS="print-config print-tool-versions"
 261     STEPS="$STEPS build"
 262     if $DOCTEST;    then STEPS="$STEPS doctest";   fi
 263     if $LIBTESTS;   then STEPS="$STEPS lib-tests"; fi
 264     if $LIBSUITE;   then STEPS="$STEPS lib-suite"; fi
 265     if $LIBSUITE && [ -n "$EXTRAHCS" ];
 266                     then STEPS="$STEPS lib-suite-extras"; fi
 267     if $CLITESTS;   then STEPS="$STEPS cli-tests"; fi
 268     if $CLISUITE;   then STEPS="$STEPS cli-suite"; fi
 269     if $BENCHMARKS; then STEPS="$STEPS solver-benchmarks-tests solver-benchmarks-run"; fi
 270     STEPS="$STEPS time-summary"
 271 fi
 272
 273 TARGETS="Cabal cabal-testsuite Cabal-tests Cabal-QuickCheck Cabal-tree-diff Cabal-described"
 274 if ! $LIBONLY;  then TARGETS="$TARGETS cabal-install cabal-install-solver cabal-benchmarks"; fi
 275 if $BENCHMARKS; then TARGETS="$TARGETS solver-benchmarks"; fi
 276
 277 if $LISTSTEPS; then
 278   echo "Targets: $TARGETS"
 279   echo "Steps:   $STEPS"
 280   exit
 281 fi
 282
 283 # Adjust runtime configuration
 284 #######################################################################
 285
 286 TESTSUITEJOBS="-j$JOBS"
 287 JOBS="-j$JOBS"
 288
 289 # assume compiler is GHC
 290 RUNHASKELL=$(echo $HC | sed -E 's/ghc(-[0-9.]*)$/runghc\1/')
 291
 292 if [ "$OSTYPE" = "msys" ]; then
 293     ARCH="x86_64-windows"
 294 elif [ "$(uname)" = "Linux" ]; then
 295     ARCH="x86_64-linux"
 296 else
 297     ARCH="x86_64-osx"
 298 fi
 299
 300 if $LIBONLY; then
 301     PROJECTFILE=cabal.project.validate.libonly
 302 else
 303     PROJECTFILE=cabal.project.validate
 304 fi
 305
 306 BASEHC=ghc-$($HC --numeric-version)
 307 BUILDDIR=dist-newstyle-validate-$BASEHC
 308 CABAL_TESTSUITE_BDIR="$(pwd)/$BUILDDIR/build/$ARCH/$BASEHC/cabal-testsuite-3"
 309
 310 CABALNEWBUILD="${CABAL} build $JOBS -w $HC --builddir=$BUILDDIR --project-file=$PROJECTFILE"
 311 CABALLISTBIN="${CABAL} list-bin --builddir=$BUILDDIR --project-file=$PROJECTFILE"
 312
 313 # header
 314 #######################################################################
 315
 316 step_print_config() {
 317 print_header print-config
 318
 319 cat <<EOF
 320 compiler:            $HC
 321 runhaskell:          $RUNHASKELL
 322 cabal-install:       $CABAL
 323 jobs:                $JOBS
 324 Cabal tests:         $LIBTESTS
 325 cabal-install tests: $CLITESTS
 326 cabal-testsuite:     $CABALSUITETESTS
 327 library only:        $LIBONLY
 328 dependencies only:   $DEPSONLY
 329 doctest:             $DOCTEST
 330 benchmarks:          $BENCHMARKS
 331 verbose:             $VERBOSE
 332 extra compilers:     $EXTRAHCS
 333
 334 EOF
 335 }
 336
 337 step_print_tool_versions() {
 338 print_header print-tool-versions
 339
 340 timed $HC --version
 341 timed $CABAL --version
 342
 343 for EXTRAHC in $EXTRAHCS; do
 344     timed $EXTRAHC --version
 345 done
 346 }
 347
 348 step_time_summary() {
 349     print_header END
 350
 351     JOB_END_TIME=$(date +%s)
 352     tduration=$((JOB_END_TIME - JOB_START_TIME))
 353
 354     echo "$CYAN!!! Validation took $tduration seconds. $RESET"
 355 }
 356
 357 # build
 358 #######################################################################
 359
 360 step_build() {
 361 print_header "build"
 362 print_header "Step Build: dry run"
 363 timed $CABALNEWBUILD $TARGETS --dry-run || exit 1
 364 print_header "Step Build: full build plan (cached and to-be-built dependencies):"
 365 jq -r '."install-plan" | map(."pkg-name" + "-" + ."pkg-version" + " " + ."component-name") | join("\n")' "$BUILDDIR/cache/plan.json"
 366 print_header "Step Build: actual build"
 367 timed $CABALNEWBUILD $TARGETS || exit 1
 368 }
 369
 370 # Cabal lib
 371 #######################################################################
 372
 373 step_doctest() {
 374 print_header "Cabal: doctest"
 375 cabal-env --name doctest-Cabal --transitive QuickCheck
 376 cabal-env --name doctest-Cabal array bytestring containers deepseq directory filepath pretty process time binary unix text parsec mtl
 377 timed doctest -package-env=doctest-Cabal --fast Cabal/Distribution Cabal/Language
 378 }
 379
 380 step_lib_tests() {
 381 print_header "Cabal: tests"
 382
 383 CMD="$($CABALLISTBIN Cabal-tests:test:unit-tests) $TESTSUITEJOBS --hide-successes --with-ghc=$HC"
 384 (cd Cabal-tests && timed $CMD) || exit 1
 385
 386 CMD="$($CABALLISTBIN Cabal-tests:test:check-tests) $TESTSUITEJOBS --hide-successes"
 387 (cd Cabal-tests && timed $CMD) || exit 1
 388
 389 CMD="$($CABALLISTBIN Cabal-tests:test:parser-tests) $TESTSUITEJOBS --hide-successes"
 390 (cd Cabal-tests && timed $CMD) || exit 1
 391
 392 CMD="$($CABALLISTBIN Cabal-tests:test:rpmvercmp) $TESTSUITEJOBS --hide-successes"
 393 (cd Cabal-tests && timed $CMD) || exit 1
 394
 395 CMD="$($CABALLISTBIN Cabal-tests:test:no-thunks-test) $TESTSUITEJOBS --hide-successes"
 396 (cd Cabal-tests && timed $CMD) || exit 1
 397
 398 CMD=$($CABALLISTBIN Cabal-tests:test:hackage-tests)
 399 (cd Cabal-tests && timed $CMD read-fields) || exit 1
 400 if $HACKAGETESTSALL; then
 401     (cd Cabal-tests && timed $CMD parsec)    || exit 1
 402     (cd Cabal-tests && timed $CMD roundtrip) || exit 1
 403 else
 404     (cd Cabal-tests && timed $CMD parsec d)    || exit 1
 405     (cd Cabal-tests && timed $CMD roundtrip k) || exit 1
 406 fi
 407 }
 408
 409 # Cabal cabal-testsuite
 410 #######################################################################
 411
 412 step_lib_suite() {
 413 print_header "Cabal: cabal-testsuite"
 414
 415 CMD="$($CABALLISTBIN cabal-testsuite:exe:cabal-tests) --builddir=$CABAL_TESTSUITE_BDIR $TESTSUITEJOBS --with-ghc=$HC --hide-successes"
 416 (cd cabal-testsuite && timed $CMD) || exit 1
 417 }
 418
 419 step_lib_suite_extras() {
 420 for EXTRAHC in $EXTRAHCS; do
 421
 422 CMD="$($CABALLISTBIN cabal-testsuite:exe:cabal-tests) --builddir=$CABAL_TESTSUITE_BDIR $TESTSUITEJOBS --with-ghc=$EXTRAHC --hide-successes"
 423 (cd cabal-testsuite && timed $CMD) || exit 1
 424
 425 done
 426 }
 427
 428 # cabal-install
 429 #######################################################################
 430
 431 step_cli_tests() {
 432 print_header "cabal-install: tests"
 433
 434 # this are sorted in asc time used, quicker tests first.
 435 CMD="$($CABALLISTBIN cabal-install:test:long-tests) $TESTSUITEJOBS --hide-successes"
 436 (cd cabal-install && timed $CMD) || exit 1
 437
 438 # This doesn't work in parallel either
 439 CMD="$($CABALLISTBIN cabal-install:test:unit-tests) -j1 --hide-successes"
 440 (cd cabal-install && timed $CMD) || exit 1
 441
 442 # Only single job, otherwise we fail with "Heap exhausted"
 443 CMD="$($CABALLISTBIN cabal-install:test:mem-use-tests) -j1 --hide-successes"
 444 (cd cabal-install && timed $CMD) || exit 1
 445
 446 # This test-suite doesn't like concurrency
 447 CMD="$($CABALLISTBIN cabal-install:test:integration-tests2) -j1 --hide-successes --with-ghc=$HC"
 448 (cd cabal-install && timed $CMD) || exit 1
 449 }
 450
 451 # cabal-install cabal-testsuite
 452 #######################################################################
 453
 454 step_cli_suite() {
 455 print_header "cabal-install: cabal-testsuite"
 456
 457 CMD="$($CABALLISTBIN cabal-testsuite:exe:cabal-tests) --builddir=$CABAL_TESTSUITE_BDIR --with-cabal=$($CABALLISTBIN cabal-install:exe:cabal) $TESTSUITEJOBS  --with-ghc=$HC --hide-successes"
 458 (cd cabal-testsuite && timed $CMD) || exit 1
 459 }
 460
 461 # solver-benchmarks
 462 #######################################################################
 463
 464 step_solver_benchmarks_tests() {
 465 print_header "solver-benchmarks: test"
 466
 467 CMD="$($CABALLISTBIN solver-benchmarks:test:unit-tests)"
 468 (cd Cabal && timed $CMD) || exit 1
 469 }
 470
 471 step_solver_benchmarks_run() {
 472 print_header "solver-benchmarks: run"
 473
 474 SOLVEPKG=Chart-diagrams
 475 CMD="$($CABALLISTBIN solver-benchmarks:exe:hackage-benchmark) --cabal1=$CABAL --cabal2=$($CABALLISTBIN cabal-install:exe:cabal) --trials=5 --packages=$SOLVEPKG --print-trials"
 476 (cd Cabal && timed $CMD) || exit 1
 477 }
 478
 479 # Steps dispatcher
 480 #######################################################################
 481
 482 for step in $STEPS; do
 483     case $step in
 484         print-config)             step_print_config            ;;
 485         print-tool-versions)      step_print_tool_versions     ;;
 486         build)                    step_build                   ;;
 487         doctest)                  step_doctest                 ;;
 488         lib-tests)                step_lib_tests               ;;
 489         cli-tests)                step_cli_tests               ;;
 490         lib-suite)                step_lib_suite               ;;
 491         lib-suite-extras)         step_lib_suite_extras        ;;
 492         cli-suite)                step_cli_suite               ;;
 493         solver-benchmarks-tests)  step_solver_benchmarks_tests ;;
 494         solver-benchmarks-run)    step_solver_benchmarks_run   ;;
 495         time-summary)             step_time_summary            ;;
 496         *)
 497             echo "Invalid step $step"
 498             exit 1
 499             ;;
 500     esac
 501 done
 502
 503 #######################################################################