Merge pull request #9701 from cabalism/doc/compare-source-repos-9665
[cabal.git] / validate.sh
blob3b56a945c8482200cd46b86d36ff7a4b40e095ec
1 #!/bin/sh
2 # shellcheck disable=SC2086
4 # default config
5 #######################################################################
7 # We use the default ghc in PATH as default
8 # Use the ghc-x.y.z trigger several errors in windows:
9 # * It triggers the max path length issue:
10 # See https://github.com/haskell/cabal/issues/6271#issuecomment-1065102255
11 # * It triggers a `createProcess: does not exist` error in units tests
12 # See https://github.com/haskell/cabal/issues/8049
13 HC=ghc
14 CABAL=cabal
15 JOBS=4
16 LIBTESTS=true
17 CLITESTS=true
18 CABALSUITETESTS=true
19 LIBONLY=false
20 DEPSONLY=false
21 DOCTEST=false
22 BENCHMARKS=false
23 VERBOSE=false
24 HACKAGETESTSALL=false
26 TARGETS=""
27 STEPS=""
28 EXTRAHCS=""
30 LISTSTEPS=false
32 # Help
33 #######################################################################
35 show_usage() {
36 cat <<EOF
37 ./validate.sh - build & test
39 Usage: ./validate.sh [options]
40 A script which runs all the tests.
42 Available options:
43 -j, --jobs JOBS cabal build -j argument (default: $JOBS)
44 --libonly Test only Cabal-the-library
45 --cli Test both Cabal-the-library and cabal-install
46 --(no-)run-lib-tests Run library tests
47 --(no-)run-cli-tests Run client tests
48 --(no-)run-lib-suite Run cabal-testsuite with library
49 --(no-)run-cli-suite Run cabal-testsuite with client
50 -w, --with-compiler HC With compiler
51 --with-cabal CABAL With cabal-install
52 --extra-hc HC Extra compiler to run test-suite with
53 --(no-)doctest Run doctest on library
54 --(no-)solver-benchmarks Build and trial run solver-benchmarks
55 --complete-hackage-tests Run hackage-tests on complete Hackage data
56 --partial-hackage-tests Run hackage-tests on parts of Hackage data
57 -v, --verbose Verbose output
58 -q, --quiet Less output
59 -s, --step STEP Run only specific step (can be specified multiple times)
60 --list-steps List steps and build-targets and exit
61 --help Print this message and exit
62 EOF
65 # "library"
66 #######################################################################
68 OUTPUT=$(mktemp)
70 # `red` and `green` are used to output also the spent time in white at the end.
71 # `blue` and `cyan` are used to print the spawned command, so they only have one
72 # argument.
73 red () {
74 printf "\033[0;31m%s\033[0m %s \n" "$1" "$2"
76 green () {
77 printf "\033[0;32m%s\033[0m %s \n" "$1" "$2"
79 blue () {
80 printf "\033[0;34m%s\033[0m\n" "$1"
82 cyan () {
83 printf "\033[0;96m%s\033[0m\n" "$1"
86 JOB_START_TIME=$(date +%s)
88 timed() {
89 PRETTYCMD=$(echo "$@" | sed -E 's/\/home[^ ]*\/([^\/])/**\/\1/g')
90 blue "$PRETTYCMD"
91 start_time=$(date +%s)
93 if $VERBOSE; then
94 "$@" 2>&1
95 else
96 "$@" > "$OUTPUT" 2>&1
98 # echo "MOCK" > "$OUTPUT"
99 RET=$?
101 end_time=$(date +%s)
102 duration=$((end_time - start_time))
103 tduration=$((end_time - JOB_START_TIME))
105 if [ $RET -eq 0 ]; then
106 if ! $VERBOSE; then
107 # if output is relatively short, show everything
108 if [ "$(wc -l < "$OUTPUT")" -le 50 ]; then
109 cat "$OUTPUT"
110 else
111 echo "..."
112 tail -n 20 "$OUTPUT"
115 rm -f "$OUTPUT"
118 green "<<< $PRETTYCMD" "($duration/$tduration sec)"
120 # bottom-margin
121 echo ""
122 else
123 if ! $VERBOSE; then
124 cat "$OUTPUT"
127 red "<<< $PRETTYCMD" "($duration/$tduration sec, $RET)"
128 red "<<< $*" "($duration/$tduration sec, $RET)"
129 rm -f "$OUTPUT"
130 exit 1
134 print_header() {
135 TITLE=$1
136 TITLEPAT="$(echo "$TITLE"|sed 's:.:=:g')"
137 cyan "===X========================================================================== $(date +%T) ===" \
138 | sed "s#X$TITLEPAT=# $TITLE #"
142 # getopt
143 #######################################################################
145 while [ $# -gt 0 ]; do
146 arg=$1
147 case $arg in
148 --help)
149 show_usage
150 exit
152 -j|--jobs)
153 JOBS="$2"
154 shift
155 shift
157 --lib-only)
158 LIBONLY=true
159 shift
161 --cli)
162 LIBONLY=false
163 shift
165 --run-lib-tests)
166 LIBTESTS=true
167 shift
169 --no-run-lib-tests)
170 LIBTESTS=false
171 shift
173 --run-cli-tests)
174 CLITESTS=true
175 shift
177 --no-run-cli-tests)
178 CLITESTS=false
179 shift
181 --run-lib-suite)
182 LIBSUITE=true
183 shift
185 --no-run-lib-suite)
186 LIBSUITE=false
187 shift
189 --run-cli-suite)
190 CLISUITE=true
191 shift
193 --no-run-cli-suite)
194 CLISUITE=false
195 shift
197 -w|--with-compiler)
198 HC=$2
199 shift
200 shift
202 --with-cabal)
203 CABAL=$2
204 shift
205 shift
207 --extra-hc)
208 EXTRAHCS="$EXTRAHCS $2"
209 shift
210 shift
212 --doctest)
213 DOCTEST=true
214 shift
216 --no-doctest)
217 DOCTEST=false
218 shift
220 --solver-benchmarks)
221 BENCHMARKS=true
222 shift
224 --no-solver-benchmarks)
225 BENCHMARKS=false
226 shift
228 --complete-hackage-tests)
229 HACKAGETESTSALL=true
230 shift
232 --partial-hackage-tests)
233 HACKAGETESTSALL=false
234 shift
236 -v|--verbose)
237 VERBOSE=true
238 shift
240 -q|--quiet)
241 VERBOSE=false
242 shift
244 -s|--step)
245 STEPS="$STEPS $2"
246 shift
247 shift
249 --list-steps)
250 LISTSTEPS=true
251 shift
254 echo "Unknown option $arg"
255 exit 1
256 esac
257 done
259 # calculate steps and build targets
260 #######################################################################
262 # If there are no explicit steps given calculate them
263 if $LIBONLY; then
264 CLITESTS=false
265 CLISUITE=false
266 BENCHMARKS=false
269 if [ -z "$STEPS" ]; then
270 STEPS="print-config print-tool-versions"
271 STEPS="$STEPS build"
272 if $DOCTEST; then STEPS="$STEPS doctest"; fi
273 if $LIBTESTS; then STEPS="$STEPS lib-tests"; fi
274 if $LIBSUITE; then STEPS="$STEPS lib-suite"; fi
275 if $LIBSUITE && [ -n "$EXTRAHCS" ];
276 then STEPS="$STEPS lib-suite-extras"; fi
277 if $CLITESTS; then STEPS="$STEPS cli-tests"; fi
278 if $CLISUITE; then STEPS="$STEPS cli-suite"; fi
279 if $BENCHMARKS; then STEPS="$STEPS solver-benchmarks-tests solver-benchmarks-run"; fi
280 STEPS="$STEPS time-summary"
283 TARGETS="Cabal Cabal-hooks cabal-testsuite Cabal-tests Cabal-QuickCheck Cabal-tree-diff Cabal-described"
284 if ! $LIBONLY; then TARGETS="$TARGETS cabal-install cabal-install-solver cabal-benchmarks"; fi
285 if $BENCHMARKS; then TARGETS="$TARGETS solver-benchmarks"; fi
287 if $LISTSTEPS; then
288 echo "Targets: $TARGETS"
289 echo "Steps: $STEPS"
290 exit
293 # Adjust runtime configuration
294 #######################################################################
296 TESTSUITEJOBS="-j$JOBS"
297 JOBS="-j$JOBS"
299 # assume compiler is GHC
300 RUNHASKELL=$(echo "$HC" | sed -E 's/ghc(-[0-9.]*)$/runghc\1/')
302 case "$(uname)" in
303 MINGW64*)
304 ARCH="x86_64-windows"
306 Linux )
307 ARCH="x86_64-linux"
310 ARCH="x86_64-osx"
312 esac
314 if $LIBONLY; then
315 PROJECTFILE=cabal.validate-libonly.project
316 else
317 PROJECTFILE=cabal.validate.project
320 BASEHC=ghc-$($HC --numeric-version)
321 BUILDDIR=dist-newstyle-validate-$BASEHC
322 CABAL_TESTSUITE_BDIR="$(pwd)/$BUILDDIR/build/$ARCH/$BASEHC/cabal-testsuite-3"
324 CABALNEWBUILD="${CABAL} build $JOBS -w $HC --builddir=$BUILDDIR --project-file=$PROJECTFILE"
325 CABALLISTBIN="${CABAL} list-bin --builddir=$BUILDDIR --project-file=$PROJECTFILE"
327 # header
328 #######################################################################
330 step_print_config() {
331 print_header print-config
333 cat <<EOF
334 compiler: $HC
335 runhaskell: $RUNHASKELL
336 cabal-install: $CABAL
337 jobs: $JOBS
338 Cabal tests: $LIBTESTS
339 cabal-install tests: $CLITESTS
340 cabal-testsuite: $CABALSUITETESTS
341 library only: $LIBONLY
342 dependencies only: $DEPSONLY
343 doctest: $DOCTEST
344 benchmarks: $BENCHMARKS
345 verbose: $VERBOSE
346 extra compilers: $EXTRAHCS
351 step_print_tool_versions() {
352 print_header print-tool-versions
354 timed "$HC" --version
355 timed "$CABAL" --version
357 for EXTRAHC in $EXTRAHCS; do
358 timed "$EXTRAHC" --version
359 done
362 step_time_summary() {
363 print_header END
365 JOB_END_TIME=$(date +%s)
366 tduration=$((JOB_END_TIME - JOB_START_TIME))
368 cyan "!!! Validation took $tduration seconds."
371 # build
372 #######################################################################
374 step_build() {
375 print_header "build"
376 print_header "Step Build: dry run"
377 timed $CABALNEWBUILD $TARGETS --dry-run || exit 1
378 print_header "Step Build: full build plan (cached and to-be-built dependencies):"
379 jq -r '."install-plan" | map(."pkg-name" + "-" + ."pkg-version" + " " + ."component-name") | join("\n")' "$BUILDDIR/cache/plan.json"
380 print_header "Step Build: actual build"
381 timed $CABALNEWBUILD $TARGETS || exit 1
384 # Cabal lib
385 #######################################################################
387 step_doctest() {
388 print_header "Cabal: doctest"
389 cabal-env --name doctest-Cabal --transitive QuickCheck
390 cabal-env --name doctest-Cabal array bytestring containers deepseq directory filepath pretty process time binary unix text parsec mtl
391 timed doctest -package-env=doctest-Cabal --fast Cabal/Distribution Cabal/Language
394 step_lib_tests() {
395 print_header "Cabal: tests"
397 CMD="$($CABALLISTBIN Cabal-tests:test:unit-tests) $TESTSUITEJOBS --hide-successes --with-ghc=$HC"
398 (cd Cabal-tests && timed $CMD) || exit 1
400 CMD="$($CABALLISTBIN Cabal-tests:test:check-tests) $TESTSUITEJOBS --hide-successes"
401 (cd Cabal-tests && timed $CMD) || exit 1
403 CMD="$($CABALLISTBIN Cabal-tests:test:parser-tests) $TESTSUITEJOBS --hide-successes"
404 (cd Cabal-tests && timed $CMD) || exit 1
406 CMD="$($CABALLISTBIN Cabal-tests:test:rpmvercmp) $TESTSUITEJOBS --hide-successes"
407 (cd Cabal-tests && timed $CMD) || exit 1
409 CMD="$($CABALLISTBIN Cabal-tests:test:no-thunks-test) $TESTSUITEJOBS --hide-successes"
410 (cd Cabal-tests && timed $CMD) || exit 1
412 CMD=$($CABALLISTBIN Cabal-tests:test:hackage-tests)
413 (cd Cabal-tests && timed $CMD read-fields) || exit 1
414 if $HACKAGETESTSALL; then
415 (cd Cabal-tests && timed $CMD parsec) || exit 1
416 (cd Cabal-tests && timed $CMD roundtrip) || exit 1
417 else
418 (cd Cabal-tests && timed $CMD parsec d) || exit 1
419 (cd Cabal-tests && timed $CMD roundtrip k) || exit 1
423 # Cabal cabal-testsuite
424 #######################################################################
426 step_lib_suite() {
427 print_header "Cabal: cabal-testsuite"
429 CMD="$($CABALLISTBIN cabal-testsuite:exe:cabal-tests) --builddir=$CABAL_TESTSUITE_BDIR $TESTSUITEJOBS --with-ghc=$HC --hide-successes"
430 (cd cabal-testsuite && timed $CMD) || exit 1
433 step_lib_suite_extras() {
434 for EXTRAHC in $EXTRAHCS; do
436 CMD="$($CABALLISTBIN cabal-testsuite:exe:cabal-tests) --builddir=$CABAL_TESTSUITE_BDIR $TESTSUITEJOBS --with-ghc=$EXTRAHC --hide-successes"
437 (cd cabal-testsuite && timed $CMD) || exit 1
439 done
442 # cabal-install
443 #######################################################################
445 step_cli_tests() {
446 print_header "cabal-install: tests"
448 # this are sorted in asc time used, quicker tests first.
449 CMD="$($CABALLISTBIN cabal-install:test:long-tests) $TESTSUITEJOBS --hide-successes"
450 (cd cabal-install && timed $CMD) || exit 1
452 # This doesn't work in parallel either
453 CMD="$($CABALLISTBIN cabal-install:test:unit-tests) -j1 --hide-successes"
454 (cd cabal-install && timed $CMD) || exit 1
456 # Only single job, otherwise we fail with "Heap exhausted"
457 CMD="$($CABALLISTBIN cabal-install:test:mem-use-tests) -j1 --hide-successes"
458 (cd cabal-install && timed $CMD) || exit 1
460 # This test-suite doesn't like concurrency
461 CMD="$($CABALLISTBIN cabal-install:test:integration-tests2) -j1 --hide-successes --with-ghc=$HC"
462 (cd cabal-install && timed $CMD) || exit 1
465 # cabal-install cabal-testsuite
466 #######################################################################
468 step_cli_suite() {
469 print_header "cabal-install: cabal-testsuite"
471 CMD="$($CABALLISTBIN cabal-testsuite:exe:cabal-tests) --builddir=$CABAL_TESTSUITE_BDIR --with-cabal=$($CABALLISTBIN cabal-install:exe:cabal) $TESTSUITEJOBS --with-ghc=$HC --hide-successes --intree-cabal-lib=$PWD --test-tmp=$PWD/testdb"
472 (cd cabal-testsuite && timed $CMD) || exit 1
475 # solver-benchmarks
476 #######################################################################
478 step_solver_benchmarks_tests() {
479 print_header "solver-benchmarks: test"
481 CMD="$($CABALLISTBIN solver-benchmarks:test:unit-tests)"
482 (cd Cabal && timed $CMD) || exit 1
485 step_solver_benchmarks_run() {
486 print_header "solver-benchmarks: run"
488 SOLVEPKG=Chart-diagrams
489 CMD="$($CABALLISTBIN solver-benchmarks:exe:hackage-benchmark) --cabal1=$CABAL --cabal2=$($CABALLISTBIN cabal-install:exe:cabal) --trials=5 --packages=$SOLVEPKG --print-trials"
490 (cd Cabal && timed $CMD) || exit 1
493 # Steps dispatcher
494 #######################################################################
496 for step in $STEPS; do
497 case $step in
498 print-config) step_print_config ;;
499 print-tool-versions) step_print_tool_versions ;;
500 build) step_build ;;
501 doctest) step_doctest ;;
502 lib-tests) step_lib_tests ;;
503 cli-tests) step_cli_tests ;;
504 lib-suite) step_lib_suite ;;
505 lib-suite-extras) step_lib_suite_extras ;;
506 cli-suite) step_cli_suite ;;
507 solver-benchmarks-tests) step_solver_benchmarks_tests ;;
508 solver-benchmarks-run) step_solver_benchmarks_run ;;
509 time-summary) step_time_summary ;;
511 echo "Invalid step $step"
512 exit 1
514 esac
515 done
517 #######################################################################