t/Makefile

   1 # Many parts shamelessly swiped from Git's t directory since that works
   2 # and is GPL2 just like TopGit
   3
   4 # Copyright (C) 2016 Kyle J. McKay
   5 # The lines swiped from Git are Copyright (C) 2005 Junio C Hamano et al.
   6
   7 #
   8 ## Make Targets
   9 ##
  10 ## all   -- default target which defaults to $(DEFAULT_TEST_TARGET) which
  11 ##          defaults to "test"
  12 ## prove -- run the tests using the $(PROVE) utility which is expected to
  13 ##          take the same arguments as Perl's prove
  14 ## test  -- run the tests without using an external helper utility
  15 #
  16
  17 #
  18 ## Make Variables
  19 ##
  20 ## An existing config.mak in the parent directory IS read first
  21 ##
  22 ## SHELL_PATH -- path to POSIX sh, default is /bin/sh if not otherwise set
  23 ## PERL_PATH  -- path to Perl, default is $(command -v perl) if not set
  24 ## GIT_PATH   -- path to git to use, default is $(command -v git) if not set
  25 ## DIFF       -- diff to use, defaults to "diff"
  26 ## PROVE      -- prove executable to run, MAY contain options
  27 ##
  28 ## TESTLIB_PROVE_OPTS
  29 ##            -- passed to $(PROVE) if the "prove" target is used
  30 ##
  31 ## DEFAULT_TEST_TARGET
  32 ##            -- defaults to "test" but can be "prove" to run with prove
  33 ##
  34 ## TESTLIB_TEST_LINT
  35 ##            -- set to "test-lint" (the default) to do some lint tests
  36 ##               may be set to empty to skip these
  37 ##
  38 ## TESTLIB_NO_CLEAN
  39 ##            -- suppresses removal of test-results directory after testing
  40 ##
  41 ## TESTLIB_NO_CACHE
  42 ##            -- suppresses use of TG-TEST-CACHE for "test" and "prove" targets
  43 ##
  44 ## TESTLIB_SKIP_TESTS
  45 ##            -- space-separated "case" patterns to match against the
  46 ##               t[0-9][0-9][0-9][0-9] portion of the test file name.  To
  47 ##               skip multiple tests use standard '*', '?' and '[...]'
  48 ##               match operators.  For example, to skip test t1234-foo.sh and
  49 ##               t3210-hi.sh use TESTLIB_SKIP_TESTS="t1234 t3210" to do that.
  50 ##
  51 ## TESTLIB_NO_TOLERATE
  52 ##            -- if non-empty turns all test_tolerate_failure calls into
  53 ##               test_expect_success calls instead
  54 ##
  55 ## TESTLIB_TEST_OPTS
  56 ##            -- provided as options to all tests (undergoes field splitting)
  57 ##               might be set to, for example: --verbose -debug
  58 ##
  59 ## T          -- space-sparated list of tests to run, must be full filename
  60 ##               WITHOUT any directory part of the test INCLUDING the .sh
  61 ##               suffix.  The default is all t\d{4}-*.sh files in this dir.
  62 ##
  63 ## TG_TEST_INSTALLED
  64 ##            -- if not empty, test whatever "tg" is found in $PATH
  65 #
  66
  67 # Default target is all
  68
  69 all::
  70
  71 # Basic setup
  72
  73 -include ../config.mak
  74 SHELL_PATH ?= /bin/sh
  75 SHELL = $(SHELL_PATH)
  76 DIFF ?= diff
  77 PROVE ?= prove
  78 DEFAULT_TEST_TARGET ?= test
  79 TEST_LINT ?= test-lint
  80 ifdef TEST_OUTPUT_DIRECTORY
  81 TEST_RESULTS_DIRECTORY = $(TEST_OUTPUT_DIRECTORY)/test-results
  82 else
  83 TEST_RESULTS_DIRECTORY = test-results
  84 endif
  85
  86 # Shell quote;
  87
  88 SHELL_PATH_SQ = $(subst ','\'',$(SHELL_PATH))
  89 PERL_PATH_SQ = $(subst ','\'',$(PERL_PATH))
  90 GIT_PATH_SQ = $(subst ','\'',$(GIT_PATH))
  91 TEST_RESULTS_DIRECTORY_SQ = $(subst ','\'',$(TEST_RESULTS_DIRECTORY))
  92
  93 # Default list of tests is all t????-*.sh files
  94
  95 ALLT = $(sort $(wildcard t[0-9][0-9][0-9][0-9]-*.sh))
  96 T ?= $(ALLT)
  97 LINTTESTS ?= $(T)
  98
  99 # Extra shell scripts to run through check-non-portable-shell.pl
 100 # These will ALWAYS be "checked" whenever the test-lint target is made
 101 # By default all $(T) test files are checked so they don't need to be
 102 # in this list
 103
 104 LINTSCRIPTS = $(sort $(filter-out $(ALLT),$(wildcard *.sh)))
 105
 106 # But all is just an alias for DEFAULT_TEST_TARGET which defaults to test
 107
 108 all:: $(DEFAULT_TEST_TARGET)
 109
 110 ifndef TESTLIB_NO_CACHE
 111
 112 define CACHE_SETUP
 113 TESTLIB_CACHE="$$($(SHELL_PATH_SQ) ./test-lib.sh --cache $(TESTLIB_TEST_OPTS) 2>/dev/null || :)"
 114 endef
 115
 116 define CACHE_SETUP_TTY
 117 ! test -t 1 || { TESTLIB_FORCETTY=1 && export TESTLIB_FORCETTY; }; $(CACHE_SETUP)
 118 endef
 119
 120 endif
 121
 122 test: pre-clean TG-TEST-SETTINGS $(TEST_LINT)
 123         @$(CACHE_SETUP_TTY) $(MAKE) aggregate-results-and-cleanup
 124
 125 prove: pre-clean TG-TEST-SETTINGS $(TEST_LINT)
 126         @echo "*** prove ***"; $(CACHE_SETUP) $(PROVE) --exec '$(SHELL_PATH_SQ)' $(TESTLIB_PROVE_OPTS) $(T) :: $(TESTLIB_TEST_OPTS)
 127 ifndef TESTLIB_NO_CLEAN
 128         @$(MAKE) -s clean-except-prove-cache
 129 endif
 130
 131 $(T):
 132         @echo "*** $@ ***"; '$(SHELL_PATH_SQ)' $@ $(TESTLIB_TEST_OPTS)
 133
 134 # How to clean up
 135
 136 pre-clean:
 137         @rm -r -f '$(TEST_RESULTS_DIRECTORY_SQ)'
 138
 139 clean-except-prove-cache:
 140         rm -r -f empty 'trash directory'.* '$(TEST_RESULTS_DIRECTORY_SQ)'
 141         rm -f TG-TEST-CACHE
 142
 143 clean: clean-except-prove-cache
 144         rm -f .prove
 145
 146 # Pick off the lint
 147
 148 test-lint: test-lint-duplicates test-lint-executable test-lint-shell-syntax \
 149         test-lint-filenames
 150
 151 test-lint-duplicates:
 152         @dups=`echo $(ALLT) | tr ' ' '\n' | sed 's/-.*//' | sort | uniq -d` && \
 153                 test -z "$$dups" || { \
 154                 echo >&2 "duplicate test numbers:" $$dups; exit 1; }
 155
 156 test-lint-executable:
 157         @bad=`for i in $(LINTTESTS); do test -x "$$i" || echo $$i; done` && \
 158                 test -z "$$bad" || { \
 159                 echo >&2 "non-executable tests:" $$bad; exit 1; }
 160
 161 test-lint-shell-syntax:
 162         @p='$(PERL_PATH_SQ)'; "$${p:-perl}" check-non-portable-shell.pl $(LINTTESTS) $(LINTSCRIPTS)
 163
 164 test-lint-filenames:
 165         @# We do *not* pass a glob to ls-files but use grep instead, to catch
 166         @# non-ASCII characters (which are quoted within double-quotes)
 167         @g='$(GIT_PATH_SQ)'; bad="$$("$${g:-git}" -c core.quotepath=true ls-files 2>/dev/null | \
 168                         grep '['\''""*:<>?\\|]')"; \
 169                 test -z "$$bad" || { \
 170                 echo >&2 "non-portable file name(s): $$bad"; exit 1; }
 171
 172 # Run the tests without using prove
 173
 174 run-individual-tests: $(T)
 175
 176 aggregate-results-and-cleanup:
 177         @ec=0; TESTLIB_TEST_PARENT_INT_ON_ERROR=1 $(MAKE) -k run-individual-tests || ec=$$?; \
 178         [ $$ec -eq 130 ] || $(MAKE) aggregate-results || exit && exit $$ec
 179 ifndef TESTLIB_NO_CLEAN
 180         @$(MAKE) -s clean
 181 endif
 182
 183 aggregate-results:
 184         @for f in '$(TEST_RESULTS_DIRECTORY_SQ)'/t*-*.counts; do \
 185                 echo "$$f"; \
 186         done | '$(SHELL_PATH_SQ)' ./aggregate-results.sh
 187
 188 .PHONY: pre-clean $(T) run-individual-tests aggregate-results clean valgrind perf
 189
 190 # Provide Makefile-determined settings in a test-available format
 191
 192 define TEST_SETTINGS
 193 : "$${SHELL_PATH:=$(SHELL_PATH)}"
 194 : "$${PERL_PATH:=$(PERL_PATH)}" "$${PERL_PATH:=perl}"
 195 : "$${GIT_PATH:=$(GIT_PATH)}" "$${GIT_PATH:=git}"
 196 : "$${DIFF:=$(DIFF)}"
 197 : "$${TESTLIB_NO_TOLERATE=$(TESTLIB_NO_TOLERATE)}"
 198 : "$${GIT_MINIMUM_VERSION:=$(GIT_MINIMUM_VERSION)}" "$${GIT_MINIMUM_VERSION:=$$TG_GIT_MINIMUM_VERSION}"
 199 : "$${TG_TEST_INSTALLED=$(TG_TEST_INSTALLED)}"
 200 endef
 201 export TEST_SETTINGS
 202
 203 TG-TEST-SETTINGS: FORCE
 204         @if test x"$$TEST_SETTINGS" != x"`cat $@ 2>/dev/null`"; then \
 205                 echo "* new test settings"; \
 206                 echo "$$TEST_SETTINGS" >$@; \
 207         fi
 208
 209 .PHONY: FORCE