4 # DO NOT EDIT THIS FILE IN MASTER. ONLY EDIT IT IN THE OLDEST SUPPORTED
5 # BRANCH, THEN MERGE FORWARD.
8 # This script is used to build Tor for continuous integration. It should
9 # be kept the same for all supported Tor versions.
11 # It's subject to the regular Tor license; see LICENSE for copying
17 # Options for this script.
18 DEBUG_CI
="${DEBUG_CI:-no}"
19 COLOR_CI
="${COLOR_CI:-yes}"
21 # Options for which CI system this is.
22 ON_GITLAB
="${ON_GITLAB:-yes}"
24 # Options for things we usually won't want to skip.
25 RUN_STAGE_CONFIGURE
="${RUN_STAGE_CONFIGURE:-yes}"
26 RUN_STAGE_BUILD
="${RUN_STAGE_BUILD:-yes}"
27 RUN_STAGE_TEST
="${RUN_STAGE_TEST:-yes}"
29 # Options for how to build Tor. All should be yes/no.
30 FATAL_WARNINGS
="${FATAL_WARNINGS:-yes}"
31 HARDENING
="${HARDENING:-no}"
32 COVERAGE
="${COVERAGE:-no}"
34 DOXYGEN
="${DOXYGEN:-no}"
35 ASCIIDOC
="${ASCIIDOC:-no}"
36 TRACING
="${TRACING:-no}"
37 ALL_BUGS_ARE_FATAL
="${ALL_BUGS_ARE_FATAL:-no}"
38 DISABLE_DIRAUTH
="${DISABLE_DIRAUTH:-no}"
39 DISABLE_RELAY
="${DISABLE_RELAY:-no}"
42 # Options for which tests to run. All should be yes/no.
45 CHUTNEY
="${CHUTNEY:-no}"
46 DISTCHECK
="${DISTCHECK:-no}"
48 # Options for where the Tor source is.
49 CI_SRCDIR
="${CI_SRCDIR:-.}"
51 # Options for where to build.
52 CI_BUILDDIR
="${CI_BUILDDIR:-./build}"
54 # How parallel should we run make?
55 MAKE_J_OPT
="${MAKE_J_OPT:--j4}"
56 # Should we stop after make finds an error?
57 MAKE_K_OPT
="${MAKE_K_OPT:--k}"
59 # What make target should we use for chutney?
60 CHUTNEY_MAKE_TARGET
="${CHUTNEY_MAKE_TARGET:-test-network}"
62 # Where do we find our additional testing tools?
63 CHUTNEY_PATH
="${CHUTNEY_PATH:-}"
64 STEM_PATH
="${STEM_PATH:-}"
66 #############################################################################
67 # Preliminary functions.
69 # Terminal coloring/emphasis stuff.
70 if [[ "${COLOR_CI}" == "yes" ]]; then
71 T_RED
=$
(tput setaf
1 || true
)
72 T_GREEN
=$
(tput setaf
2 || true
)
73 T_YELLOW
=$
(tput setaf
3 || true
)
74 T_DIM
=$
(tput dim || true
)
75 T_BOLD
=$
(tput bold || true
)
76 T_RESET
=$
(tput sgr0 || true
)
88 echo "${T_BOLD}${T_RED}ERROR:${T_RESET} $*" 1>&2
92 echo "${T_BOLD}${T_RED}FATAL ERROR:${T_RESET} $*" 1>&2
98 echo "${T_BOLD}${T_YELLOW}Skipping $*${T_RESET}"
103 echo "${T_BOLD}${T_GREEN}$*${T_RESET}"
106 if [[ "${DEBUG_CI}" == "yes" ]]; then
109 echo "${T_DIM}(debug): $*${T_RESET}"
121 local value
="${!varname}"
122 debug
"${varname} is ${value}"
123 if [[ "${value}" != 'yes' && "${value}" != 'no' ]]; then
124 die
"${varname} must be 'yes' or 'no'. Got unexpected value ${value}".
128 function incompatible
()
132 local val1
="${!varname1}"
133 local val2
="${!varname2}"
134 if [[ "${val1}" = 'yes' && "${val2}" = 'yes' ]]; then
135 die
"Cannot set both ${varname1} and ${varname2}: they are incompatible."
141 echo "${T_BOLD}\$ $*${T_RESET}"
143 error
"command '$*' has failed."
148 function show_git_version
()
152 local version
="?????"
153 if [[ -e "$dir/.git" ]] ; then
154 version
=$
(cd "$dir"; git rev-parse HEAD
)
156 echo "${T_BOLD}$tool:${T_RESET} $version"
159 if [[ "${ON_GITLAB}" == "yes" ]]; then
160 function start_section
()
165 printf "section_start:%s:%s\r\e[0K" "$stamp" "$label"
166 echo "${T_BOLD}${T_GREEN}========= $label${T_RESET}"
168 function end_section
()
173 printf "section_end:%s:%s\r\e[0K" "$stamp" "$label"
176 function start_section
()
180 function end_section
()
186 #############################################################################
189 debug Validating inputs
193 yes_or_no FATAL_WARNINGS
200 yes_or_no ALL_BUGS_ARE_FATAL
201 yes_or_no DISABLE_DIRAUTH
202 yes_or_no DISABLE_RELAY
205 yes_or_no RUN_STAGE_CONFIGURE
206 yes_or_no RUN_STAGE_BUILD
207 yes_or_no RUN_STAGE_TEST
213 incompatible DISTCHECK CHECK
214 incompatible DISTCHECK CHUTNEY
215 incompatible DISTCHECK STEM
216 incompatible DISTCHECK COVERAGE
217 incompatible DISTCHECK DOXYGEN
219 if [[ "${CHUTNEY}" = yes && "${CHUTNEY_PATH}" = '' ]] ; then
220 die
"CHUTNEY is set to 'yes', but CHUTNEY_PATH was not specified."
223 if [[ "${STEM}" = yes && "${STEM_PATH}" = '' ]] ; then
224 die
"STEM is set to 'yes', but STEM_PATH was not specified."
227 #############################################################################
228 # Set up options for make and configure.
231 if [[ "$MAKE_J_OPT" != "" ]]; then
232 make_options
+=("$MAKE_J_OPT")
234 if [[ "$MAKE_K_OPT" != "" ]]; then
235 make_options
+=("$MAKE_K_OPT")
239 if [[ "$FATAL_WARNINGS" == "yes" ]]; then
240 configure_options
+=("--enable-fatal-warnings")
242 if [[ "$HARDENING" == "yes" ]]; then
243 configure_options
+=("--enable-fragile-hardening")
245 if [[ "$COVERAGE" == "yes" ]]; then
246 configure_options
+=("--enable-coverage")
248 if [[ "$RUST" == "yes" ]]; then
249 configure_options
+=("--enable-rust")
251 if [[ "$ASCIIDOC" != "yes" ]]; then
252 configure_options
+=("--disable-asciidoc")
254 if [[ "$TRACING" == "yes" ]]; then
255 configure_options
+=("--enable-tracing-instrumentation-lttng")
257 if [[ "$ALL_BUGS_ARE_FATAL" == "yes" ]]; then
258 configure_options
+=("--enable-all-bugs-are-fatal")
260 if [[ "$DISABLE_DIRAUTH" == "yes" ]]; then
261 configure_options
+=("--disable-module-dirauth")
263 if [[ "$DISABLE_RELAY" == "yes" ]]; then
264 configure_options
+=("--disable-module-relay")
266 if [[ "$NSS" == "yes" ]]; then
267 configure_options
+=("--enable-nss")
270 #############################################################################
271 # Tell the user about our versions of different tools and packages.
275 python
-V ||
echo "no 'python' binary."
277 python3
-V ||
echo "no 'python3' binary."
279 show_git_version Tor
"${CI_SRCDIR}"
280 if [[ "${STEM}" = "yes" ]]; then
281 show_git_version Stem
"${STEM_PATH}"
283 if [[ "${CHUTNEY}" = "yes" ]]; then
284 show_git_version Chutney
"${CHUTNEY_PATH}"
287 #############################################################################
288 # Determine the version of Tor.
290 TOR_VERSION
=$
(grep -m 1 AC_INIT
"${CI_SRCDIR}"/configure.ac |
sed -e 's/.*\[//; s/\].*//;')
292 # Use variables like these when we need to behave differently depending on
293 # Tor version. Only create the variables we need.
294 TOR_VER_AT_LEAST_043
=no
295 TOR_VER_AT_LEAST_044
=no
297 # These are the currently supported Tor versions; no need to work with anything
298 # ancient in this script.
299 case "$TOR_VERSION" in
301 TOR_VER_AT_LEAST_043
=no
302 TOR_VER_AT_LEAST_044
=no
305 TOR_VER_AT_LEAST_043
=no
306 TOR_VER_AT_LEAST_044
=no
309 TOR_VER_AT_LEAST_043
=yes
310 TOR_VER_AT_LEAST_044
=no
313 TOR_VER_AT_LEAST_043
=yes
314 TOR_VER_AT_LEAST_044
=yes
317 TOR_VER_AT_LEAST_043
=yes
318 TOR_VER_AT_LEAST_044
=yes
321 TOR_VER_AT_LEAST_043
=yes
322 TOR_VER_AT_LEAST_044
=yes
326 #############################################################################
327 # Make sure the directories are all there.
329 # Make sure CI_SRCDIR exists and has a file we expect.
330 if [[ ! -d "$CI_SRCDIR" ]] ; then
331 die
"CI_SRCDIR=${CI_SRCDIR} is not a directory"
333 if [[ ! -f "$CI_SRCDIR/src/core/or/or.h" ]] ; then
334 die
"CI_SRCDIR=${CI_SRCDIR} does not look like a Tor directory."
337 # Make CI_SRCDIR absolute.
338 CI_SRCDIR
=$
(cd "$CI_SRCDIR" && pwd)
340 # Create an "artifacts" directory to copy artifacts into.
343 if [[ "$RUN_STAGE_CONFIGURE" = "yes" ]]; then
345 start_section
"Autogen"
346 runcmd
cd "${CI_SRCDIR}"
348 runcmd mkdir
-p "${CI_BUILDDIR}"
349 runcmd
cd "${CI_BUILDDIR}"
350 end_section
"Autogen"
352 # make the builddir absolute too.
355 start_section
"Configure"
356 if ! runcmd
"${CI_SRCDIR}"/configure
"${configure_options[@]}" ; then
357 error
"Here is the end of config.log:"
358 runcmd
tail config.log
359 die
"Unable to continue"
361 end_section
"Configure"
363 debug
"Skipping configure stage. Making sure that ${CI_BUILDDIR}/config.log exists."
364 if [[ ! -d "${CI_BUILDDIR}" ]]; then
365 die
"Build directory ${CI_BUILDDIR} did not exist!";
367 if [[ ! -f "${CI_BUILDDIR}/config.log" ]]; then
368 die
"Tor was not configured in ${CI_BUILDDIR}!";
371 cp config.log
"${CI_SRCDIR}"/artifacts
373 runcmd
cd "${CI_BUILDDIR}"
377 ###############################
380 if [[ "$RUN_STAGE_BUILD" = "yes" ]] ; then
381 if [[ "$DISTCHECK" = "no" ]]; then
382 start_section
"Build"
383 runcmd
make "${make_options[@]}" all
384 cp src
/app
/tor
"${CI_SRCDIR}"/artifacts
387 export DISTCHECK_CONFIGURE_FLAGS
="${configure_options[*]}"
388 # XXXX Set make options?
389 start_section Distcheck
390 if runcmd
make "${make_options[@]}" distcheck
; then
391 hooray
"Distcheck was successful. Nothing further will be done."
392 # We have to exit early here, since we can't do any other tests.
393 cp tor-
*.
tar.gz
"${CI_SRCDIR}"/artifacts
397 runcmd
make show-distdir-testlog || true
398 runcmd
make show-distdir-core || true
399 die
"Unable to continue."
401 end_section Distcheck
405 ##############################
408 if [[ "$RUN_STAGE_TEST" == "no" ]]; then
409 echo "Skipping tests. Exiting now."
415 if [[ "${DOXYGEN}" = 'yes' ]]; then
416 start_section Doxygen
417 if [[ "${TOR_VER_AT_LEAST_043}" = 'yes' ]]; then
418 if runcmd
make doxygen
; then
419 hooray
"make doxygen has succeeded."
421 FAILED_TESTS
="${FAILED_TESTS} doxygen"
424 skipping
"make doxygen: doxygen is broken for Tor < 0.4.3"
429 if [[ "${ASCIIDOC}" = 'yes' ]]; then
430 start_section Asciidoc
431 if runcmd
make manpages
; then
432 hooray
"make manpages has succeeded."
434 FAILED_TESTS
="${FAILED_TESTS} asciidoc"
439 if [[ "${CHECK}" = "yes" ]]; then
440 start_section
"Check"
441 if runcmd
make "${make_options[@]}" check
; then
442 hooray
"make check has succeeded."
444 error
"Here are the contents of the test suite output:"
445 runcmd
cat test-suite.log || true
446 FAILED_TESTS
="${FAILED_TESTS} check"
451 if [[ "${CHUTNEY}" = "yes" ]]; then
452 start_section
"Chutney"
453 export CHUTNEY_TOR_SANDBOX
=0
454 export CHUTNEY_ALLOW_FAILURES
=2
455 if runcmd
make "${CHUTNEY_MAKE_TARGET}"; then
456 hooray
"Chutney tests have succeeded"
458 error
"Chutney says:"
459 export CHUTNEY_DATA_DIR
="${CHUTNEY_PATH}/net"
460 runcmd
"${CHUTNEY_PATH}"/tools
/diagnostics.sh || true
461 # XXXX These next two should be part of a make target.
462 runcmd
ls test_network_log || true
463 runcmd
cat test_network_log || true
464 FAILED_TESTS
="${FAILED_TESTS} chutney"
466 end_section
"Chutney"
469 if [[ "${STEM}" = "yes" ]]; then
471 # 0.3.5 and onward have now disabled onion service v2 so we need to exclude
472 # these Stem tests from now on.
473 EXCLUDE_TESTS
="--exclude-test control.controller.test_ephemeral_hidden_services_v2 --exclude-test control.controller.test_hidden_services_conf --exclude-test control.controller.test_with_ephemeral_hidden_services_basic_auth --exclude-test control.controller.test_without_ephemeral_hidden_services --exclude-test control.controller.test_with_ephemeral_hidden_services_basic_auth_no_credentials"
474 if [[ "${TOR_VER_AT_LEAST_044}" = 'yes' ]]; then
475 # XXXX This should probably be part of some test-stem make target.
477 # Disable the check around EXCLUDE_TESTS that requires double quote. We
478 # need it to be expanded.
479 # shellcheck disable=SC2086
480 if runcmd timelimit
-p -t 520 -s USR1
-T 30 -S ABRT \
481 python3
"${STEM_PATH}/run_tests.py" \
483 --integ --test control.controller \
485 --test control.base_controller \
488 --log-file stem.log
; then
489 hooray
"Stem tests have succeeded"
492 runcmd
tail -1000 "${STEM_PATH}"/test
/data
/tor_log
493 runcmd
grep -v "SocketClosed" stem.log |
tail -1000
494 FAILED_TESTS
="${FAILED_TESTS} stem"
497 skipping
"Stem: broken with <= 0.4.3. See bug tor#40077"
504 if [[ "${FAILED_TESTS}" != "" ]]; then
505 die
"Failed tests: ${FAILED_TESTS}"
508 hooray
"Everything seems fine."