2 # run tests on all Samba subprojects and push to a git tree on success
3 # Copyright Andrew Tridgell 2010
4 # released under GNU GPL v3 or later
6 from __future__
import print_function
7 from subprocess
import call
, check_call
, check_output
, Popen
, PIPE
13 from optparse
import OptionParser
16 from email
.mime
.text
import MIMEText
17 from email
.mime
.base
import MIMEBase
18 from email
.mime
.application
import MIMEApplication
19 from email
.mime
.multipart
import MIMEMultipart
20 from distutils
.sysconfig
import get_python_lib
24 from waflib
.Build
import CACHE_SUFFIX
26 sys
.path
.insert(0, "./third_party/waf")
27 from waflib
.Build
import CACHE_SUFFIX
30 os
.environ
["PYTHONUNBUFFERED"] = "1"
32 # This speeds up testing remarkably.
33 os
.environ
['TDB_NO_FSYNC'] = '1'
37 '''get to the top of the git repo'''
40 if os
.path
.exists(os
.path
.join(p
, ".git")):
42 p
= os
.path
.abspath(os
.path
.join(p
, '..'))
46 gitroot
= find_git_root()
48 raise Exception("Failed to find git root")
51 def_testbase
= os
.getenv("AUTOBUILD_TESTBASE", "/memdisk/%s" % os
.getenv('USER'))
53 parser
= OptionParser()
54 parser
.add_option("--tail", help="show output while running", default
=False, action
="store_true")
55 parser
.add_option("--keeplogs", help="keep logs", default
=False, action
="store_true")
56 parser
.add_option("--nocleanup", help="don't remove test tree", default
=False, action
="store_true")
57 parser
.add_option("--testbase", help="base directory to run tests in (default %s)" % def_testbase
,
59 parser
.add_option("--full-testbase", help="full base directory to run tests in (default %s/b$PID)" % def_testbase
,
61 parser
.add_option("--passcmd", help="command to run on success", default
=None)
62 parser
.add_option("--verbose", help="show all commands as they are run",
63 default
=False, action
="store_true")
64 parser
.add_option("--rebase", help="rebase on the given tree before testing",
65 default
=None, type='str')
66 parser
.add_option("--pushto", help="push to a git url on success",
67 default
=None, type='str')
68 parser
.add_option("--mark", help="add a Tested-By signoff before pushing",
69 default
=False, action
="store_true")
70 parser
.add_option("--fix-whitespace", help="fix whitespace on rebase",
71 default
=False, action
="store_true")
72 parser
.add_option("--retry", help="automatically retry if master changes",
73 default
=False, action
="store_true")
74 parser
.add_option("--email", help="send email to the given address on failure",
75 type='str', default
=None)
76 parser
.add_option("--email-from", help="send email from the given address",
77 type='str', default
="autobuild@samba.org")
78 parser
.add_option("--email-server", help="send email via the given server",
79 type='str', default
='localhost')
80 parser
.add_option("--always-email", help="always send email, even on success",
82 parser
.add_option("--daemon", help="daemonize after initial setup",
84 parser
.add_option("--branch", help="the branch to work on (default=master)",
85 default
="master", type='str')
86 parser
.add_option("--log-base", help="location where the logs can be found (default=cwd)",
87 default
=gitroot
, type='str')
88 parser
.add_option("--attach-logs", help="Attach logs to mails sent on success/failure?",
89 default
=False, action
="store_true")
90 parser
.add_option("--restrict-tests", help="run as make test with this TESTS= regex",
92 parser
.add_option("--enable-coverage", dest
='enable_coverage',
93 action
="store_const", const
='--enable-coverage', default
='',
94 help="Add --enable-coverage option while configure")
96 (options
, args
) = parser
.parse_args()
99 if options
.rebase
is None:
100 raise Exception('You can only use --retry if you also rebase')
102 if options
.full_testbase
is not None:
103 testbase
= options
.full_testbase
105 testbase
= "%s/b%u" % (options
.testbase
, os
.getpid())
106 test_master
= "%s/master" % testbase
107 test_prefix
= "%s/prefix" % testbase
108 test_tmpdir
= "%s/tmp" % testbase
109 os
.environ
['TMPDIR'] = test_tmpdir
111 if options
.enable_coverage
:
112 LCOV_CMD
= "cd ${TEST_SOURCE_DIR} && lcov --capture --directory . --output-file ${LOG_BASE}/${NAME}.info --rc 'geninfo_adjust_src_path=${TEST_SOURCE_DIR}/'"
114 LCOV_CMD
= 'echo "lcov skipped since no --enable-coverage specified"'
117 # If we are only running specific test,
118 # do not sleep randomly to wait for it to start
119 def random_sleep(low
, high
):
122 def random_sleep(low
, high
):
123 return 'sleep {}'.format(random
.randint(low
, high
))
131 "talloc": "lib/talloc",
132 "replace": "lib/replace",
133 "tevent": "lib/tevent",
135 "docs-xml": "docs-xml"
138 ctdb_configure_params
= " --enable-developer ${PREFIX}"
139 samba_configure_params
= " ${ENABLE_COVERAGE} ${PREFIX} --with-profiling-data"
141 samba_libs_envvars
= "PYTHONPATH=${PYTHON_PREFIX}:$PYTHONPATH"
142 samba_libs_envvars
+= " PKG_CONFIG_PATH=$PKG_CONFIG_PATH:${PREFIX_DIR}/lib/pkgconfig"
143 samba_libs_envvars
+= " ADDITIONAL_CFLAGS='-Wmissing-prototypes'"
144 samba_libs_configure_base
= samba_libs_envvars
+ " ./configure --abi-check ${ENABLE_COVERAGE} --enable-debug -C ${PREFIX}"
145 samba_libs_configure_libs
= samba_libs_configure_base
+ " --bundled-libraries=cmocka,popt,NONE"
146 samba_libs_configure_bundled_libs
= " --bundled-libraries=!talloc,!pytalloc-util,!tdb,!pytdb,!ldb,!pyldb,!pyldb-util,!tevent,!pytevent,!popt"
147 samba_libs_configure_samba
= samba_libs_configure_base
+ samba_libs_configure_bundled_libs
150 def format_option(name
, value
=None):
151 """Format option as str list."""
152 if value
is None: # boolean option
154 if not isinstance(value
, list): # single value option
157 return ['{}={}'.format(name
, item
) for item
in value
]
169 test_options
= format_option('--include-env', include_envs
)
171 test_options
= format_option('--exclude-env', exclude_envs
)
173 # join envs options to original test options
174 TESTS
= (TESTS
+ ' ' + ' '.join(test_options
)).strip()
178 _options
.append('FAIL_IMMEDIATELY=1')
180 _options
.append("TESTS='{}'".format(TESTS
))
182 return ' '.join([cmd
] + _options
)
185 # When updating this list, also update .gitlab-ci.yml to add the job
186 # and to make it a dependency of 'page' for the coverage report.
190 ("random-sleep", random_sleep(300, 900)),
191 ("configure", "./configure " + ctdb_configure_params
),
192 ("make", "make all"),
193 ("install", "make install"),
194 ("test", "make autotest"),
195 ("check-clean-tree", "../script/clean-source-tree.sh"),
196 ("clean", "make clean"),
200 ("random-sleep", random_sleep(300, 900)),
201 ("autoconf", "autoconf"),
202 ("configure", "./configure"),
203 ("make", "make html htmlman"),
204 ("clean", "make clean"),
207 # We have 'test' before 'install' because, 'test' should work without 'install (runs all the other envs)'
209 ("random-sleep", random_sleep(300, 900)),
210 ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params
),
212 ("test", make_test(exclude_envs
=[
225 "ad_dc_default_smb1",
233 "ad_member_idmap_rid",
234 "ad_member_idmap_ad",
241 "fileserver_smb1_done",
255 "ad_dc_default_smb1",
256 "ad_dc_default_smb1_done",
262 ("test-slow-none", make_test(cmd
='make test', TESTS
="--include=selftest/slow-none", include_envs
=["none"])),
264 ("install", "make install"),
265 ("check-clean-tree", "script/clean-source-tree.sh"),
266 ("clean", "make clean"),
269 # We have 'test' before 'install' because, 'test' should work without 'install (runs all the other envs)'
271 ("random-sleep", random_sleep(300, 900)),
272 ("configure", "./configure.developer --with-selftest-prefix=./bin/ab --with-system-mitkrb5 --with-experimental-mit-ad-dc" + samba_configure_params
),
274 ("test", make_test(exclude_envs
=[
287 "ad_dc_default_smb1",
288 "ad_dc_default_smb1_done",
296 "ad_member_idmap_rid",
297 "ad_member_idmap_ad",
304 "fileserver_smb1_done",
318 "ad_dc_default_smb1",
319 "ad_dc_default_smb1_done",
326 ("install", "make install"),
327 ("check-clean-tree", "script/clean-source-tree.sh"),
328 ("clean", "make clean"),
332 ("random-sleep", random_sleep(300, 900)),
333 ("configure", "./configure.developer --without-ad-dc --without-ldap --without-ads --without-json --with-selftest-prefix=./bin/ab" + samba_configure_params
),
335 ("test", make_test(include_envs
=[
344 ("check-clean-tree", "script/clean-source-tree.sh"),
345 ("clean", "make clean"),
348 "samba-fileserver": [
349 ("random-sleep", random_sleep(300, 900)),
350 ("configure", "./configure.developer --without-ad-dc --with-system-heimdalkrb5 --with-selftest-prefix=./bin/ab" + samba_configure_params
),
352 ("test", make_test(include_envs
=[
355 "fileserver_smb1_done",
357 "ktest", # ktest is also tested in samba and samba-mitkrb5
358 # but is tested here against a system Heimdal
361 ("check-clean-tree", "script/clean-source-tree.sh"),
365 ("random-sleep", random_sleep(300, 900)),
366 ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params
),
368 ("test", make_test(include_envs
=[
370 "ad_member_idmap_rid",
371 "ad_member_idmap_ad",
375 ("check-clean-tree", "script/clean-source-tree.sh"),
379 ("random-sleep", random_sleep(1, 1)),
380 ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params
),
382 ("test", make_test(include_envs
=[
390 ("check-clean-tree", "script/clean-source-tree.sh"),
394 ("random-sleep", random_sleep(1, 1)),
395 ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params
),
397 ("test", make_test(include_envs
=[
403 ("check-clean-tree", "script/clean-source-tree.sh"),
407 ("random-sleep", random_sleep(1, 1)),
408 ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params
),
410 ("test", make_test(include_envs
=[
417 ("check-clean-tree", "script/clean-source-tree.sh"),
421 ("random-sleep", random_sleep(1, 1)),
422 ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params
),
424 ("test", make_test(include_envs
=[
431 ("check-clean-tree", "script/clean-source-tree.sh"),
435 ("random-sleep", random_sleep(1, 1)),
436 ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params
),
438 ("test", make_test(include_envs
=[
439 "ad_dc_default", "ad_dc_default_smb1", "ad_dc_default_smb1_done"])),
441 ("check-clean-tree", "script/clean-source-tree.sh"),
445 ("random-sleep", random_sleep(1, 1)),
446 ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params
),
448 ("test", make_test(include_envs
=["ad_dc_slowtests"])),
450 ("check-clean-tree", "script/clean-source-tree.sh"),
453 "samba-schemaupgrade": [
454 ("random-sleep", random_sleep(1, 1)),
455 ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params
),
457 ("test", make_test(include_envs
=["schema_dc", "schema_pair_dc"])),
459 ("check-clean-tree", "script/clean-source-tree.sh"),
462 # We split out the ad_dc_ntvfs tests (which are long) so other test do not wait
463 # This is currently the longest task, so we don't randomly delay it.
464 "samba-ad-dc-ntvfs": [
465 ("random-sleep", random_sleep(1, 1)),
466 ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params
),
468 ("test", make_test(include_envs
=["ad_dc_ntvfs"])),
470 ("check-clean-tree", "script/clean-source-tree.sh"),
473 # Test fips compliance
475 ("random-sleep", random_sleep(100, 500)),
476 ("configure", "./configure.developer --with-selftest-prefix=./bin/ab --with-system-mitkrb5 --with-experimental-mit-ad-dc" + samba_configure_params
),
478 ("test", make_test(include_envs
=["ad_dc_fips", "ad_member_fips"])),
480 ("check-clean-tree", "script/clean-source-tree.sh"),
483 # run the backup/restore testenvs separately as they're fairly standalone
484 # (and CI seems to max out at ~8 different DCs running at once)
485 "samba-ad-dc-backup": [
486 ("random-sleep", random_sleep(300, 900)),
487 ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params
),
489 ("test", make_test(include_envs
=[
498 ("check-clean-tree", "script/clean-source-tree.sh"),
502 ("random-sleep", random_sleep(1, 1)),
503 ("configure", "./configure.developer --with-selftest-prefix=./bin/ab --with-system-mitkrb5 --with-experimental-mit-ad-dc" + samba_configure_params
),
505 ("test", make_test(include_envs
=[
507 "ad_member_idmap_rid",
508 "ad_member_idmap_ad",
512 ("check-clean-tree", "script/clean-source-tree.sh"),
515 "samba-ad-dc-1-mitkrb5": [
516 ("random-sleep", random_sleep(1, 1)),
517 ("configure", "./configure.developer --with-selftest-prefix=./bin/ab --with-system-mitkrb5 --with-experimental-mit-ad-dc" + samba_configure_params
),
519 ("test", make_test(include_envs
=[
527 ("check-clean-tree", "script/clean-source-tree.sh"),
530 "samba-ad-dc-4-mitkrb5": [
531 ("random-sleep", random_sleep(1, 1)),
532 ("configure", "./configure.developer --with-selftest-prefix=./bin/ab --with-system-mitkrb5 --with-experimental-mit-ad-dc" + samba_configure_params
),
534 ("test", make_test(include_envs
=[
541 ("check-clean-tree", "script/clean-source-tree.sh"),
545 ("configure", "./configure.developer --with-selftest-prefix=./bin/ab --abi-check-disable" + samba_configure_params
),
547 ("test", make_test(TESTS
="${TESTS}")),
551 # Test cross-compile infrastructure
553 ("random-sleep", random_sleep(900, 1500)),
554 ("configure-native", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params
),
555 ("configure-cross-execute", "./configure.developer --out ./bin-xe --cross-compile --cross-execute=script/identity_cc.sh" \
556 " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xe/ab" + samba_configure_params
),
557 ("verify-cross-execute-output", "grep '^Checking value of NSIG' ./bin-xe/cross-answers.txt"),
558 ("configure-cross-answers", "./configure.developer --out ./bin-xa --cross-compile" \
559 " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xa/ab" + samba_configure_params
),
560 ("compare-results", "script/compare_cc_results.py "
561 "./bin/c4che/default{} "
562 "./bin-xe/c4che/default{} "
563 "./bin-xa/c4che/default{}".format(*([CACHE_SUFFIX
]*3))),
564 ("modify-cross-answers", "sed -i.bak -e 's/^\\(Checking value of NSIG:\\) .*/\\1 \"1234\"/' ./bin-xe/cross-answers.txt"),
565 ("configure-cross-answers-modified", "./configure.developer --out ./bin-xa2 --cross-compile" \
566 " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xa2/ab" + samba_configure_params
),
567 ("verify-cross-answers", "test $(sed -n -e 's/VALUEOF_NSIG = \\(.*\\)/\\1/p' ./bin-xa2/c4che/default{})" \
568 " = \"'1234'\"".format(CACHE_SUFFIX
)),
569 ("invalidate-cross-answers", "sed -i.bak -e '/^Checking value of NSIG/d' ./bin-xe/cross-answers.txt"),
570 ("configure-cross-answers-fail", "./configure.developer --out ./bin-xa3 --cross-compile" \
571 " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xa3/ab" + samba_configure_params
+ \
575 # test build with -O3 -- catches extra warnings and bugs, tests the ad_dc environments
577 ("random-sleep", random_sleep(300, 900)),
578 ("configure", "ADDITIONAL_CFLAGS='-O3 -Wp,-D_FORTIFY_SOURCE=2' ./configure.developer --with-selftest-prefix=./bin/ab --abi-check-disable" + samba_configure_params
),
580 ("test", make_test(cmd
='make test', TESTS
="--exclude=selftest/slow-none", include_envs
=["none"])),
581 ("quicktest", make_test(cmd
='make quicktest', include_envs
=["ad_dc", "ad_dc_smb1", "ad_dc_smb1_done"])),
583 ("install", "make install"),
584 ("check-clean-tree", "script/clean-source-tree.sh"),
585 ("clean", "make clean"),
589 ("random-sleep", random_sleep(900, 1500)),
591 # make sure we have tdb around:
592 ("tdb-configure", "cd lib/tdb && PYTHONPATH=${PYTHON_PREFIX}:$PYTHONPATH PKG_CONFIG_PATH=$PKG_CONFIG_PATH:${PREFIX_DIR}/lib/pkgconfig ./configure --bundled-libraries=NONE --abi-check --enable-debug -C ${PREFIX}"),
593 ("tdb-make", "cd lib/tdb && make"),
594 ("tdb-install", "cd lib/tdb && make install"),
596 # build samba with cluster support (also building ctdb):
598 "PYTHONPATH=${PYTHON_PREFIX}:$PYTHONPATH "
599 "PKG_CONFIG_PATH=${PREFIX_DIR}/lib/pkgconfig:${PKG_CONFIG_PATH} "
600 "./configure.developer ${PREFIX} "
601 "--with-selftest-prefix=./bin/ab "
602 "--with-cluster-support "
604 "--bundled-libraries=!tdb"),
605 ("samba-make", "make"),
606 ("samba-check", "./bin/smbd -b | grep CLUSTER_SUPPORT"),
607 ("samba-install", "make install"),
608 ("ctdb-check", "test -e ${PREFIX_DIR}/sbin/ctdbd"),
611 make_test(cmd
='make test',
612 include_envs
=["clusteredmember"])
616 ("check-clean-tree", "script/clean-source-tree.sh"),
617 ("clean", "make clean"),
618 ("ctdb-clean", "cd ./ctdb && make clean"),
622 ("random-sleep", random_sleep(300, 900)),
623 ("talloc-configure", "cd lib/talloc && " + samba_libs_configure_libs
),
624 ("talloc-make", "cd lib/talloc && make"),
625 ("talloc-install", "cd lib/talloc && make install"),
627 ("tdb-configure", "cd lib/tdb && " + samba_libs_configure_libs
),
628 ("tdb-make", "cd lib/tdb && make"),
629 ("tdb-install", "cd lib/tdb && make install"),
631 ("tevent-configure", "cd lib/tevent && " + samba_libs_configure_libs
),
632 ("tevent-make", "cd lib/tevent && make"),
633 ("tevent-install", "cd lib/tevent && make install"),
635 ("ldb-configure", "cd lib/ldb && " + samba_libs_configure_libs
),
636 ("ldb-make", "cd lib/ldb && make"),
637 ("ldb-install", "cd lib/ldb && make install"),
639 ("nondevel-configure", "./configure ${PREFIX}"),
640 ("nondevel-make", "make -j"),
641 ("nondevel-check", "./bin/smbd -b | grep WITH_NTVFS_FILESERVER && exit 1; exit 0"),
642 ("nondevel-install", "make install"),
643 ("nondevel-dist", "make dist"),
645 # retry with all modules shared
646 ("allshared-distclean", "make distclean"),
647 ("allshared-configure", samba_libs_configure_samba
+ " --with-shared-modules=ALL"),
648 ("allshared-make", "make -j"),
652 # build the fuzzers (static) via the oss-fuzz script
653 ("fuzzers-mkdir-prefix", "mkdir -p ${PREFIX_DIR}"),
654 ("fuzzers-build", "OUT=${PREFIX_DIR} LIB_FUZZING_ENGINE= SANITIZER=address CXX= CFLAGS= ADDITIONAL_LDFLAGS='-fuse-ld=bfd' ./lib/fuzzing/oss-fuzz/build_samba.sh --enable-afl"),
657 # * Test smbd and smbtorture can build semi-static
659 # * Test Samba without python still builds.
661 # When this test fails due to more use of Python, the expectations
662 # is that the newly failing part of the code should be disabled
663 # when --disable-python is set (rather than major work being done
664 # to support this environment).
666 # The target here is for vendors shipping a minimal smbd.
667 "samba-minimal-smbd": [
668 ("random-sleep", random_sleep(300, 900)),
670 # build with all modules static
671 ("allstatic-configure", "./configure.developer " + samba_configure_params
+ " --with-static-modules=ALL"),
672 ("allstatic-make", "make -j"),
673 ("allstatic-test", make_test(TESTS
="samba3.smb2.create.*nt4_dc")),
676 # retry with nonshared smbd and smbtorture
677 ("nonshared-distclean", "make distclean"),
678 ("nonshared-configure", "./configure.developer " + samba_configure_params
+ " --bundled-libraries=ALL --with-static-modules=ALL --nonshared-binary=smbtorture,smbd/smbd"),
679 ("nonshared-make", "make -j"),
681 ("configure", "./configure.developer ${ENABLE_COVERAGE} ${PREFIX} --with-profiling-data --disable-python --without-ad-dc"),
683 ("find-python", "script/find_python.sh ${PREFIX}"),
684 ("test", "make test-nopython"),
686 ("check-clean-tree", "script/clean-source-tree.sh"),
687 ("clean", "make clean"),
689 ("talloc-configure", "cd lib/talloc && " + samba_libs_configure_base
+ " --bundled-libraries=cmocka,NONE --disable-python"),
690 ("talloc-make", "cd lib/talloc && make"),
691 ("talloc-install", "cd lib/talloc && make install"),
693 ("tdb-configure", "cd lib/tdb && " + samba_libs_configure_base
+ " --bundled-libraries=cmocka,NONE --disable-python"),
694 ("tdb-make", "cd lib/tdb && make"),
695 ("tdb-install", "cd lib/tdb && make install"),
697 ("tevent-configure", "cd lib/tevent && " + samba_libs_configure_base
+ " --bundled-libraries=cmocka,NONE --disable-python"),
698 ("tevent-make", "cd lib/tevent && make"),
699 ("tevent-install", "cd lib/tevent && make install"),
701 ("ldb-configure", "cd lib/ldb && " + samba_libs_configure_base
+ " --bundled-libraries=cmocka,NONE --disable-python"),
702 ("ldb-make", "cd lib/ldb && make"),
703 ("ldb-install", "cd lib/ldb && make install"),
705 # retry against installed library packages, but no required modules
706 ("libs-configure", samba_libs_configure_base
+ samba_libs_configure_bundled_libs
+ " --disable-python --without-ad-dc --with-static-modules=!FORCED,!DEFAULT --with-shared-modules=!FORCED,!DEFAULT"),
707 ("libs-make", "make -j"),
708 ("libs-install", "make install"),
709 ("libs-check-clean-tree", "script/clean-source-tree.sh"),
710 ("libs-clean", "make clean"),
715 ("random-sleep", random_sleep(60, 600)),
716 ("configure", "./configure ${ENABLE_COVERAGE} --enable-developer -C ${PREFIX}"),
718 ("install", "make install"),
719 ("test", "make test"),
721 ("clean", "make clean"),
722 ("configure-no-lmdb", "./configure ${ENABLE_COVERAGE} --enable-developer --without-ldb-lmdb -C ${PREFIX}"),
723 ("make-no-lmdb", "make"),
724 ("test-no-lmdb", "make test"),
725 ("lcov-no-lmdb", LCOV_CMD
),
726 ("install-no-lmdb", "make install"),
727 ("check-clean-tree", "../../script/clean-source-tree.sh"),
728 ("distcheck", "make distcheck"),
729 ("clean", "make clean"),
733 ("random-sleep", random_sleep(60, 600)),
734 ("configure", "./configure ${ENABLE_COVERAGE} --enable-developer -C ${PREFIX}"),
736 ("install", "make install"),
737 ("test", "make test"),
739 ("check-clean-tree", "../../script/clean-source-tree.sh"),
740 ("distcheck", "make distcheck"),
741 ("clean", "make clean"),
745 ("random-sleep", random_sleep(60, 600)),
746 ("configure", "./configure ${ENABLE_COVERAGE} --enable-developer -C ${PREFIX}"),
748 ("install", "make install"),
749 ("test", "make test"),
751 ("check-clean-tree", "../../script/clean-source-tree.sh"),
752 ("distcheck", "make distcheck"),
753 ("clean", "make clean"),
757 ("random-sleep", random_sleep(60, 600)),
758 ("configure", "./configure ${ENABLE_COVERAGE} --enable-developer -C ${PREFIX}"),
760 ("install", "make install"),
761 ("test", "make test"),
763 ("check-clean-tree", "../../script/clean-source-tree.sh"),
764 ("distcheck", "make distcheck"),
765 ("clean", "make clean"),
769 ("random-sleep", random_sleep(60, 600)),
770 ("configure", "./configure ${ENABLE_COVERAGE} --enable-developer -C ${PREFIX}"),
772 ("install", "make install"),
773 ("test", "make test"),
775 ("check-clean-tree", "../../script/clean-source-tree.sh"),
776 ("distcheck", "make distcheck"),
777 ("clean", "make clean"),
781 ("random-sleep", random_sleep(60, 600)),
782 ("configure", "perl Makefile.PL PREFIX=${PREFIX_DIR}"),
783 ("touch", "touch *.yp"),
785 ("test", "make test"),
786 ("install", "make install"),
787 ("checkout-yapp-generated", "git checkout lib/Parse/Pidl/IDL.pm lib/Parse/Pidl/Expr.pm"),
788 ("check-clean-tree", "../script/clean-source-tree.sh"),
789 ("clean", "make clean"),
792 # these are useful for debugging autobuild
793 'pass': [("pass", 'echo passing && /bin/true')],
794 'fail': [("fail", 'echo failing && /bin/false')],
797 defaulttasks
= list(tasks
.keys())
799 defaulttasks
.remove("pass")
800 defaulttasks
.remove("fail")
801 defaulttasks
.remove("samba-test-only")
802 defaulttasks
.remove("samba-fuzz")
803 defaulttasks
.remove("samba-fips")
804 if os
.environ
.get("AUTOBUILD_SKIP_SAMBA_O3", "0") == "1":
805 defaulttasks
.remove("samba-o3")
814 def run_cmd(cmd
, dir=".", show
=None, output
=False, checkfail
=True):
816 show
= options
.verbose
818 do_print("Running: '%s' in '%s'" % (cmd
, dir))
820 out
= check_output([cmd
], shell
=True, cwd
=dir)
821 return out
.decode(encoding
='utf-8', errors
='backslashreplace')
823 return check_call(cmd
, shell
=True, cwd
=dir)
825 return call(cmd
, shell
=True, cwd
=dir)
828 class builder(object):
829 '''handle build of one directory'''
831 def __init__(self
, name
, sequence
, cp
=True):
833 self
.dir = builddirs
.get(name
, '.')
834 self
.tag
= self
.name
.replace('/', '_')
835 self
.sequence
= sequence
837 self
.stdout_path
= "%s/%s.stdout" % (gitroot
, self
.tag
)
838 self
.stderr_path
= "%s/%s.stderr" % (gitroot
, self
.tag
)
840 do_print("stdout for %s in %s" % (self
.name
, self
.stdout_path
))
841 do_print("stderr for %s in %s" % (self
.name
, self
.stderr_path
))
842 run_cmd("rm -f %s %s" % (self
.stdout_path
, self
.stderr_path
))
843 self
.stdout
= open(self
.stdout_path
, 'w')
844 self
.stderr
= open(self
.stderr_path
, 'w')
845 self
.stdin
= open("/dev/null", 'r')
846 self
.test_source_dir
= "%s/%s" % (testbase
, self
.tag
)
847 self
.cwd
= "%s/%s" % (self
.test_source_dir
, self
.dir)
848 self
.prefix
= "%s/%s" % (test_prefix
, self
.tag
)
849 run_cmd("rm -rf %s" % self
.test_source_dir
)
850 run_cmd("rm -rf %s" % self
.prefix
)
852 run_cmd("cp -R -a -l %s %s" % (test_master
, self
.test_source_dir
), dir=test_master
, show
=True)
854 run_cmd("git clone --recursive --shared %s %s" % (test_master
, self
.test_source_dir
), dir=test_master
, show
=True)
857 def start_next(self
):
858 if self
.next
== len(self
.sequence
):
859 if not options
.nocleanup
:
860 run_cmd("rm -rf %s" % self
.test_source_dir
)
861 run_cmd("rm -rf %s" % self
.prefix
)
862 do_print('%s: Completed OK' % self
.name
)
865 (self
.stage
, self
.cmd
) = self
.sequence
[self
.next
]
866 self
.cmd
= self
.cmd
.replace("${PYTHON_PREFIX}", get_python_lib(plat_specific
=1, standard_lib
=0, prefix
=self
.prefix
))
867 self
.cmd
= self
.cmd
.replace("${PREFIX}", "--prefix=%s" % self
.prefix
)
868 self
.cmd
= self
.cmd
.replace("${PREFIX_DIR}", "%s" % self
.prefix
)
869 self
.cmd
= self
.cmd
.replace("${TESTS}", options
.restrict_tests
)
870 self
.cmd
= self
.cmd
.replace("${TEST_SOURCE_DIR}", self
.test_source_dir
)
871 self
.cmd
= self
.cmd
.replace("${LOG_BASE}", options
.log_base
)
872 self
.cmd
= self
.cmd
.replace("${NAME}", self
.name
)
873 self
.cmd
= self
.cmd
.replace("${ENABLE_COVERAGE}", options
.enable_coverage
)
874 do_print('%s: [%s] Running %s in %r' % (self
.name
, self
.stage
, self
.cmd
, self
.cwd
))
875 self
.proc
= Popen(self
.cmd
, shell
=True,
876 close_fds
=True, cwd
=self
.cwd
,
877 stdout
=self
.stdout
, stderr
=self
.stderr
, stdin
=self
.stdin
)
881 class buildlist(object):
882 '''handle build of multiple directories'''
884 def __init__(self
, tasknames
, rebase_url
, rebase_branch
="master"):
885 self
.tail_proc
= None
888 if options
.restrict_tests
:
889 tasknames
= ["samba-test-only"]
891 tasknames
= defaulttasks
893 self
.tlist
= [builder(n
, tasks
[n
], cp
=(n
!= "pidl")) for n
in tasknames
]
896 rebase_remote
= "rebaseon"
897 retry_task
= [("retry",
899 git remote add -t %s %s %s
903 git describe %s/%s > old_remote_branch.desc
905 git describe %s/%s > remote_branch.desc
906 diff old_remote_branch.desc remote_branch.desc
909 rebase_branch
, rebase_remote
, rebase_url
,
911 rebase_remote
, rebase_branch
,
913 rebase_remote
, rebase_branch
916 self
.retry
= builder('retry', retry_task
, cp
=False)
917 self
.need_retry
= False
920 if self
.tail_proc
is not None:
921 self
.tail_proc
.terminate()
922 self
.tail_proc
.wait()
923 self
.tail_proc
= None
924 if self
.retry
is not None:
925 self
.retry
.proc
.terminate()
926 self
.retry
.proc
.wait()
929 if b
.proc
is not None:
930 run_cmd("killbysubdir %s > /dev/null 2>&1" % b
.test_source_dir
, checkfail
=False)
942 b
.status
= b
.proc
.poll()
948 ret
= self
.retry
.proc
.poll()
950 self
.need_retry
= True
960 if options
.retry
and self
.need_retry
:
962 do_print("retry needed")
963 return (0, None, None, None, "retry")
966 if os
.WIFSIGNALED(b
.status
) or os
.WEXITSTATUS(b
.status
) != 0:
968 return (b
.status
, b
.name
, b
.stage
, b
.tag
, "%s: [%s] failed '%s' with status %d" % (b
.name
, b
.stage
, b
.cmd
, b
.status
))
971 return (0, None, None, None, "All OK")
973 def write_system_info(self
, filename
):
974 with
open(filename
, 'w') as f
:
975 for cmd
in ['uname -a',
982 'df -m %s' % testbase
]:
984 out
= run_cmd(cmd
, output
=True, checkfail
=False)
985 except subprocess
.CalledProcessError
as e
:
986 out
= "<failed: %s>" % str(e
)
987 print('### %s' % cmd
, file=f
)
991 def tarlogs(self
, fname
):
992 with tarfile
.open(fname
, "w:gz") as tar
:
994 tar
.add(b
.stdout_path
, arcname
="%s.stdout" % b
.tag
)
995 tar
.add(b
.stderr_path
, arcname
="%s.stderr" % b
.tag
)
996 if os
.path
.exists("autobuild.log"):
997 tar
.add("autobuild.log")
998 filename
= 'system-info.txt'
999 self
.write_system_info(filename
)
1002 def remove_logs(self
):
1003 for b
in self
.tlist
:
1004 os
.unlink(b
.stdout_path
)
1005 os
.unlink(b
.stderr_path
)
1007 def start_tail(self
):
1008 cmd
= ["tail", "-f"]
1009 for b
in self
.tlist
:
1010 cmd
.append(b
.stdout_path
)
1011 cmd
.append(b
.stderr_path
)
1012 self
.tail_proc
= Popen(cmd
, close_fds
=True)
1016 if options
.nocleanup
:
1018 run_cmd("stat %s || true" % test_tmpdir
, show
=True)
1019 run_cmd("stat %s" % testbase
, show
=True)
1020 do_print("Cleaning up %r" % cleanup_list
)
1021 for d
in cleanup_list
:
1022 run_cmd("rm -rf %s" % d
)
1025 def daemonize(logfile
):
1027 if pid
== 0: # Parent
1030 if pid
!= 0: # Actual daemon
1035 import resource
# Resource usage information.
1036 maxfd
= resource
.getrlimit(resource
.RLIMIT_NOFILE
)[1]
1037 if maxfd
== resource
.RLIM_INFINITY
:
1038 maxfd
= 1024 # Rough guess at maximum number of open file descriptors.
1039 for fd
in range(0, maxfd
):
1044 os
.open(logfile
, os
.O_RDWR | os
.O_CREAT
)
1049 def write_pidfile(fname
):
1050 '''write a pid file, cleanup on exit'''
1051 with
open(fname
, mode
='w') as f
:
1052 f
.write("%u\n" % os
.getpid())
1055 def rebase_tree(rebase_url
, rebase_branch
="master"):
1056 rebase_remote
= "rebaseon"
1057 do_print("Rebasing on %s" % rebase_url
)
1058 run_cmd("git describe HEAD", show
=True, dir=test_master
)
1059 run_cmd("git remote add -t %s %s %s" %
1060 (rebase_branch
, rebase_remote
, rebase_url
),
1061 show
=True, dir=test_master
)
1062 run_cmd("git fetch %s" % rebase_remote
, show
=True, dir=test_master
)
1063 if options
.fix_whitespace
:
1064 run_cmd("git rebase --force-rebase --whitespace=fix %s/%s" %
1065 (rebase_remote
, rebase_branch
),
1066 show
=True, dir=test_master
)
1068 run_cmd("git rebase --force-rebase %s/%s" %
1069 (rebase_remote
, rebase_branch
),
1070 show
=True, dir=test_master
)
1071 diff
= run_cmd("git --no-pager diff HEAD %s/%s" %
1072 (rebase_remote
, rebase_branch
),
1073 dir=test_master
, output
=True)
1075 do_print("No differences between HEAD and %s/%s - exiting" %
1076 (rebase_remote
, rebase_branch
))
1078 run_cmd("git describe %s/%s" %
1079 (rebase_remote
, rebase_branch
),
1080 show
=True, dir=test_master
)
1081 run_cmd("git describe HEAD", show
=True, dir=test_master
)
1082 run_cmd("git --no-pager diff --stat HEAD %s/%s" %
1083 (rebase_remote
, rebase_branch
),
1084 show
=True, dir=test_master
)
1087 def push_to(push_url
, push_branch
="master"):
1088 push_remote
= "pushto"
1089 do_print("Pushing to %s" % push_url
)
1091 run_cmd("git config --replace-all core.editor script/commit_mark.sh", dir=test_master
)
1092 run_cmd("git commit --amend -c HEAD", dir=test_master
)
1093 # the notes method doesn't work yet, as metze hasn't allowed refs/notes/* in master
1094 # run_cmd("EDITOR=script/commit_mark.sh git notes edit HEAD", dir=test_master)
1095 run_cmd("git remote add -t %s %s %s" %
1096 (push_branch
, push_remote
, push_url
),
1097 show
=True, dir=test_master
)
1098 run_cmd("git push %s +HEAD:%s" %
1099 (push_remote
, push_branch
),
1100 show
=True, dir=test_master
)
1103 def send_email(subject
, text
, log_tar
):
1104 if options
.email
is None:
1105 do_print("not sending email because the recipient is not set")
1106 do_print("the text content would have been:\n\nSubject: %s\n\n%s" %
1109 outer
= MIMEMultipart()
1110 outer
['Subject'] = subject
1111 outer
['To'] = options
.email
1112 outer
['From'] = options
.email_from
1113 outer
['Date'] = email
.utils
.formatdate(localtime
=True)
1114 outer
.preamble
= 'Autobuild mails are now in MIME because we optionally attach the logs.\n'
1115 outer
.attach(MIMEText(text
, 'plain'))
1116 if options
.attach_logs
:
1117 with
open(log_tar
, 'rb') as fp
:
1118 msg
= MIMEApplication(fp
.read(), 'gzip', email
.encoders
.encode_base64
)
1119 # Set the filename parameter
1120 msg
.add_header('Content-Disposition', 'attachment', filename
=os
.path
.basename(log_tar
))
1122 content
= outer
.as_string()
1123 s
= smtplib
.SMTP(options
.email_server
)
1124 email_user
= os
.getenv('SMTP_USERNAME')
1125 email_password
= os
.getenv('SMTP_PASSWORD')
1126 if email_user
is not None:
1128 s
.login(email_user
, email_password
)
1130 s
.sendmail(options
.email_from
, [options
.email
], content
)
1135 def email_failure(status
, failed_task
, failed_stage
, failed_tag
, errstr
,
1136 elapsed_time
, log_base
=None, add_log_tail
=True):
1137 '''send an email to options.email about the failure'''
1138 elapsed_minutes
= elapsed_time
/ 60.0
1139 if log_base
is None:
1144 Your autobuild on %s failed after %.1f minutes
1145 when trying to test %s with the following error:
1149 the autobuild has been abandoned. Please fix the error and resubmit.
1151 A summary of the autobuild process is here:
1154 ''' % (platform
.node(), elapsed_minutes
, failed_task
, errstr
, log_base
)
1156 if options
.restrict_tests
:
1158 The build was restricted to tests matching %s\n""" % options
.restrict_tests
1160 if failed_task
!= 'rebase':
1162 You can see logs of the failed task here:
1167 or you can get full logs of all tasks in this job here:
1171 The top commit for the tree that was built was:
1175 ''' % (log_base
, failed_tag
, log_base
, failed_tag
, log_base
, top_commit_msg
)
1178 f
= open("%s/%s.stdout" % (gitroot
, failed_tag
), 'r')
1179 lines
= f
.readlines()
1180 log_tail
= "".join(lines
[-50:])
1181 num_lines
= len(lines
)
1183 # Also include stderr (compile failures) if < 50 lines of stdout
1184 f
= open("%s/%s.stderr" % (gitroot
, failed_tag
), 'r')
1185 log_tail
+= "".join(f
.readlines()[-(50 - num_lines
):])
1188 The last 50 lines of log messages:
1194 logs
= os
.path
.join(gitroot
, 'logs.tar.gz')
1195 send_email('autobuild[%s] failure on %s for task %s during %s'
1196 % (options
.branch
, platform
.node(), failed_task
, failed_stage
),
1200 def email_success(elapsed_time
, log_base
=None):
1201 '''send an email to options.email about a successful build'''
1202 if log_base
is None:
1207 Your autobuild on %s has succeeded after %.1f minutes.
1209 ''' % (platform
.node(), elapsed_time
/ 60.)
1211 if options
.restrict_tests
:
1213 The build was restricted to tests matching %s\n""" % options
.restrict_tests
1215 if options
.keeplogs
:
1218 you can get full logs of all tasks in this job here:
1225 The top commit for the tree that was built was:
1228 ''' % top_commit_msg
1230 logs
= os
.path
.join(gitroot
, 'logs.tar.gz')
1231 send_email('autobuild[%s] success on %s' % (options
.branch
, platform
.node()),
1235 # get the top commit message, for emails
1236 top_commit_msg
= run_cmd("git log -1", dir=gitroot
, output
=True)
1239 os
.makedirs(testbase
)
1240 except Exception as reason
:
1241 raise Exception("Unable to create %s : %s" % (testbase
, reason
))
1242 cleanup_list
.append(testbase
)
1245 logfile
= os
.path
.join(testbase
, "log")
1246 do_print("Forking into the background, writing progress to %s" % logfile
)
1249 write_pidfile(gitroot
+ "/autobuild.pid")
1251 start_time
= time
.time()
1255 run_cmd("rm -rf %s" % test_tmpdir
, show
=True)
1256 os
.makedirs(test_tmpdir
)
1257 # The waf uninstall code removes empty directories all the way
1258 # up the tree. Creating a file in test_tmpdir stops it from
1260 run_cmd("touch %s" % os
.path
.join(test_tmpdir
,
1261 ".directory-is-not-empty"), show
=True)
1262 run_cmd("stat %s" % test_tmpdir
, show
=True)
1263 run_cmd("stat %s" % testbase
, show
=True)
1264 run_cmd("git clone --recursive --shared %s %s" % (gitroot
, test_master
), show
=True, dir=gitroot
)
1270 if options
.rebase
is not None:
1271 rebase_tree(options
.rebase
, rebase_branch
=options
.branch
)
1273 cleanup_list
.append(gitroot
+ "/autobuild.pid")
1275 elapsed_time
= time
.time() - start_time
1276 email_failure(-1, 'rebase', 'rebase', 'rebase',
1277 'rebase on %s failed' % options
.branch
,
1278 elapsed_time
, log_base
=options
.log_base
)
1282 blist
= buildlist(args
, options
.rebase
, rebase_branch
=options
.branch
)
1285 (status
, failed_task
, failed_stage
, failed_tag
, errstr
) = blist
.run()
1286 if status
!= 0 or errstr
!= "retry":
1293 cleanup_list
.append(gitroot
+ "/autobuild.pid")
1299 do_print("waiting for tail to flush")
1302 elapsed_time
= time
.time() - start_time
1304 if options
.passcmd
is not None:
1305 do_print("Running passcmd: %s" % options
.passcmd
)
1306 run_cmd(options
.passcmd
, dir=test_master
)
1307 if options
.pushto
is not None:
1308 push_to(options
.pushto
, push_branch
=options
.branch
)
1309 if options
.keeplogs
or options
.attach_logs
:
1310 blist
.tarlogs("logs.tar.gz")
1311 do_print("Logs in logs.tar.gz")
1312 if options
.always_email
:
1313 email_success(elapsed_time
, log_base
=options
.log_base
)
1319 # something failed, gather a tar of the logs
1320 blist
.tarlogs("logs.tar.gz")
1322 if options
.email
is not None:
1323 email_failure(status
, failed_task
, failed_stage
, failed_tag
, errstr
,
1324 elapsed_time
, log_base
=options
.log_base
)
1326 elapsed_minutes
= elapsed_time
/ 60.0
1329 ####################################################################
1333 Your autobuild[%s] on %s failed after %.1f minutes
1334 when trying to test %s with the following error:
1338 the autobuild has been abandoned. Please fix the error and resubmit.
1340 ####################################################################
1342 ''' % (options
.branch
, platform
.node(), elapsed_minutes
, failed_task
, errstr
))
1346 do_print("Logs in logs.tar.gz")