2 # run tests on all Samba subprojects and push to a git tree on success
3 # Copyright Andrew Tridgell 2010
4 # released under GNU GPL v3 or later
6 from subprocess
import call
, check_call
, check_output
, Popen
, PIPE
, CalledProcessError
12 from optparse
import OptionParser
15 from email
.mime
.text
import MIMEText
16 from email
.mime
.base
import MIMEBase
17 from email
.mime
.application
import MIMEApplication
18 from email
.mime
.multipart
import MIMEMultipart
19 from distutils
.sysconfig
import get_python_lib
23 from waflib
.Build
import CACHE_SUFFIX
25 sys
.path
.insert(0, "./third_party/waf")
26 from waflib
.Build
import CACHE_SUFFIX
29 os
.environ
["PYTHONUNBUFFERED"] = "1"
31 # This speeds up testing remarkably.
32 os
.environ
['TDB_NO_FSYNC'] = '1'
36 '''get to the top of the git repo'''
39 if os
.path
.exists(os
.path
.join(p
, ".git")):
41 p
= os
.path
.abspath(os
.path
.join(p
, '..'))
45 gitroot
= find_git_root()
47 raise Exception("Failed to find git root")
50 def_testbase
= os
.getenv("AUTOBUILD_TESTBASE", "/memdisk/%s" % os
.getenv('USER'))
52 parser
= OptionParser()
53 parser
.add_option("--tail", help="show output while running", default
=False, action
="store_true")
54 parser
.add_option("--keeplogs", help="keep logs", default
=False, action
="store_true")
55 parser
.add_option("--nocleanup", help="don't remove test tree", default
=False, action
="store_true")
56 parser
.add_option("--skip-dependencies", help="skip to run task dependency tasks", default
=False, action
="store_true")
57 parser
.add_option("--testbase", help="base directory to run tests in (default %s)" % def_testbase
,
59 parser
.add_option("--full-testbase", help="full base directory to run tests in (default %s/b$PID)" % def_testbase
,
61 parser
.add_option("--passcmd", help="command to run on success", default
=None)
62 parser
.add_option("--verbose", help="show all commands as they are run",
63 default
=False, action
="store_true")
64 parser
.add_option("--rebase", help="rebase on the given tree before testing",
65 default
=None, type='str')
66 parser
.add_option("--pushto", help="push to a git url on success",
67 default
=None, type='str')
68 parser
.add_option("--mark", help="add a Tested-By signoff before pushing",
69 default
=False, action
="store_true")
70 parser
.add_option("--fix-whitespace", help="fix whitespace on rebase",
71 default
=False, action
="store_true")
72 parser
.add_option("--retry", help="automatically retry if master changes",
73 default
=False, action
="store_true")
74 parser
.add_option("--email", help="send email to the given address on failure",
75 type='str', default
=None)
76 parser
.add_option("--email-from", help="send email from the given address",
77 type='str', default
="autobuild@samba.org")
78 parser
.add_option("--email-server", help="send email via the given server",
79 type='str', default
='localhost')
80 parser
.add_option("--always-email", help="always send email, even on success",
82 parser
.add_option("--daemon", help="daemonize after initial setup",
84 parser
.add_option("--branch", help="the branch to work on (default=master)",
85 default
="master", type='str')
86 parser
.add_option("--log-base", help="location where the logs can be found (default=cwd)",
87 default
=gitroot
, type='str')
88 parser
.add_option("--attach-logs", help="Attach logs to mails sent on success/failure?",
89 default
=False, action
="store_true")
90 parser
.add_option("--restrict-tests", help="run as make test with this TESTS= regex",
92 parser
.add_option("--enable-coverage", dest
='enable_coverage',
93 action
="store_const", const
='--enable-coverage', default
='',
94 help="Add --enable-coverage option while configure")
96 (options
, args
) = parser
.parse_args()
99 if options
.rebase
is None:
100 raise Exception('You can only use --retry if you also rebase')
102 if options
.full_testbase
is not None:
103 testbase
= options
.full_testbase
105 testbase
= "%s/b%u" % (options
.testbase
, os
.getpid())
106 test_master
= "%s/master" % testbase
107 test_prefix
= "%s/prefix" % testbase
108 test_tmpdir
= "%s/tmp" % testbase
109 os
.environ
['TMPDIR'] = test_tmpdir
111 if options
.enable_coverage
:
112 LCOV_CMD
= "cd ${TEST_SOURCE_DIR} && lcov --capture --directory . --output-file ${LOG_BASE}/${NAME}.info --rc 'geninfo_adjust_src_path=${TEST_SOURCE_DIR}/'"
114 LCOV_CMD
= 'echo "lcov skipped since no --enable-coverage specified"'
116 if options
.enable_coverage
:
117 PUBLISH_DOCS
= "mkdir -p ${LOG_BASE}/public && mv output/htmldocs ${LOG_BASE}/public/htmldocs"
119 PUBLISH_DOCS
= 'echo "HTML documentation publishing skipped since no --enable-coverage specified"'
121 CLEAN_SOURCE_TREE_CMD
= "cd ${TEST_SOURCE_DIR} && script/clean-source-tree.sh"
123 def nm_grep_symbols(sofile
, expected_symbols
=""):
124 return "nm " + sofile
+ " | " + \
125 "egrep -v ' (__bss_start|_edata|_init|_fini|_end)' | " + \
126 "egrep -v '" + expected_symbols
+ "' |" + \
127 "egrep ' [BDGTRVWS] ' && exit 1; exit 0;"
130 # If we are only running specific test,
131 # do not sleep randomly to wait for it to start
132 def random_sleep(low
, high
):
135 def random_sleep(low
, high
):
136 return 'sleep {}'.format(random
.randint(low
, high
))
144 "talloc": "lib/talloc",
145 "replace": "lib/replace",
146 "tevent": "lib/tevent",
148 "docs-xml": "docs-xml"
151 ctdb_configure_params
= " --enable-developer ${PREFIX}"
152 samba_configure_params
= " ${ENABLE_COVERAGE} ${PREFIX} --with-profiling-data"
154 samba_libs_envvars
= "PYTHONPATH=${PYTHON_PREFIX}:$PYTHONPATH"
155 samba_libs_envvars
+= " PKG_CONFIG_PATH=$PKG_CONFIG_PATH:${PREFIX_DIR}/lib/pkgconfig"
156 samba_libs_envvars
+= " ADDITIONAL_CFLAGS='-Wmissing-prototypes'"
157 samba_libs_configure_base
= samba_libs_envvars
+ " ./configure --abi-check ${ENABLE_COVERAGE} --enable-debug -C ${PREFIX}"
158 samba_libs_configure_libs
= samba_libs_configure_base
+ " --bundled-libraries=cmocka,popt,NONE"
159 samba_libs_configure_bundled_libs
= " --bundled-libraries=!talloc,!pytalloc-util,!tdb,!pytdb,!ldb,!pyldb,!pyldb-util,!tevent,!pytevent,!popt"
160 samba_libs_configure_samba
= samba_libs_configure_base
+ samba_libs_configure_bundled_libs
163 def format_option(name
, value
=None):
164 """Format option as str list."""
165 if value
is None: # boolean option
167 if not isinstance(value
, list): # single value option
170 return ['{}={}'.format(name
, item
) for item
in value
]
175 INJECT_SELFTEST_PREFIX
=1,
182 test_options
= format_option('--include-env', include_envs
)
184 test_options
= format_option('--exclude-env', exclude_envs
)
186 # join envs options to original test options
187 TESTS
= (TESTS
+ ' ' + ' '.join(test_options
)).strip()
191 # Allow getting a full CI with
192 # git push -o ci.variable='AUTOBUILD_FAIL_IMMEDIATELY=0'
194 FAIL_IMMEDIATELY
= os
.getenv("AUTOBUILD_FAIL_IMMEDIATELY", "1")
196 if int(FAIL_IMMEDIATELY
):
197 _options
.append('FAIL_IMMEDIATELY=1')
199 _options
.append("TESTS='{}'".format(TESTS
))
201 if INJECT_SELFTEST_PREFIX
:
202 _options
.append("TEST_OPTIONS='--with-selftest-prefix={}'".format("${SELFTEST_PREFIX}"))
203 _options
.append("--directory='{}'".format("${TEST_SOURCE_DIR}"))
205 return ' '.join([cmd
] + _options
)
208 # When updating this list, also update .gitlab-ci.yml to add the job
209 # and to make it a dependency of 'page' for the coverage report.
214 ("random-sleep", random_sleep(300, 900)),
215 ("configure", "./configure " + ctdb_configure_params
),
216 ("make", "make all"),
217 ("install", "make install"),
218 ("test", "make autotest"),
219 ("check-clean-tree", CLEAN_SOURCE_TREE_CMD
),
220 ("clean", "make clean"),
225 ("random-sleep", random_sleep(300, 900)),
226 ("autoconf", "autoconf"),
227 ("configure", "./configure"),
228 ("make", "make html htmlman"),
229 ("publish-docs", PUBLISH_DOCS
),
230 ("clean", "make clean"),
235 "git-clone-required": True,
237 ("configure", "./configure.developer" + samba_configure_params
),
239 ("check-clean-tree", CLEAN_SOURCE_TREE_CMD
),
240 ("chmod-R-a-w", "chmod -R a-w ."),
245 "git-clone-required": True,
247 ("configure", "./configure.developer --with-system-mitkrb5 --with-experimental-mit-ad-dc" + samba_configure_params
),
249 ("check-clean-tree", CLEAN_SOURCE_TREE_CMD
),
250 ("chmod-R-a-w", "chmod -R a-w ."),
255 "git-clone-required": True,
257 ("configure", "./configure.developer --without-ad-dc --without-ldap --without-ads --without-json" + samba_configure_params
),
259 ("check-clean-tree", CLEAN_SOURCE_TREE_CMD
),
260 ("chmod-R-a-w", "chmod -R a-w ."),
265 "git-clone-required": True,
267 ("configure", "./configure.developer --without-ad-dc --with-system-heimdalkrb5" + samba_configure_params
),
269 ("check-clean-tree", CLEAN_SOURCE_TREE_CMD
),
270 ("chmod-R-a-w", "chmod -R a-w ."),
274 "samba-no-opath-build": {
275 "git-clone-required": True,
277 ("configure", "ADDITIONAL_CFLAGS='-DDISABLE_OPATH=1' ./configure.developer --without-ad-dc " + samba_configure_params
),
279 ("check-clean-tree", CLEAN_SOURCE_TREE_CMD
),
280 ("chmod-R-a-w", "chmod -R a-w ."),
284 # We have 'test' before 'install' because, 'test' should work without 'install (runs all the other envs)'
287 ("random-sleep", random_sleep(300, 900)),
288 ("configure", "./configure.developer" + samba_configure_params
),
290 ("test", make_test(exclude_envs
=[
303 "ad_dc_default_smb1",
311 "ad_member_idmap_rid",
312 "ad_member_idmap_ad",
320 "fileserver_smb1_done",
334 "ad_dc_default_smb1",
335 "ad_dc_default_smb1_done",
343 ("test-slow-none", make_test(cmd
='make test', TESTS
="--include=selftest/slow-none", include_envs
=["none"])),
345 ("install", "make install"),
346 ("check-clean-tree", CLEAN_SOURCE_TREE_CMD
),
347 ("clean", "make clean"),
351 # We have 'test' before 'install' because, 'test' should work without 'install (runs all the other envs)'
354 ("random-sleep", random_sleep(300, 900)),
355 ("configure", "./configure.developer --with-system-mitkrb5 --with-experimental-mit-ad-dc" + samba_configure_params
),
357 ("test", make_test(exclude_envs
=[
370 "ad_dc_default_smb1",
371 "ad_dc_default_smb1_done",
379 "ad_member_idmap_rid",
380 "ad_member_idmap_ad",
388 "fileserver_smb1_done",
402 "ad_dc_default_smb1",
403 "ad_dc_default_smb1_done",
412 ("install", "make install"),
413 ("check-clean-tree", CLEAN_SOURCE_TREE_CMD
),
414 ("clean", "make clean"),
419 "dependency": "samba-nt4-build",
421 ("random-sleep", random_sleep(300, 900)),
422 ("test", make_test(include_envs
=[
431 ("check-clean-tree", CLEAN_SOURCE_TREE_CMD
),
435 "samba-fileserver": {
436 "dependency": "samba-h5l-build",
438 ("random-sleep", random_sleep(300, 900)),
439 ("test", make_test(include_envs
=[
442 "fileserver_smb1_done",
444 "ktest", # ktest is also tested in samba-ktest-mit samba
445 # and samba-mitkrb5 but is tested here against
449 ("check-clean-tree", CLEAN_SOURCE_TREE_CMD
),
453 # This is a full build without the AD DC so we test the build with
454 # MIT Kerberos from the current system. Runtime behaviour is
455 # confirmed via the ktest (static ccache and keytab) environment
459 ("random-sleep", random_sleep(300, 900)),
460 ("configure", "./configure.developer --without-ad-dc --with-system-mitkrb5 " + samba_configure_params
),
462 ("test", make_test(include_envs
=[
463 "ktest", # ktest is also tested in fileserver, samba and
464 # samba-mitkrb5 but is tested here against a
468 ("check-clean-tree", CLEAN_SOURCE_TREE_CMD
),
473 "dependency": "samba-def-build",
475 ("random-sleep", random_sleep(300, 900)),
476 ("test", make_test(include_envs
=[
478 "ad_member_idmap_rid",
479 "ad_member_idmap_ad",
481 "ad_member_offlogon",
484 ("check-clean-tree", CLEAN_SOURCE_TREE_CMD
),
489 "dependency": "samba-no-opath-build",
491 ("random-sleep", random_sleep(300, 900)),
493 cmd
="make testonly DISABLE_OPATH=1",
503 ("check-clean-tree", "script/clean-source-tree.sh"),
508 "dependency": "samba-no-opath-build",
510 ("random-sleep", random_sleep(300, 900)),
512 cmd
="make testonly DISABLE_OPATH=1",
516 "fileserver_smb1_done",
519 ("check-clean-tree", "script/clean-source-tree.sh"),
524 "dependency": "samba-def-build",
526 ("random-sleep", random_sleep(1, 1)),
527 ("test", make_test(include_envs
=[
535 ("check-clean-tree", CLEAN_SOURCE_TREE_CMD
),
540 "dependency": "samba-def-build",
542 ("random-sleep", random_sleep(1, 1)),
543 ("test", make_test(include_envs
=[
549 ("check-clean-tree", CLEAN_SOURCE_TREE_CMD
),
554 "dependency": "samba-def-build",
556 ("random-sleep", random_sleep(1, 1)),
557 ("test", make_test(include_envs
=[
564 ("check-clean-tree", CLEAN_SOURCE_TREE_CMD
),
569 "dependency": "samba-def-build",
571 ("random-sleep", random_sleep(1, 1)),
572 ("test", make_test(include_envs
=[
578 ("check-clean-tree", CLEAN_SOURCE_TREE_CMD
),
582 "dependency": "samba-def-build",
584 ("random-sleep", random_sleep(1, 1)),
585 ("test", make_test(include_envs
=[
590 ("check-clean-tree", CLEAN_SOURCE_TREE_CMD
),
595 "dependency": "samba-def-build",
597 ("random-sleep", random_sleep(1, 1)),
598 ("test", make_test(include_envs
=[
599 "ad_dc_default", "ad_dc_default_smb1", "ad_dc_default_smb1_done"])),
601 ("check-clean-tree", CLEAN_SOURCE_TREE_CMD
),
606 "dependency": "samba-def-build",
608 ("random-sleep", random_sleep(1, 1)),
609 ("test", make_test(include_envs
=["ad_dc_slowtests", "ad_dc_backup"])),
611 ("check-clean-tree", CLEAN_SOURCE_TREE_CMD
),
615 "samba-schemaupgrade": {
616 "dependency": "samba-def-build",
618 ("random-sleep", random_sleep(1, 1)),
619 ("test", make_test(include_envs
=["schema_dc", "schema_pair_dc"])),
621 ("check-clean-tree", CLEAN_SOURCE_TREE_CMD
),
625 # We split out the ad_dc_ntvfs tests (which are long) so other test do not wait
626 # This is currently the longest task, so we don't randomly delay it.
627 "samba-ad-dc-ntvfs": {
628 "dependency": "samba-def-build",
630 ("random-sleep", random_sleep(1, 1)),
631 ("test", make_test(include_envs
=["ad_dc_ntvfs"])),
633 ("check-clean-tree", CLEAN_SOURCE_TREE_CMD
),
637 # Test fips compliance
639 "dependency": "samba-mit-build",
641 ("random-sleep", random_sleep(1, 1)),
642 ("test", make_test(include_envs
=["ad_dc_fips", "ad_member_fips"])),
643 # TODO: This seems to generate only an empty samba-fips.info ("lcov", LCOV_CMD),
644 ("check-clean-tree", CLEAN_SOURCE_TREE_CMD
),
648 # run the backup/restore testenvs separately as they're fairly standalone
649 # (and CI seems to max out at ~3 different DCs running at once)
651 "dependency": "samba-def-build",
653 ("random-sleep", random_sleep(300, 900)),
654 ("test", make_test(include_envs
=[
660 ("check-clean-tree", CLEAN_SOURCE_TREE_CMD
),
664 "dependency": "samba-def-build",
666 ("random-sleep", random_sleep(300, 900)),
667 ("test", make_test(include_envs
=[
673 ("check-clean-tree", CLEAN_SOURCE_TREE_CMD
),
678 "dependency": "samba-mit-build",
680 ("random-sleep", random_sleep(1, 1)),
681 ("test", make_test(include_envs
=[
683 "ad_member_idmap_rid",
684 "ad_member_idmap_ad",
686 "ad_member_offlogon",
689 ("check-clean-tree", CLEAN_SOURCE_TREE_CMD
),
693 "samba-addc-mit-1": {
694 "dependency": "samba-mit-build",
696 ("random-sleep", random_sleep(1, 1)),
697 ("test", make_test(include_envs
=[
705 ("check-clean-tree", CLEAN_SOURCE_TREE_CMD
),
709 "samba-addc-mit-4a": {
710 "dependency": "samba-mit-build",
712 ("random-sleep", random_sleep(1, 1)),
713 ("test", make_test(include_envs
=[
719 ("check-clean-tree", CLEAN_SOURCE_TREE_CMD
),
722 "samba-addc-mit-4b": {
723 "dependency": "samba-mit-build",
725 ("random-sleep", random_sleep(1, 1)),
726 ("test", make_test(include_envs
=[
731 ("check-clean-tree", CLEAN_SOURCE_TREE_CMD
),
737 ("configure", "./configure.developer --abi-check-disable" + samba_configure_params
),
739 ("test", make_test(TESTS
="${TESTS}")),
744 # Test cross-compile infrastructure
747 ("random-sleep", random_sleep(900, 1500)),
748 ("configure-native", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params
),
749 ("configure-cross-execute", "./configure.developer --out ./bin-xe --cross-compile --cross-execute=script/identity_cc.sh" \
750 " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xe/ab" + samba_configure_params
),
751 ("verify-cross-execute-output", "grep '^Checking value of NSIG' ./bin-xe/cross-answers.txt"),
752 ("configure-cross-answers", "./configure.developer --out ./bin-xa --cross-compile" \
753 " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xa/ab" + samba_configure_params
),
754 ("compare-results", "script/compare_cc_results.py "
755 "./bin/c4che/default{} "
756 "./bin-xe/c4che/default{} "
757 "./bin-xa/c4che/default{}".format(*([CACHE_SUFFIX
]*3))),
758 ("modify-cross-answers", "sed -i.bak -e 's/^\\(Checking value of NSIG:\\) .*/\\1 \"1234\"/' ./bin-xe/cross-answers.txt"),
759 ("configure-cross-answers-modified", "./configure.developer --out ./bin-xa2 --cross-compile" \
760 " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xa2/ab" + samba_configure_params
),
761 ("verify-cross-answers", "test $(sed -n -e 's/VALUEOF_NSIG = \\(.*\\)/\\1/p' ./bin-xa2/c4che/default{})" \
762 " = \"'1234'\"".format(CACHE_SUFFIX
)),
763 ("invalidate-cross-answers", "sed -i.bak -e '/^Checking value of NSIG/d' ./bin-xe/cross-answers.txt"),
764 ("configure-cross-answers-fail", "./configure.developer --out ./bin-xa3 --cross-compile" \
765 " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xa3/ab" + samba_configure_params
+ \
770 # test build with -O3 -- catches extra warnings and bugs, tests the ad_dc environments
773 ("random-sleep", random_sleep(300, 900)),
774 ("configure", "ADDITIONAL_CFLAGS='-O3 -Wp,-D_FORTIFY_SOURCE=2' ./configure.developer --abi-check-disable" + samba_configure_params
),
776 ("test", make_test(cmd
='make test', TESTS
="--exclude=selftest/slow-none", include_envs
=["none"])),
777 ("quicktest", make_test(cmd
='make quicktest', include_envs
=["ad_dc", "ad_dc_smb1", "ad_dc_smb1_done"])),
779 ("install", "make install"),
780 ("check-clean-tree", CLEAN_SOURCE_TREE_CMD
),
781 ("clean", "make clean"),
787 ("random-sleep", random_sleep(900, 1500)),
789 # make sure we have tdb around:
790 ("tdb-configure", "cd lib/tdb && PYTHONPATH=${PYTHON_PREFIX}:$PYTHONPATH PKG_CONFIG_PATH=$PKG_CONFIG_PATH:${PREFIX_DIR}/lib/pkgconfig ./configure --bundled-libraries=NONE --abi-check --enable-debug -C ${PREFIX}"),
791 ("tdb-make", "cd lib/tdb && make"),
792 ("tdb-install", "cd lib/tdb && make install"),
794 # build samba with cluster support (also building ctdb):
796 "PYTHONPATH=${PYTHON_PREFIX}:$PYTHONPATH "
797 "PKG_CONFIG_PATH=${PREFIX_DIR}/lib/pkgconfig:${PKG_CONFIG_PATH} "
798 "./configure.developer ${PREFIX} "
799 "--with-selftest-prefix=./bin/ab "
800 "--with-cluster-support "
802 "--bundled-libraries=!tdb"),
803 ("samba-make", "make"),
804 ("samba-check", "./bin/smbd --configfile=/dev/null -b | grep CLUSTER_SUPPORT"),
805 ("samba-install", "make install"),
806 ("ctdb-check", "test -e ${PREFIX_DIR}/sbin/ctdbd"),
810 INJECT_SELFTEST_PREFIX
=0,
811 include_envs
=["clusteredmember"])
815 ("check-clean-tree", CLEAN_SOURCE_TREE_CMD
),
816 ("clean", "make clean"),
817 ("ctdb-clean", "cd ./ctdb && make clean"),
823 ("random-sleep", random_sleep(300, 900)),
824 ("talloc-configure", "cd lib/talloc && " + samba_libs_configure_libs
),
825 ("talloc-make", "cd lib/talloc && make"),
826 ("talloc-install", "cd lib/talloc && make install"),
828 ("tdb-configure", "cd lib/tdb && " + samba_libs_configure_libs
),
829 ("tdb-make", "cd lib/tdb && make"),
830 ("tdb-install", "cd lib/tdb && make install"),
832 ("tevent-configure", "cd lib/tevent && " + samba_libs_configure_libs
),
833 ("tevent-make", "cd lib/tevent && make"),
834 ("tevent-install", "cd lib/tevent && make install"),
836 ("ldb-configure", "cd lib/ldb && " + samba_libs_configure_libs
),
837 ("ldb-make", "cd lib/ldb && make"),
838 ("ldb-install", "cd lib/ldb && make install"),
840 ("nondevel-configure", samba_libs_envvars
+ " ./configure ${PREFIX}"),
841 ("nondevel-make", "make -j"),
842 ("nondevel-check", "./bin/smbd -b | grep WITH_NTVFS_FILESERVER && exit 1; exit 0"),
843 ("nondevel-no-libtalloc", "find ./bin | grep -v 'libtalloc-report' | grep 'libtalloc' && exit 1; exit 0"),
844 ("nondevel-no-libtdb", "find ./bin | grep -v 'libtdb-wrap' | grep 'libtdb' && exit 1; exit 0"),
845 ("nondevel-no-libtevent", "find ./bin | grep -v 'libtevent-util' | grep 'libtevent' && exit 1; exit 0"),
846 ("nondevel-no-libldb", "find ./bin | grep -v 'module' | grep -v 'libldbsamba' | grep 'libldb' && exit 1; exit 0"),
847 ("nondevel-no-samba-nss_winbind", "ldd ./bin/plugins/libnss_winbind.so.2 | grep 'samba' && exit 1; exit 0"),
848 ("nondevel-no-samba-nss_wins", "ldd ./bin/plugins/libnss_wins.so.2 | grep 'samba' && exit 1; exit 0"),
849 ("nondevel-no-samba-libwbclient", "ldd ./bin/shared/libwbclient.so.0 | grep 'samba' && exit 1; exit 0"),
850 ("nondevel-no-samba-pam_winbind", "ldd ./bin/plugins/pam_winbind.so | grep -v 'libtalloc.so.2' | grep 'samba' && exit 1; exit 0"),
851 ("nondevel-no-public-nss_winbind",
852 nm_grep_symbols("./bin/plugins/libnss_winbind.so.2", " T _nss_winbind_")),
853 ("nondevel-no-public-nss_wins",
854 nm_grep_symbols("./bin/plugins/libnss_wins.so.2", " T _nss_wins_")),
855 ("nondevel-no-public-libwbclient",
856 nm_grep_symbols("./bin/shared/libwbclient.so.0", " T wbc")),
857 ("nondevel-no-public-pam_winbind",
858 nm_grep_symbols("./bin/plugins/pam_winbind.so", "T pam_sm_")),
859 ("nondevel-no-public-winbind_krb5_locator",
860 nm_grep_symbols("./bin/plugins/winbind_krb5_locator.so", " D resolve\>")),
861 ("nondevel-no-public-async_dns_krb5_locator",
862 nm_grep_symbols("./bin/plugins/async_dns_krb5_locator.so", " D resolve\>")),
863 ("nondevel-install", "make -j install"),
864 ("nondevel-dist", "make dist"),
866 ("prefix-no-private-libtalloc", "find ${PREFIX_DIR} | grep -v 'libtalloc-report' | grep 'private.*libtalloc' && exit 1; exit 0"),
867 ("prefix-no-private-libtdb", "find ${PREFIX_DIR} | grep -v 'libtdb-wrap' | grep 'private.*libtdb' && exit 1; exit 0"),
868 ("prefix-no-private-libtevent", "find ${PREFIX_DIR} | grep -v 'libtevent-util' | grep 'private.*libtevent' && exit 1; exit 0"),
869 ("prefix-no-private-libldb", "find ${PREFIX_DIR} | grep -v 'module' | grep -v 'libldbsamba' | grep 'private.*libldb' && exit 1; exit 0"),
870 ("prefix-no-samba-nss_winbind", "ldd ${PREFIX_DIR}/lib/libnss_winbind.so.2 | grep 'samba' && exit 1; exit 0"),
871 ("prefix-no-samba-nss_wins", "ldd ${PREFIX_DIR}/lib/libnss_wins.so.2 | grep 'samba' && exit 1; exit 0"),
872 ("prefix-no-samba-libwbclient", "ldd ${PREFIX_DIR}/lib/libwbclient.so.0 | grep 'samba' && exit 1; exit 0"),
873 ("prefix-no-samba-pam_winbind", "ldd ${PREFIX_DIR}/lib/security/pam_winbind.so | grep -v 'libtalloc.so.2' | grep 'samba' && exit 1; exit 0"),
874 ("prefix-no-public-nss_winbind",
875 nm_grep_symbols("${PREFIX_DIR}/lib/libnss_winbind.so.2", " T _nss_winbind_")),
876 ("prefix-no-public-nss_wins",
877 nm_grep_symbols("${PREFIX_DIR}/lib/libnss_wins.so.2", " T _nss_wins_")),
878 ("prefix-no-public-libwbclient",
879 nm_grep_symbols("${PREFIX_DIR}/lib/libwbclient.so.0", " T wbc")),
880 ("prefix-no-public-pam_winbind",
881 nm_grep_symbols("${PREFIX_DIR}/lib/security/pam_winbind.so", "T pam_sm_")),
882 ("prefix-no-public-winbind_krb5_locator",
883 nm_grep_symbols("${PREFIX_DIR}/lib/krb5/winbind_krb5_locator.so", " D resolve\>")),
884 ("prefix-no-public-async_dns_krb5_locator",
885 nm_grep_symbols("${PREFIX_DIR}/lib/krb5/async_dns_krb5_locator.so", " D resolve\>")),
887 # retry with all modules shared
888 ("allshared-distclean", "make distclean"),
889 ("allshared-configure", samba_libs_configure_samba
+ " --with-shared-modules=ALL"),
890 ("allshared-make", "make -j"),
891 ("allshared-no-libtalloc", "find ./bin | grep -v 'libtalloc-report' | grep 'libtalloc' && exit 1; exit 0"),
892 ("allshared-no-libtdb", "find ./bin | grep -v 'libtdb-wrap' | grep 'libtdb' && exit 1; exit 0"),
893 ("allshared-no-libtevent", "find ./bin | grep -v 'libtevent-util' | grep 'libtevent' && exit 1; exit 0"),
894 ("allshared-no-libldb", "find ./bin | grep -v 'module' | grep -v 'libldbsamba' | grep 'libldb' && exit 1; exit 0"),
895 ("allshared-no-samba-nss_winbind", "ldd ./bin/plugins/libnss_winbind.so.2 | grep 'samba' && exit 1; exit 0"),
896 ("allshared-no-samba-nss_wins", "ldd ./bin/plugins/libnss_wins.so.2 | grep 'samba' && exit 1; exit 0"),
897 ("allshared-no-samba-libwbclient", "ldd ./bin/shared/libwbclient.so.0 | grep 'samba' && exit 1; exit 0"),
898 ("allshared-no-samba-pam_winbind", "ldd ./bin/plugins/pam_winbind.so | grep -v 'libtalloc.so.2' | grep 'samba' && exit 1; exit 0"),
899 ("allshared-no-public-nss_winbind",
900 nm_grep_symbols("./bin/plugins/libnss_winbind.so.2", " T _nss_winbind_")),
901 ("allshared-no-public-nss_wins",
902 nm_grep_symbols("./bin/plugins/libnss_wins.so.2", " T _nss_wins_")),
903 ("allshared-no-public-libwbclient",
904 nm_grep_symbols("./bin/shared/libwbclient.so.0", " T wbc")),
905 ("allshared-no-public-pam_winbind",
906 nm_grep_symbols("./bin/plugins/pam_winbind.so", "T pam_sm_")),
907 ("allshared-no-public-winbind_krb5_locator",
908 nm_grep_symbols("./bin/plugins/winbind_krb5_locator.so", " D resolve\>")),
909 ("allshared-no-public-async_dns_krb5_locator",
910 nm_grep_symbols("./bin/plugins/async_dns_krb5_locator.so", " D resolve\>")),
916 # build the fuzzers (static) via the oss-fuzz script
917 ("fuzzers-mkdir-prefix", "mkdir -p ${PREFIX_DIR}"),
918 ("fuzzers-build", "OUT=${PREFIX_DIR} LIB_FUZZING_ENGINE= SANITIZER=address CXX= CFLAGS= ADDITIONAL_LDFLAGS='-fuse-ld=bfd' ./lib/fuzzing/oss-fuzz/build_samba.sh --enable-afl-fuzzer"),
922 # * Test smbd and smbtorture can build semi-static
924 # * Test Samba without python still builds.
926 # When this test fails due to more use of Python, the expectations
927 # is that the newly failing part of the code should be disabled
928 # when --disable-python is set (rather than major work being done
929 # to support this environment).
931 # The target here is for vendors shipping a minimal smbd.
932 "samba-minimal-smbd": {
934 ("random-sleep", random_sleep(300, 900)),
936 # build with all modules static
937 ("allstatic-configure", "./configure.developer " + samba_configure_params
+ " --with-static-modules=ALL"),
938 ("allstatic-make", "make -j"),
939 ("allstatic-test", make_test(TESTS
="samba3.smb2.create.*nt4_dc")),
940 ("allstatic-lcov", LCOV_CMD
),
942 # retry with nonshared smbd and smbtorture
943 ("nonshared-distclean", "make distclean"),
944 ("nonshared-configure", "./configure.developer " + samba_configure_params
+ " --bundled-libraries=ALL --with-static-modules=ALL --nonshared-binary=smbtorture,smbd/smbd"),
945 ("nonshared-make", "make -j"),
946 # TODO ("nonshared-test", make_test(TESTS="samba3.smb2.create.*nt4_dc")),
947 # TODO ("nonshared-lcov", LCOV_CMD),
949 ("check-clean-tree", CLEAN_SOURCE_TREE_CMD
),
950 ("clean", "make clean"),
956 ("random-sleep", random_sleep(300, 900)),
958 ("configure", "./configure.developer ${ENABLE_COVERAGE} ${PREFIX} --with-profiling-data --disable-python --without-ad-dc"),
960 ("find-python", "script/find_python.sh ${PREFIX}"),
961 ("test", "make test-nopython"),
963 ("check-clean-tree", CLEAN_SOURCE_TREE_CMD
),
964 ("clean", "make clean"),
966 ("talloc-configure", "cd lib/talloc && " + samba_libs_configure_base
+ " --bundled-libraries=cmocka,NONE --disable-python"),
967 ("talloc-make", "cd lib/talloc && make"),
968 ("talloc-install", "cd lib/talloc && make install"),
970 ("tdb-configure", "cd lib/tdb && " + samba_libs_configure_base
+ " --bundled-libraries=cmocka,NONE --disable-python"),
971 ("tdb-make", "cd lib/tdb && make"),
972 ("tdb-install", "cd lib/tdb && make install"),
974 ("tevent-configure", "cd lib/tevent && " + samba_libs_configure_base
+ " --bundled-libraries=cmocka,NONE --disable-python"),
975 ("tevent-make", "cd lib/tevent && make"),
976 ("tevent-install", "cd lib/tevent && make install"),
978 ("ldb-configure", "cd lib/ldb && " + samba_libs_configure_base
+ " --bundled-libraries=cmocka,NONE --disable-python"),
979 ("ldb-make", "cd lib/ldb && make"),
980 ("ldb-install", "cd lib/ldb && make install"),
982 # retry against installed library packages, but no required modules
983 ("libs-configure", samba_libs_configure_base
+ samba_libs_configure_bundled_libs
+ " --disable-python --without-ad-dc --with-static-modules=!FORCED,!DEFAULT --with-shared-modules=!FORCED,!DEFAULT"),
984 ("libs-make", "make -j"),
985 ("libs-install", "make install"),
986 ("libs-check-clean-tree", CLEAN_SOURCE_TREE_CMD
),
987 ("libs-clean", "make clean"),
994 ("random-sleep", random_sleep(60, 600)),
995 ("configure", "./configure ${ENABLE_COVERAGE} --enable-developer -C ${PREFIX}"),
997 ("install", "make install"),
998 ("test", "make test"),
1000 ("clean", "make clean"),
1001 ("configure-no-lmdb", "./configure ${ENABLE_COVERAGE} --enable-developer --without-ldb-lmdb -C ${PREFIX}"),
1002 ("make-no-lmdb", "make"),
1003 ("test-no-lmdb", "make test"),
1004 ("lcov-no-lmdb", LCOV_CMD
),
1005 ("install-no-lmdb", "make install"),
1006 ("check-clean-tree", CLEAN_SOURCE_TREE_CMD
),
1007 ("distcheck", "make distcheck"),
1008 ("clean", "make clean"),
1014 ("random-sleep", random_sleep(60, 600)),
1015 ("configure", "./configure ${ENABLE_COVERAGE} --enable-developer -C ${PREFIX}"),
1017 ("install", "make install"),
1018 ("test", "make test"),
1020 ("check-clean-tree", CLEAN_SOURCE_TREE_CMD
),
1021 ("distcheck", "make distcheck"),
1022 ("clean", "make clean"),
1028 ("random-sleep", random_sleep(60, 600)),
1029 ("configure", "./configure ${ENABLE_COVERAGE} --enable-developer -C ${PREFIX}"),
1031 ("install", "make install"),
1032 ("test", "make test"),
1034 ("check-clean-tree", CLEAN_SOURCE_TREE_CMD
),
1035 ("distcheck", "make distcheck"),
1036 ("clean", "make clean"),
1042 ("random-sleep", random_sleep(60, 600)),
1043 ("configure", "./configure ${ENABLE_COVERAGE} --enable-developer -C ${PREFIX}"),
1045 ("install", "make install"),
1046 ("test", "make test"),
1048 ("check-clean-tree", CLEAN_SOURCE_TREE_CMD
),
1049 ("distcheck", "make distcheck"),
1050 ("clean", "make clean"),
1056 ("random-sleep", random_sleep(60, 600)),
1057 ("configure", "./configure ${ENABLE_COVERAGE} --enable-developer -C ${PREFIX}"),
1059 ("install", "make install"),
1060 ("test", "make test"),
1062 ("check-clean-tree", CLEAN_SOURCE_TREE_CMD
),
1063 ("distcheck", "make distcheck"),
1064 ("clean", "make clean"),
1069 "git-clone-required": True,
1071 ("random-sleep", random_sleep(60, 600)),
1072 ("configure", "perl Makefile.PL PREFIX=${PREFIX_DIR}"),
1073 ("touch", "touch *.yp"),
1075 ("test", "make test"),
1076 ("install", "make install"),
1077 ("checkout-yapp-generated", "git checkout lib/Parse/Pidl/IDL.pm lib/Parse/Pidl/Expr.pm"),
1078 ("check-clean-tree", CLEAN_SOURCE_TREE_CMD
),
1079 ("clean", "make clean"),
1083 # these are useful for debugging autobuild
1086 ("pass", 'echo passing && /bin/true'),
1091 ("fail", 'echo failing && /bin/false'),
1096 defaulttasks
= list(tasks
.keys())
1098 defaulttasks
.remove("pass")
1099 defaulttasks
.remove("fail")
1101 # The build tasks will be brought in by the test tasks as needed
1102 defaulttasks
.remove("samba-def-build")
1103 defaulttasks
.remove("samba-nt4-build")
1104 defaulttasks
.remove("samba-mit-build")
1105 defaulttasks
.remove("samba-h5l-build")
1106 defaulttasks
.remove("samba-no-opath-build")
1108 # This is not a normal test, but a task to support manually running
1109 # one test under autobuild
1110 defaulttasks
.remove("samba-test-only")
1112 # Only built on GitLab CI and not in the default autobuild because it
1113 # uses too much space (4GB of semi-static binaries)
1114 defaulttasks
.remove("samba-fuzz")
1116 # The FIPS build runs only in GitLab CI on a current Fedora Docker
1117 # container where a simulated FIPS mode is possible.
1118 defaulttasks
.remove("samba-fips")
1120 # The MIT build runs on a current Fedora where an up to date MIT KDC
1121 # is already packaged. This avoids needing to backport a current MIT
1122 # to the default Ubuntu 18.04, particularly during development, and
1123 # the need to install on the shared sn-devel-184.
1125 defaulttasks
.remove("samba-mitkrb5")
1126 defaulttasks
.remove("samba-admem-mit")
1127 defaulttasks
.remove("samba-addc-mit-1")
1128 defaulttasks
.remove("samba-addc-mit-4a")
1129 defaulttasks
.remove("samba-addc-mit-4b")
1131 if os
.environ
.get("AUTOBUILD_SKIP_SAMBA_O3", "0") == "1":
1132 defaulttasks
.remove("samba-o3")
1141 def run_cmd(cmd
, dir=".", show
=None, output
=False, checkfail
=True):
1143 show
= options
.verbose
1145 do_print("Running: '%s' in '%s'" % (cmd
, dir))
1147 out
= check_output([cmd
], shell
=True, cwd
=dir)
1148 return out
.decode(encoding
='utf-8', errors
='backslashreplace')
1150 return check_call(cmd
, shell
=True, cwd
=dir)
1152 return call(cmd
, shell
=True, cwd
=dir)
1154 def rmdir_force(dirname
, re_raise
=True):
1156 run_cmd("test -d %s && chmod -R +w %s; rm -rf %s" % (
1157 dirname
, dirname
, dirname
), output
=True, show
=True)
1158 except CalledProcessError
as e
:
1159 do_print("Failed: '%s'" % (str(e
)))
1160 run_cmd("tree %s" % dirname
, output
=True, show
=True)
1166 class builder(object):
1167 '''handle build of one directory'''
1169 def __init__(self
, name
, definition
):
1171 self
.dir = builddirs
.get(name
, '.')
1172 self
.tag
= self
.name
.replace('/', '_')
1173 self
.definition
= definition
1174 self
.sequence
= definition
["sequence"]
1175 self
.git_clone_required
= False
1176 if "git-clone-required" in definition
:
1177 self
.git_clone_required
= bool(definition
["git-clone-required"])
1181 self
.stdout_path
= "%s/%s.stdout" % (gitroot
, self
.tag
)
1182 self
.stderr_path
= "%s/%s.stderr" % (gitroot
, self
.tag
)
1184 do_print("stdout for %s in %s" % (self
.name
, self
.stdout_path
))
1185 do_print("stderr for %s in %s" % (self
.name
, self
.stderr_path
))
1186 run_cmd("rm -f %s %s" % (self
.stdout_path
, self
.stderr_path
))
1187 self
.stdout
= open(self
.stdout_path
, 'w')
1188 self
.stderr
= open(self
.stderr_path
, 'w')
1189 self
.stdin
= open("/dev/null", 'r')
1190 self
.builder_dir
= "%s/%s" % (testbase
, self
.tag
)
1191 self
.test_source_dir
= self
.builder_dir
1192 self
.cwd
= "%s/%s" % (self
.builder_dir
, self
.dir)
1193 self
.selftest_prefix
= "%s/bin/ab" % (self
.cwd
)
1194 self
.prefix
= "%s/%s" % (test_prefix
, self
.tag
)
1196 self
.producer
= None
1198 if self
.git_clone_required
:
1199 assert "dependency" not in definition
1201 def mark_existing(self
):
1202 do_print('%s: Mark as existing dependency' % self
.name
)
1203 self
.next
= len(self
.sequence
)
1206 def add_consumer(self
, consumer
):
1207 do_print("%s: add consumer: %s" % (self
.name
, consumer
.name
))
1208 consumer
.producer
= self
1209 consumer
.test_source_dir
= self
.test_source_dir
1210 self
.consumers
.append(consumer
)
1212 def start_next(self
):
1213 if self
.producer
is not None:
1214 if not self
.producer
.done
:
1215 do_print("%s: Waiting for producer: %s" % (self
.name
, self
.producer
.name
))
1219 rmdir_force(self
.builder_dir
)
1220 rmdir_force(self
.prefix
)
1221 if self
.producer
is not None:
1222 run_cmd("mkdir %s" % (self
.builder_dir
), dir=test_master
, show
=True)
1223 elif not self
.git_clone_required
:
1224 run_cmd("cp -R -a -l %s %s" % (test_master
, self
.builder_dir
), dir=test_master
, show
=True)
1226 run_cmd("git clone --recursive --shared %s %s" % (test_master
, self
.builder_dir
), dir=test_master
, show
=True)
1228 if self
.next
== len(self
.sequence
):
1230 do_print('%s: Completed OK' % self
.name
)
1232 if not options
.nocleanup
and len(self
.consumers
) == 0:
1233 do_print('%s: Cleaning up' % self
.name
)
1234 rmdir_force(self
.builder_dir
)
1235 rmdir_force(self
.prefix
)
1236 for consumer
in self
.consumers
:
1237 if consumer
.next
!= 0:
1239 do_print('%s: Starting consumer %s' % (self
.name
, consumer
.name
))
1240 consumer
.start_next()
1241 if self
.producer
is not None:
1242 self
.producer
.consumers
.remove(self
)
1243 assert self
.producer
.done
1244 self
.producer
.start_next()
1245 do_print('%s: Remaining consumers %u' % (self
.name
, len(self
.consumers
)))
1247 (self
.stage
, self
.cmd
) = self
.sequence
[self
.next
]
1248 self
.cmd
= self
.cmd
.replace("${PYTHON_PREFIX}", get_python_lib(plat_specific
=1, standard_lib
=0, prefix
=self
.prefix
))
1249 self
.cmd
= self
.cmd
.replace("${PREFIX}", "--prefix=%s" % self
.prefix
)
1250 self
.cmd
= self
.cmd
.replace("${PREFIX_DIR}", "%s" % self
.prefix
)
1251 self
.cmd
= self
.cmd
.replace("${TESTS}", options
.restrict_tests
)
1252 self
.cmd
= self
.cmd
.replace("${TEST_SOURCE_DIR}", self
.test_source_dir
)
1253 self
.cmd
= self
.cmd
.replace("${SELFTEST_PREFIX}", self
.selftest_prefix
)
1254 self
.cmd
= self
.cmd
.replace("${LOG_BASE}", options
.log_base
)
1255 self
.cmd
= self
.cmd
.replace("${NAME}", self
.name
)
1256 self
.cmd
= self
.cmd
.replace("${ENABLE_COVERAGE}", options
.enable_coverage
)
1257 do_print('%s: [%s] Running %s in %r' % (self
.name
, self
.stage
, self
.cmd
, self
.cwd
))
1258 self
.proc
= Popen(self
.cmd
, shell
=True,
1259 close_fds
=True, cwd
=self
.cwd
,
1260 stdout
=self
.stdout
, stderr
=self
.stderr
, stdin
=self
.stdin
)
1263 def expand_dependencies(n
):
1265 if "dependency" in tasks
[n
]:
1266 depname
= tasks
[n
]["dependency"]
1267 assert depname
in tasks
1268 sdeps
= expand_dependencies(depname
)
1269 assert n
not in sdeps
1272 deps
.append(depname
)
1276 class buildlist(object):
1277 '''handle build of multiple directories'''
1279 def __init__(self
, tasknames
, rebase_url
, rebase_branch
="master"):
1280 self
.tail_proc
= None
1283 if options
.restrict_tests
:
1284 tasknames
= ["samba-test-only"]
1286 tasknames
= defaulttasks
1288 given_tasknames
= tasknames
.copy()
1289 implicit_tasknames
= []
1290 for n
in given_tasknames
:
1291 deps
= expand_dependencies(n
)
1293 if dep
in given_tasknames
:
1295 if dep
in implicit_tasknames
:
1297 implicit_tasknames
.append(dep
)
1299 tasknames
= implicit_tasknames
.copy()
1300 tasknames
.extend(given_tasknames
)
1301 do_print("given_tasknames: %s" % given_tasknames
)
1302 do_print("implicit_tasknames: %s" % implicit_tasknames
)
1303 do_print("tasknames: %s" % tasknames
)
1304 self
.tlist
= [builder(n
, tasks
[n
]) for n
in tasknames
]
1307 rebase_remote
= "rebaseon"
1309 "git-clone-required": True,
1313 git remote add -t %s %s %s
1317 git describe %s/%s > old_remote_branch.desc
1319 git describe %s/%s > remote_branch.desc
1320 diff old_remote_branch.desc remote_branch.desc
1323 rebase_branch
, rebase_remote
, rebase_url
,
1325 rebase_remote
, rebase_branch
,
1327 rebase_remote
, rebase_branch
1330 self
.retry
= builder('retry', retry_task
)
1331 self
.need_retry
= False
1333 if options
.skip_dependencies
:
1334 for b
in self
.tlist
:
1335 if b
.name
in implicit_tasknames
:
1338 for b
in self
.tlist
:
1339 do_print("b.name=%s" % b
.name
)
1340 if "dependency" not in b
.definition
:
1342 depname
= b
.definition
["dependency"]
1343 do_print("b.name=%s: dependency:%s" % (b
.name
, depname
))
1344 for p
in self
.tlist
:
1345 if p
.name
== depname
:
1348 def kill_kids(self
):
1349 if self
.tail_proc
is not None:
1350 self
.tail_proc
.terminate()
1351 self
.tail_proc
.wait()
1352 self
.tail_proc
= None
1353 if self
.retry
is not None:
1354 self
.retry
.proc
.terminate()
1355 self
.retry
.proc
.wait()
1357 for b
in self
.tlist
:
1358 if b
.proc
is not None:
1359 run_cmd("killbysubdir %s > /dev/null 2>&1" % b
.test_source_dir
, checkfail
=False)
1367 for b
in self
.tlist
:
1370 none_running
= False
1371 b
.status
= b
.proc
.poll()
1372 if b
.status
is None:
1377 ret
= self
.retry
.proc
.poll()
1379 self
.need_retry
= True
1387 for b
in self
.tlist
:
1390 self
.retry
.start_next()
1393 if options
.retry
and self
.need_retry
:
1395 do_print("retry needed")
1396 return (0, None, None, None, "retry")
1399 if os
.WIFSIGNALED(b
.status
) or os
.WEXITSTATUS(b
.status
) != 0:
1401 return (b
.status
, b
.name
, b
.stage
, b
.tag
, "%s: [%s] failed '%s' with status %d" % (b
.name
, b
.stage
, b
.cmd
, b
.status
))
1404 return (0, None, None, None, "All OK")
1406 def write_system_info(self
, filename
):
1407 with
open(filename
, 'w') as f
:
1408 for cmd
in ['uname -a',
1412 'cat /proc/cpuinfo',
1415 'df -m %s' % testbase
]:
1417 out
= run_cmd(cmd
, output
=True, checkfail
=False)
1418 except CalledProcessError
as e
:
1419 out
= "<failed: %s>" % str(e
)
1420 print('### %s' % cmd
, file=f
)
1424 def tarlogs(self
, fname
):
1425 with tarfile
.open(fname
, "w:gz") as tar
:
1426 for b
in self
.tlist
:
1427 tar
.add(b
.stdout_path
, arcname
="%s.stdout" % b
.tag
)
1428 tar
.add(b
.stderr_path
, arcname
="%s.stderr" % b
.tag
)
1429 if os
.path
.exists("autobuild.log"):
1430 tar
.add("autobuild.log")
1431 filename
= 'system-info.txt'
1432 self
.write_system_info(filename
)
1435 def remove_logs(self
):
1436 for b
in self
.tlist
:
1437 os
.unlink(b
.stdout_path
)
1438 os
.unlink(b
.stderr_path
)
1440 def start_tail(self
):
1441 cmd
= ["tail", "-f"]
1442 for b
in self
.tlist
:
1443 cmd
.append(b
.stdout_path
)
1444 cmd
.append(b
.stderr_path
)
1445 self
.tail_proc
= Popen(cmd
, close_fds
=True)
1448 def cleanup(do_raise
=False):
1449 if options
.nocleanup
:
1451 run_cmd("stat %s || true" % test_tmpdir
, show
=True)
1452 run_cmd("stat %s" % testbase
, show
=True)
1453 do_print("Cleaning up %r" % cleanup_list
)
1454 for d
in cleanup_list
:
1455 ok
= rmdir_force(d
, re_raise
=False)
1458 if os
.path
.isdir(d
):
1459 do_print("Killing, waiting and retry")
1460 run_cmd("killbysubdir %s > /dev/null 2>&1" % d
, checkfail
=False)
1462 do_print("Waiting and retry")
1464 rmdir_force(d
, re_raise
=do_raise
)
1467 def daemonize(logfile
):
1469 if pid
== 0: # Parent
1472 if pid
!= 0: # Actual daemon
1477 import resource
# Resource usage information.
1478 maxfd
= resource
.getrlimit(resource
.RLIMIT_NOFILE
)[1]
1479 if maxfd
== resource
.RLIM_INFINITY
:
1480 maxfd
= 1024 # Rough guess at maximum number of open file descriptors.
1481 for fd
in range(0, maxfd
):
1486 os
.open(logfile
, os
.O_RDWR | os
.O_CREAT
)
1491 def write_pidfile(fname
):
1492 '''write a pid file, cleanup on exit'''
1493 with
open(fname
, mode
='w') as f
:
1494 f
.write("%u\n" % os
.getpid())
1497 def rebase_tree(rebase_url
, rebase_branch
="master"):
1498 rebase_remote
= "rebaseon"
1499 do_print("Rebasing on %s" % rebase_url
)
1500 run_cmd("git describe HEAD", show
=True, dir=test_master
)
1501 run_cmd("git remote add -t %s %s %s" %
1502 (rebase_branch
, rebase_remote
, rebase_url
),
1503 show
=True, dir=test_master
)
1504 run_cmd("git fetch %s" % rebase_remote
, show
=True, dir=test_master
)
1505 if options
.fix_whitespace
:
1506 run_cmd("git rebase --force-rebase --whitespace=fix %s/%s" %
1507 (rebase_remote
, rebase_branch
),
1508 show
=True, dir=test_master
)
1510 run_cmd("git rebase --force-rebase %s/%s" %
1511 (rebase_remote
, rebase_branch
),
1512 show
=True, dir=test_master
)
1513 diff
= run_cmd("git --no-pager diff HEAD %s/%s" %
1514 (rebase_remote
, rebase_branch
),
1515 dir=test_master
, output
=True)
1517 do_print("No differences between HEAD and %s/%s - exiting" %
1518 (rebase_remote
, rebase_branch
))
1520 run_cmd("git describe %s/%s" %
1521 (rebase_remote
, rebase_branch
),
1522 show
=True, dir=test_master
)
1523 run_cmd("git describe HEAD", show
=True, dir=test_master
)
1524 run_cmd("git --no-pager diff --stat HEAD %s/%s" %
1525 (rebase_remote
, rebase_branch
),
1526 show
=True, dir=test_master
)
1529 def push_to(push_url
, push_branch
="master"):
1530 push_remote
= "pushto"
1531 do_print("Pushing to %s" % push_url
)
1533 run_cmd("git config --replace-all core.editor script/commit_mark.sh", dir=test_master
)
1534 run_cmd("git commit --amend -c HEAD", dir=test_master
)
1535 # the notes method doesn't work yet, as metze hasn't allowed refs/notes/* in master
1536 # run_cmd("EDITOR=script/commit_mark.sh git notes edit HEAD", dir=test_master)
1537 run_cmd("git remote add -t %s %s %s" %
1538 (push_branch
, push_remote
, push_url
),
1539 show
=True, dir=test_master
)
1540 run_cmd("git push %s +HEAD:%s" %
1541 (push_remote
, push_branch
),
1542 show
=True, dir=test_master
)
1545 def send_email(subject
, text
, log_tar
):
1546 if options
.email
is None:
1547 do_print("not sending email because the recipient is not set")
1548 do_print("the text content would have been:\n\nSubject: %s\n\n%s" %
1551 outer
= MIMEMultipart()
1552 outer
['Subject'] = subject
1553 outer
['To'] = options
.email
1554 outer
['From'] = options
.email_from
1555 outer
['Date'] = email
.utils
.formatdate(localtime
=True)
1556 outer
.preamble
= 'Autobuild mails are now in MIME because we optionally attach the logs.\n'
1557 outer
.attach(MIMEText(text
, 'plain', 'utf-8'))
1558 if options
.attach_logs
:
1559 with
open(log_tar
, 'rb') as fp
:
1560 msg
= MIMEApplication(fp
.read(), 'gzip', email
.encoders
.encode_base64
)
1561 # Set the filename parameter
1562 msg
.add_header('Content-Disposition', 'attachment', filename
=os
.path
.basename(log_tar
))
1564 content
= outer
.as_string()
1565 s
= smtplib
.SMTP(options
.email_server
)
1566 email_user
= os
.getenv('SMTP_USERNAME')
1567 email_password
= os
.getenv('SMTP_PASSWORD')
1568 if email_user
is not None:
1570 s
.login(email_user
, email_password
)
1572 s
.sendmail(options
.email_from
, [options
.email
], content
)
1577 def email_failure(status
, failed_task
, failed_stage
, failed_tag
, errstr
,
1578 elapsed_time
, log_base
=None, add_log_tail
=True):
1579 '''send an email to options.email about the failure'''
1580 elapsed_minutes
= elapsed_time
/ 60.0
1581 if log_base
is None:
1586 Your autobuild on %s failed after %.1f minutes
1587 when trying to test %s with the following error:
1591 the autobuild has been abandoned. Please fix the error and resubmit.
1593 A summary of the autobuild process is here:
1596 ''' % (platform
.node(), elapsed_minutes
, failed_task
, errstr
, log_base
)
1598 if options
.restrict_tests
:
1600 The build was restricted to tests matching %s\n""" % options
.restrict_tests
1602 if failed_task
!= 'rebase':
1604 You can see logs of the failed task here:
1609 or you can get full logs of all tasks in this job here:
1613 The top commit for the tree that was built was:
1617 ''' % (log_base
, failed_tag
, log_base
, failed_tag
, log_base
, top_commit_msg
)
1620 f
= open("%s/%s.stdout" % (gitroot
, failed_tag
), 'r')
1621 lines
= f
.readlines()
1622 log_tail
= "".join(lines
[-50:])
1623 num_lines
= len(lines
)
1625 # Also include stderr (compile failures) if < 50 lines of stdout
1626 f
= open("%s/%s.stderr" % (gitroot
, failed_tag
), 'r')
1627 log_tail
+= "".join(f
.readlines()[-(50 - num_lines
):])
1630 The last 50 lines of log messages:
1636 logs
= os
.path
.join(gitroot
, 'logs.tar.gz')
1637 send_email('autobuild[%s] failure on %s for task %s during %s'
1638 % (options
.branch
, platform
.node(), failed_task
, failed_stage
),
1642 def email_success(elapsed_time
, log_base
=None):
1643 '''send an email to options.email about a successful build'''
1644 if log_base
is None:
1649 Your autobuild on %s has succeeded after %.1f minutes.
1651 ''' % (platform
.node(), elapsed_time
/ 60.)
1653 if options
.restrict_tests
:
1655 The build was restricted to tests matching %s\n""" % options
.restrict_tests
1657 if options
.keeplogs
:
1660 you can get full logs of all tasks in this job here:
1667 The top commit for the tree that was built was:
1670 ''' % top_commit_msg
1672 logs
= os
.path
.join(gitroot
, 'logs.tar.gz')
1673 send_email('autobuild[%s] success on %s' % (options
.branch
, platform
.node()),
1677 # get the top commit message, for emails
1678 top_commit_msg
= run_cmd("git log -1", dir=gitroot
, output
=True)
1681 if options
.skip_dependencies
:
1682 run_cmd("stat %s" % testbase
, dir=testbase
, output
=True)
1684 os
.makedirs(testbase
)
1685 except Exception as reason
:
1686 raise Exception("Unable to create %s : %s" % (testbase
, reason
))
1687 cleanup_list
.append(testbase
)
1690 logfile
= os
.path
.join(testbase
, "log")
1691 do_print("Forking into the background, writing progress to %s" % logfile
)
1694 write_pidfile(gitroot
+ "/autobuild.pid")
1696 start_time
= time
.time()
1700 run_cmd("rm -rf %s" % test_tmpdir
, show
=True)
1701 os
.makedirs(test_tmpdir
)
1702 # The waf uninstall code removes empty directories all the way
1703 # up the tree. Creating a file in test_tmpdir stops it from
1705 run_cmd("touch %s" % os
.path
.join(test_tmpdir
,
1706 ".directory-is-not-empty"), show
=True)
1707 run_cmd("stat %s" % test_tmpdir
, show
=True)
1708 run_cmd("stat %s" % testbase
, show
=True)
1709 if options
.skip_dependencies
:
1710 run_cmd("stat %s" % test_master
, dir=testbase
, output
=True)
1712 run_cmd("git clone --recursive --shared %s %s" % (gitroot
, test_master
), show
=True, dir=gitroot
)
1718 if options
.rebase
is not None:
1719 rebase_tree(options
.rebase
, rebase_branch
=options
.branch
)
1721 cleanup_list
.append(gitroot
+ "/autobuild.pid")
1723 elapsed_time
= time
.time() - start_time
1724 email_failure(-1, 'rebase', 'rebase', 'rebase',
1725 'rebase on %s failed' % options
.branch
,
1726 elapsed_time
, log_base
=options
.log_base
)
1730 blist
= buildlist(args
, options
.rebase
, rebase_branch
=options
.branch
)
1733 (status
, failed_task
, failed_stage
, failed_tag
, errstr
) = blist
.run()
1734 if status
!= 0 or errstr
!= "retry":
1736 cleanup(do_raise
=True)
1741 cleanup_list
.append(gitroot
+ "/autobuild.pid")
1747 do_print("waiting for tail to flush")
1750 elapsed_time
= time
.time() - start_time
1752 if options
.passcmd
is not None:
1753 do_print("Running passcmd: %s" % options
.passcmd
)
1754 run_cmd(options
.passcmd
, dir=test_master
)
1755 if options
.pushto
is not None:
1756 push_to(options
.pushto
, push_branch
=options
.branch
)
1757 if options
.keeplogs
or options
.attach_logs
:
1758 blist
.tarlogs("logs.tar.gz")
1759 do_print("Logs in logs.tar.gz")
1760 if options
.always_email
:
1761 email_success(elapsed_time
, log_base
=options
.log_base
)
1767 # something failed, gather a tar of the logs
1768 blist
.tarlogs("logs.tar.gz")
1770 if options
.email
is not None:
1771 email_failure(status
, failed_task
, failed_stage
, failed_tag
, errstr
,
1772 elapsed_time
, log_base
=options
.log_base
)
1774 elapsed_minutes
= elapsed_time
/ 60.0
1777 ####################################################################
1781 Your autobuild[%s] on %s failed after %.1f minutes
1782 when trying to test %s with the following error:
1786 the autobuild has been abandoned. Please fix the error and resubmit.
1788 ####################################################################
1790 ''' % (options
.branch
, platform
.node(), elapsed_minutes
, failed_task
, errstr
))
1794 do_print("Logs in logs.tar.gz")