2 # main.py: a shared, automated test suite for Subversion
4 # Subversion is a tool for revision control.
5 # See http://subversion.tigris.org for more information.
7 # ====================================================================
8 # Licensed to the Apache Software Foundation (ASF) under one
9 # or more contributor license agreements. See the NOTICE file
10 # distributed with this work for additional information
11 # regarding copyright ownership. The ASF licenses this file
12 # to you under the Apache License, Version 2.0 (the
13 # "License"); you may not use this file except in compliance
14 # with the License. You may obtain a copy of the License at
16 # http://www.apache.org/licenses/LICENSE-2.0
18 # Unless required by applicable law or agreed to in writing,
19 # software distributed under the License is distributed on an
20 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
21 # KIND, either express or implied. See the License for the
22 # specific language governing permissions and limitations
24 ######################################################################
39 from urlparse
import urlparse
44 from urllib
.parse
import quote
as urllib_parse_quote
45 from urllib
.parse
import unquote
as urllib_parse_unquote
49 from urllib
import quote
as urllib_parse_quote
50 from urllib
import unquote
as urllib_parse_unquote
53 from svntest
import Failure
54 from svntest
import Skip
58 ######################################################################
60 # HOW TO USE THIS MODULE:
62 # Write a new python script that
64 # 1) imports this 'svntest' package
66 # 2) contains a number of related 'test' routines. (Each test
67 # routine should take no arguments, and return None on success
68 # or throw a Failure exception on failure. Each test should
69 # also contain a short docstring.)
71 # 3) places all the tests into a list that begins with None.
73 # 4) calls svntest.main.client_test() on the list.
75 # Also, your tests will probably want to use some of the common
76 # routines in the 'Utilities' section below.
78 #####################################################################
81 default_num_threads
= 5
83 # Don't try to use this before calling execute_tests()
87 class SVNProcessTerminatedBySignal(Failure
):
88 "Exception raised if a spawned process segfaulted, aborted, etc."
91 class SVNLineUnequal(Failure
):
92 "Exception raised if two lines are unequal"
95 class SVNUnmatchedError(Failure
):
96 "Exception raised if an expected error is not found"
99 class SVNCommitFailure(Failure
):
100 "Exception raised if a commit failed"
103 class SVNRepositoryCopyFailure(Failure
):
104 "Exception raised if unable to copy a repository"
107 class SVNRepositoryCreateFailure(Failure
):
108 "Exception raised if unable to create a repository"
112 if sys
.platform
== 'win32':
114 file_scheme_prefix
= 'file:'
117 os
.environ
['SVN_DBG_STACKTRACES_TO_STDERR'] = 'y'
120 file_scheme_prefix
= 'file://'
124 # The location of our mock svneditor script.
126 svneditor_script
= os
.path
.join(sys
.path
[0], 'svneditor.bat')
128 svneditor_script
= os
.path
.join(sys
.path
[0], 'svneditor.py')
130 # Username and password used by the working copies
131 wc_author
= 'jrandom'
132 wc_passwd
= 'rayjandom'
134 # Username and password used by the working copies for "second user"
136 wc_author2
= 'jconstant' # use the same password as wc_author
138 # Set C locale for command line programs
139 os
.environ
['LC_ALL'] = 'C'
141 ######################################################################
142 # The locations of the svn, svnadmin and svnlook binaries, relative to
143 # the only scripts that import this file right now (they live in ../).
144 # Use --bin to override these defaults.
145 svn_binary
= os
.path
.abspath('../../svn/svn' + _exe
)
146 svnadmin_binary
= os
.path
.abspath('../../svnadmin/svnadmin' + _exe
)
147 svnlook_binary
= os
.path
.abspath('../../svnlook/svnlook' + _exe
)
148 svnrdump_binary
= os
.path
.abspath('../../svnrdump/svnrdump' + _exe
)
149 svnsync_binary
= os
.path
.abspath('../../svnsync/svnsync' + _exe
)
150 svnversion_binary
= os
.path
.abspath('../../svnversion/svnversion' + _exe
)
151 svndumpfilter_binary
= os
.path
.abspath('../../svndumpfilter/svndumpfilter' + \
153 svnmucc_binary
=os
.path
.abspath('../../svnmucc/svnmucc' + _exe
)
154 entriesdump_binary
= os
.path
.abspath('entries-dump' + _exe
)
155 atomic_ra_revprop_change_binary
= os
.path
.abspath('atomic-ra-revprop-change' + \
157 wc_lock_tester_binary
= os
.path
.abspath('../libsvn_wc/wc-lock-tester' + _exe
)
158 wc_incomplete_tester_binary
= os
.path
.abspath('../libsvn_wc/wc-incomplete-tester' + _exe
)
160 # Location to the pristine repository, will be calculated from test_area_url
161 # when we know what the user specified for --url.
162 pristine_greek_repos_url
= None
164 # Global variable to track all of our options
167 # End of command-line-set global variables.
168 ######################################################################
170 # All temporary repositories and working copies are created underneath
171 # this dir, so there's one point at which to mount, e.g., a ramdisk.
172 work_dir
= "svn-test-work"
174 # Constant for the merge info property.
175 SVN_PROP_MERGEINFO
= "svn:mergeinfo"
177 # Where we want all the repositories and working copies to live.
178 # Each test will have its own!
179 general_repo_dir
= os
.path
.join(work_dir
, "repositories")
180 general_wc_dir
= os
.path
.join(work_dir
, "working_copies")
182 # temp directory in which we will create our 'pristine' local
183 # repository and other scratch data. This should be removed when we
184 # quit and when we startup.
185 temp_dir
= os
.path
.join(work_dir
, 'local_tmp')
187 # (derivatives of the tmp dir.)
188 pristine_greek_repos_dir
= os
.path
.join(temp_dir
, "repos")
189 greek_dump_dir
= os
.path
.join(temp_dir
, "greekfiles")
190 default_config_dir
= os
.path
.abspath(os
.path
.join(temp_dir
, "config"))
193 # Our pristine greek-tree state.
195 # If a test wishes to create an "expected" working-copy tree, it should
196 # call main.greek_state.copy(). That method will return a copy of this
197 # State object which can then be edited.
199 _item
= svntest
.wc
.StateItem
200 greek_state
= svntest
.wc
.State('', {
201 'iota' : _item("This is the file 'iota'.\n"),
203 'A/mu' : _item("This is the file 'mu'.\n"),
205 'A/B/lambda' : _item("This is the file 'lambda'.\n"),
207 'A/B/E/alpha' : _item("This is the file 'alpha'.\n"),
208 'A/B/E/beta' : _item("This is the file 'beta'.\n"),
212 'A/D/gamma' : _item("This is the file 'gamma'.\n"),
214 'A/D/G/pi' : _item("This is the file 'pi'.\n"),
215 'A/D/G/rho' : _item("This is the file 'rho'.\n"),
216 'A/D/G/tau' : _item("This is the file 'tau'.\n"),
218 'A/D/H/chi' : _item("This is the file 'chi'.\n"),
219 'A/D/H/psi' : _item("This is the file 'psi'.\n"),
220 'A/D/H/omega' : _item("This is the file 'omega'.\n"),
224 ######################################################################
225 # Utilities shared by the tests
226 def wrap_ex(func
, output
):
227 "Wrap a function, catch, print and ignore exceptions"
228 def w(*args
, **kwds
):
230 return func(*args
, **kwds
)
232 if ex
.__class
__ != Failure
or ex
.args
:
235 logger
.warn('EXCEPTION: %s: %s', ex
.__class
__.__name
__, ex_args
)
237 logger
.warn('EXCEPTION: %s', ex
.__class
__.__name
__)
240 def setup_development_mode():
241 "Wraps functions in module actions"
242 l
= [ 'run_and_verify_svn',
243 'run_and_verify_svnversion',
244 'run_and_verify_load',
245 'run_and_verify_dump',
246 'run_and_verify_checkout',
247 'run_and_verify_export',
248 'run_and_verify_update',
249 'run_and_verify_merge',
250 'run_and_verify_switch',
251 'run_and_verify_commit',
252 'run_and_verify_unquiet_status',
253 'run_and_verify_status',
254 'run_and_verify_diff_summarize',
255 'run_and_verify_diff_summarize_xml',
256 'run_and_validate_lock']
259 setattr(svntest
.actions
, func
, wrap_ex(getattr(svntest
.actions
, func
)))
261 def get_admin_name():
262 "Return name of SVN administrative subdirectory."
264 if (windows
or sys
.platform
== 'cygwin') \
265 and 'SVN_ASP_DOT_NET_HACK' in os
.environ
:
270 def wc_is_singledb(wcpath
):
271 """Temporary function that checks whether a working copy directory looks
272 like it is part of a single-db working copy."""
274 pristine
= os
.path
.join(wcpath
, get_admin_name(), 'pristine')
275 if not os
.path
.exists(pristine
):
278 # Now we must be looking at a multi-db WC dir or the root dir of a
279 # single-DB WC. Sharded 'pristine' dir => single-db, else => multi-db.
280 for name
in os
.listdir(pristine
):
283 elif len(name
) == 40:
288 def get_start_commit_hook_path(repo_dir
):
289 "Return the path of the start-commit-hook conf file in REPO_DIR."
291 return os
.path
.join(repo_dir
, "hooks", "start-commit")
293 def get_pre_commit_hook_path(repo_dir
):
294 "Return the path of the pre-commit-hook conf file in REPO_DIR."
296 return os
.path
.join(repo_dir
, "hooks", "pre-commit")
298 def get_post_commit_hook_path(repo_dir
):
299 "Return the path of the post-commit-hook conf file in REPO_DIR."
301 return os
.path
.join(repo_dir
, "hooks", "post-commit")
303 def get_pre_revprop_change_hook_path(repo_dir
):
304 "Return the path of the pre-revprop-change hook script in REPO_DIR."
306 return os
.path
.join(repo_dir
, "hooks", "pre-revprop-change")
308 def get_pre_lock_hook_path(repo_dir
):
309 "Return the path of the pre-lock hook script in REPO_DIR."
311 return os
.path
.join(repo_dir
, "hooks", "pre-lock")
313 def get_pre_unlock_hook_path(repo_dir
):
314 "Return the path of the pre-unlock hook script in REPO_DIR."
316 return os
.path
.join(repo_dir
, "hooks", "pre-unlock")
318 def get_svnserve_conf_file_path(repo_dir
):
319 "Return the path of the svnserve.conf file in REPO_DIR."
321 return os
.path
.join(repo_dir
, "conf", "svnserve.conf")
323 def get_fsfs_conf_file_path(repo_dir
):
324 "Return the path of the fsfs.conf file in REPO_DIR."
326 return os
.path
.join(repo_dir
, "db", "fsfs.conf")
328 def get_fsfs_format_file_path(repo_dir
):
329 "Return the path of the format file in REPO_DIR."
331 return os
.path
.join(repo_dir
, "db", "format")
333 def filter_dbg(lines
):
335 if not line
.startswith('DBG:'):
338 # Run any binary, logging the command line and return code
339 def run_command(command
, error_expected
, binary_mode
=0, *varargs
):
340 """Run COMMAND with VARARGS. Return exit code as int; stdout, stderr
341 as lists of lines (including line terminators). See run_command_stdin()
342 for details. If ERROR_EXPECTED is None, any stderr output will be
343 printed and any stderr output or a non-zero exit code will raise an
346 return run_command_stdin(command
, error_expected
, 0, binary_mode
,
349 # A regular expression that matches arguments that are trivially safe
350 # to pass on a command line without quoting on any supported operating
352 _safe_arg_re
= re
.compile(r
'^[A-Za-z\d\.\_\/\-\:\@]+$')
355 """Quote ARG for a command line.
357 Return a quoted version of the string ARG, or just ARG if it contains
358 only universally harmless characters.
360 WARNING: This function cannot handle arbitrary command-line
361 arguments: it is just good enough for what we need here."""
364 if _safe_arg_re
.match(arg
):
368 # Note: subprocess.list2cmdline is Windows-specific.
369 return subprocess
.list2cmdline([arg
])
371 # Quoting suitable for most Unix shells.
372 return "'" + arg
.replace("'", "'\\''") + "'"
374 def open_pipe(command
, bufsize
=-1, stdin
=None, stdout
=None, stderr
=None):
375 """Opens a subprocess.Popen pipe to COMMAND using STDIN,
376 STDOUT, and STDERR. BUFSIZE is passed to subprocess.Popen's
377 argument of the same name.
379 Returns (infile, outfile, errfile, waiter); waiter
380 should be passed to wait_on_pipe."""
381 command
= [str(x
) for x
in command
]
383 # On Windows subprocess.Popen() won't accept a Python script as
384 # a valid program to execute, rather it wants the Python executable.
385 if (sys
.platform
== 'win32') and (command
[0].endswith('.py')):
386 command
.insert(0, sys
.executable
)
388 command_string
= command
[0] + ' ' + ' '.join(map(_quote_arg
, command
[1:]))
391 stdin
= subprocess
.PIPE
393 stdout
= subprocess
.PIPE
395 stderr
= subprocess
.PIPE
397 p
= subprocess
.Popen(command
,
402 close_fds
=not windows
)
403 return p
.stdin
, p
.stdout
, p
.stderr
, (p
, command_string
)
405 def wait_on_pipe(waiter
, binary_mode
, stdin
=None):
406 """WAITER is (KID, COMMAND_STRING). Wait for KID (opened with open_pipe)
407 to finish, dying if it does. If KID fails, create an error message
408 containing any stdout and stderr from the kid. Show COMMAND_STRING in
409 diagnostic messages. Normalize Windows line endings of stdout and stderr
410 if not BINARY_MODE. Return KID's exit code as int; stdout, stderr as
411 lists of lines (including line terminators)."""
415 kid
, command_string
= waiter
416 stdout
, stderr
= kid
.communicate(stdin
)
417 exit_code
= kid
.returncode
419 # Normalize Windows line endings if in text mode.
420 if windows
and not binary_mode
:
421 stdout
= stdout
.replace('\r\n', '\n')
422 stderr
= stderr
.replace('\r\n', '\n')
424 # Convert output strings to lists.
425 stdout_lines
= stdout
.splitlines(True)
426 stderr_lines
= stderr
.splitlines(True)
430 exit_signal
= os
.WTERMSIG(-exit_code
)
432 exit_signal
= exit_code
434 if stdout_lines
is not None:
435 logger
.info("".join(stdout_lines
))
436 if stderr_lines
is not None:
437 logger
.warning("".join(stderr_lines
))
438 # show the whole path to make it easier to start a debugger
439 logger
.warning("CMD: %s terminated by signal %d"
440 % (command_string
, exit_signal
))
441 raise SVNProcessTerminatedBySignal
444 logger
.info("CMD: %s exited with %d" % (command_string
, exit_code
))
445 return stdout_lines
, stderr_lines
, exit_code
447 def spawn_process(command
, bufsize
=-1, binary_mode
=0, stdin_lines
=None,
449 """Run any binary, supplying input text, logging the command line.
450 BUFSIZE dictates the pipe buffer size used in communication with the
451 subprocess: 0 means unbuffered, 1 means line buffered, any other
452 positive value means use a buffer of (approximately) that size.
453 A negative bufsize means to use the system default, which usually
454 means fully buffered. The default value for bufsize is 0 (unbuffered).
455 Normalize Windows line endings of stdout and stderr if not BINARY_MODE.
456 Return exit code as int; stdout, stderr as lists of lines (including
457 line terminators)."""
458 if stdin_lines
and not isinstance(stdin_lines
, list):
459 raise TypeError("stdin_lines should have list type")
461 # Log the command line
462 if not command
.endswith('.py'):
463 logger
.info('CMD: %s %s' % (os
.path
.basename(command
),
464 ' '.join([_quote_arg(x
) for x
in varargs
])))
466 infile
, outfile
, errfile
, kid
= open_pipe([command
] + list(varargs
), bufsize
)
469 for x
in stdin_lines
:
472 stdout_lines
, stderr_lines
, exit_code
= wait_on_pipe(kid
, binary_mode
)
478 return exit_code
, stdout_lines
, stderr_lines
480 def run_command_stdin(command
, error_expected
, bufsize
=-1, binary_mode
=0,
481 stdin_lines
=None, *varargs
):
482 """Run COMMAND with VARARGS; input STDIN_LINES (a list of strings
483 which should include newline characters) to program via stdin - this
484 should not be very large, as if the program outputs more than the OS
485 is willing to buffer, this will deadlock, with both Python and
486 COMMAND waiting to write to each other for ever. For tests where this
487 is a problem, setting BUFSIZE to a sufficiently large value will prevent
488 the deadlock, see spawn_process().
489 Normalize Windows line endings of stdout and stderr if not BINARY_MODE.
490 Return exit code as int; stdout, stderr as lists of lines (including
492 If ERROR_EXPECTED is None, any stderr output will be printed and any
493 stderr output or a non-zero exit code will raise an exception."""
497 exit_code
, stdout_lines
, stderr_lines
= spawn_process(command
,
503 def _line_contains_repos_diskpath(line
):
504 # ### Note: this assumes that either svn-test-work isn't a symlink,
505 # ### or the diskpath isn't realpath()'d somewhere on the way from
506 # ### the server's configuration and the client's stderr. We could
507 # ### check for both the symlinked path and the realpath.
509 os
.path
.join('cmdline', 'svn-test-work', 'repositories') in line \
510 or os
.path
.join('cmdline', 'svn-test-work', 'local_tmp', 'repos') in line
512 for lines
, name
in [[stdout_lines
, "stdout"], [stderr_lines
, "stderr"]]:
513 if is_ra_type_file() or 'svnadmin' in command
or 'svnlook' in command
:
515 # Does the server leak the repository on-disk path?
516 # (prop_tests-12 installs a hook script that does that intentionally)
517 if any(map(_line_contains_repos_diskpath
, lines
)) \
518 and not any(map(lambda arg
: 'prop_tests-12' in arg
, varargs
)):
519 raise Failure("Repository diskpath in %s: %r" % (name
, lines
))
522 logger
.info('<TIME = %.6f>' % (stop
- start
))
523 for x
in stdout_lines
:
524 logger
.info(x
.rstrip())
525 for x
in stderr_lines
:
526 logger
.info(x
.rstrip())
528 if (not error_expected
) and ((stderr_lines
) or (exit_code
!= 0)):
529 for x
in stderr_lines
:
530 logger
.warning(x
.rstrip())
534 [line
for line
in stdout_lines
if not line
.startswith("DBG:")], \
537 def create_config_dir(cfgdir
, config_contents
=None, server_contents
=None,
538 ssl_cert
=None, ssl_url
=None):
539 "Create config directories and files"
542 cfgfile_cfg
= os
.path
.join(cfgdir
, 'config')
543 cfgfile_srv
= os
.path
.join(cfgdir
, 'servers')
545 # create the directory
546 if not os
.path
.isdir(cfgdir
):
549 # define default config file contents if none provided
550 if config_contents
is None:
551 config_contents
= """
557 interactive-conflicts = false
560 # define default server file contents if none provided
561 if server_contents
is None:
562 http_library_str
= ""
563 if options
.http_library
:
564 http_library_str
= "http-library=%s" % (options
.http_library
)
565 server_contents
= """
569 store-plaintext-passwords=yes
571 """ % (http_library_str
)
573 file_write(cfgfile_cfg
, config_contents
)
574 file_write(cfgfile_srv
, server_contents
)
576 if (ssl_cert
and ssl_url
):
577 trust_ssl_cert(cfgdir
, ssl_cert
, ssl_url
)
578 elif cfgdir
!= default_config_dir
:
579 copy_trust(cfgdir
, default_config_dir
)
582 def trust_ssl_cert(cfgdir
, ssl_cert
, ssl_url
):
583 """Setup config dir to trust the given ssl_cert for the given ssl_url
587 fp
= open(ssl_cert
, 'r')
588 for line
in fp
.readlines()[1:-1]:
589 cert_rep
= cert_rep
+ line
.strip()
591 parsed_url
= urlparse(ssl_url
)
592 netloc_url
= '%s://%s' % (parsed_url
.scheme
, parsed_url
.netloc
)
593 ssl_dir
= os
.path
.join(cfgdir
, 'auth', 'svn.ssl.server')
594 if not os
.path
.isdir(ssl_dir
):
596 md5_name
= hashlib
.md5(netloc_url
).hexdigest()
597 md5_file
= os
.path
.join(ssl_dir
, md5_name
)
598 md5_file_contents
= """K 10
611 """ % (len(cert_rep
), cert_rep
, len(netloc_url
), netloc_url
)
613 file_write(md5_file
, md5_file_contents
)
615 def copy_trust(dst_cfgdir
, src_cfgdir
):
616 """Copy svn.ssl.server files from one config dir to another.
619 src_ssl_dir
= os
.path
.join(src_cfgdir
, 'auth', 'svn.ssl.server')
620 dst_ssl_dir
= os
.path
.join(dst_cfgdir
, 'auth', 'svn.ssl.server')
621 if not os
.path
.isdir(dst_ssl_dir
):
622 os
.makedirs(dst_ssl_dir
)
623 for f
in os
.listdir(src_ssl_dir
):
624 shutil
.copy(os
.path
.join(src_ssl_dir
, f
), os
.path
.join(dst_ssl_dir
, f
))
626 def _with_config_dir(args
):
627 if '--config-dir' in args
:
630 return args
+ ('--config-dir', default_config_dir
)
632 def _with_auth(args
):
633 assert '--password' not in args
634 args
= args
+ ('--password', wc_passwd
,
636 if '--username' in args
:
639 return args
+ ('--username', wc_author
)
641 def _with_log_message(args
):
643 if '-m' in args
or '--message' in args
or '-F' in args
:
646 return args
+ ('--message', 'default log message')
648 # For running subversion and returning the output
649 def run_svn(error_expected
, *varargs
):
650 """Run svn with VARARGS; return exit code as int; stdout, stderr as
651 lists of lines (including line terminators). If ERROR_EXPECTED is
652 None, any stderr output will be printed and any stderr output or a
653 non-zero exit code will raise an exception. If
654 you're just checking that something does/doesn't come out of
655 stdout/stderr, you might want to use actions.run_and_verify_svn()."""
656 return run_command(svn_binary
, error_expected
, 0,
657 *(_with_auth(_with_config_dir(varargs
))))
659 # For running svnadmin. Ignores the output.
660 def run_svnadmin(*varargs
):
661 """Run svnadmin with VARARGS, returns exit code as int; stdout, stderr as
662 list of lines (including line terminators)."""
663 return run_command(svnadmin_binary
, 1, 0, *varargs
)
665 # For running svnlook. Ignores the output.
666 def run_svnlook(*varargs
):
667 """Run svnlook with VARARGS, returns exit code as int; stdout, stderr as
668 list of lines (including line terminators)."""
669 return run_command(svnlook_binary
, 1, 0, *varargs
)
671 def run_svnrdump(stdin_input
, *varargs
):
672 """Run svnrdump with VARARGS, returns exit code as int; stdout, stderr as
673 list of lines (including line terminators). Use binary mode for output."""
675 return run_command_stdin(svnrdump_binary
, 1, 1, 1, stdin_input
,
676 *(_with_auth(_with_config_dir(varargs
))))
678 return run_command(svnrdump_binary
, 1, 1,
679 *(_with_auth(_with_config_dir(varargs
))))
681 def run_svnsync(*varargs
):
682 """Run svnsync with VARARGS, returns exit code as int; stdout, stderr as
683 list of lines (including line terminators)."""
684 return run_command(svnsync_binary
, 1, 0, *(_with_config_dir(varargs
)))
686 def run_svnversion(*varargs
):
687 """Run svnversion with VARARGS, returns exit code as int; stdout, stderr
688 as list of lines (including line terminators)."""
689 return run_command(svnversion_binary
, 1, 0, *varargs
)
691 def run_svnmucc(*varargs
):
692 """Run svnmucc with VARARGS, returns exit code as int; stdout, stderr as
693 list of lines (including line terminators). Use binary mode for output."""
694 return run_command(svnmucc_binary
, 1, 1,
695 *(_with_auth(_with_config_dir(_with_log_message(varargs
)))))
697 def run_entriesdump(path
):
698 """Run the entries-dump helper, returning a dict of Entry objects."""
699 # use spawn_process rather than run_command to avoid copying all the data
700 # to stdout in verbose mode.
701 exit_code
, stdout_lines
, stderr_lines
= spawn_process(entriesdump_binary
,
703 if exit_code
or stderr_lines
:
704 ### report on this? or continue to just skip it?
710 exec(''.join([line
for line
in stdout_lines
if not line
.startswith("DBG:")]))
713 def run_entriesdump_subdirs(path
):
714 """Run the entries-dump helper, returning a list of directory names."""
715 # use spawn_process rather than run_command to avoid copying all the data
716 # to stdout in verbose mode.
717 exit_code
, stdout_lines
, stderr_lines
= spawn_process(entriesdump_binary
,
718 0, 0, None, '--subdirs', path
)
719 return [line
.strip() for line
in stdout_lines
if not line
.startswith("DBG:")]
721 def run_atomic_ra_revprop_change(url
, revision
, propname
, skel
, want_error
):
722 """Run the atomic-ra-revprop-change helper, returning its exit code, stdout,
723 and stderr. For HTTP, default HTTP library is used."""
724 # use spawn_process rather than run_command to avoid copying all the data
725 # to stdout in verbose mode.
726 #exit_code, stdout_lines, stderr_lines = spawn_process(entriesdump_binary,
729 # This passes HTTP_LIBRARY in addition to our params.
730 return run_command(atomic_ra_revprop_change_binary
, True, False,
731 url
, revision
, propname
, skel
,
732 want_error
and 1 or 0, default_config_dir
)
734 def run_wc_lock_tester(recursive
, path
):
735 "Run the wc-lock obtainer tool, returning its exit code, stdout and stderr"
740 return run_command(wc_lock_tester_binary
, False, False, option
, path
)
742 def run_wc_incomplete_tester(wc_dir
, revision
):
743 "Run the wc-incomplete tool, returning its exit code, stdout and stderr"
744 return run_command(wc_incomplete_tester_binary
, False, False,
747 def youngest(repos_path
):
748 "run 'svnlook youngest' on REPOS_PATH, returns revision as int"
749 exit_code
, stdout_lines
, stderr_lines
= run_command(svnlook_binary
, None, 0,
750 'youngest', repos_path
)
751 if exit_code
or stderr_lines
:
752 raise Failure("Unexpected failure of 'svnlook youngest':\n%s" % stderr_lines
)
753 if len(stdout_lines
) != 1:
754 raise Failure("Wrong output from 'svnlook youngest':\n%s" % stdout_lines
)
755 return int(stdout_lines
[0].rstrip())
757 # Chmod recursively on a whole subtree
758 def chmod_tree(path
, mode
, mask
):
759 for dirpath
, dirs
, files
in os
.walk(path
):
760 for name
in dirs
+ files
:
761 fullname
= os
.path
.join(dirpath
, name
)
762 if not os
.path
.islink(fullname
):
763 new_mode
= (os
.stat(fullname
)[stat
.ST_MODE
] & ~mask
) | mode
764 os
.chmod(fullname
, new_mode
)
766 # For clearing away working copies
767 def safe_rmtree(dirname
, retry
=0):
768 """Remove the tree at DIRNAME, making it writable first.
769 If DIRNAME is a symlink, only remove the symlink, not its target."""
771 chmod_tree(dirname
, 0666, 0666)
772 shutil
.rmtree(dirname
)
774 if os
.path
.islink(dirname
):
778 if not os
.path
.exists(dirname
):
782 for delay
in (0.5, 1, 2, 4):
793 # For making local mods to files
794 def file_append(path
, new_text
):
795 "Append NEW_TEXT to file at PATH"
796 open(path
, 'a').write(new_text
)
798 # Append in binary mode
799 def file_append_binary(path
, new_text
):
800 "Append NEW_TEXT to file at PATH in binary mode"
801 open(path
, 'ab').write(new_text
)
803 # For creating new files, and making local mods to existing files.
804 def file_write(path
, contents
, mode
='w'):
805 """Write the CONTENTS to the file at PATH, opening file using MODE,
806 which is (w)rite by default."""
807 open(path
, mode
).write(contents
)
809 # For replacing parts of contents in an existing file, with new content.
810 def file_substitute(path
, contents
, new_contents
):
811 """Replace the CONTENTS in the file at PATH using the NEW_CONTENTS"""
812 fcontent
= open(path
, 'r').read().replace(contents
, new_contents
)
813 open(path
, 'w').write(fcontent
)
815 # For creating blank new repositories
816 def create_repos(path
, minor_version
= None):
817 """Create a brand-new SVN repository at PATH. If PATH does not yet
820 if not os
.path
.exists(path
):
821 os
.makedirs(path
) # this creates all the intermediate dirs, if neccessary
823 opts
= ("--bdb-txn-nosync",)
824 if not minor_version
or minor_version
> options
.server_minor_version
:
825 minor_version
= options
.server_minor_version
826 if minor_version
< 4:
827 opts
+= ("--pre-1.4-compatible",)
828 elif minor_version
< 5:
829 opts
+= ("--pre-1.5-compatible",)
830 elif minor_version
< 6:
831 opts
+= ("--pre-1.6-compatible",)
832 if options
.fs_type
is not None:
833 opts
+= ("--fs-type=" + options
.fs_type
,)
834 exit_code
, stdout
, stderr
= run_command(svnadmin_binary
, 1, 0, "create",
837 # Skip tests if we can't create the repository.
840 if line
.find('Unknown FS type') != -1:
842 # If the FS type is known, assume the repos couldn't be created
843 # (e.g. due to a missing 'svnadmin' binary).
844 raise SVNRepositoryCreateFailure("".join(stderr
).rstrip())
846 # Require authentication to write to the repos, for ra_svn testing.
847 file_write(get_svnserve_conf_file_path(path
),
848 "[general]\nauth-access = write\n");
849 if options
.enable_sasl
:
850 file_append(get_svnserve_conf_file_path(path
),
851 "realm = svntest\n[sasl]\nuse-sasl = true\n")
853 file_append(get_svnserve_conf_file_path(path
), "password-db = passwd\n")
854 # This actually creates TWO [users] sections in the file (one of them is
855 # uncommented in `svnadmin create`'s template), so we exercise the .ini
856 # files reading code's handling of duplicates, too. :-)
857 file_append(os
.path
.join(path
, "conf", "passwd"),
858 "[users]\njrandom = rayjandom\njconstant = rayjandom\n");
860 if options
.fs_type
is None or options
.fs_type
== 'fsfs':
862 if options
.config_file
is not None:
863 shutil
.copy(options
.config_file
, get_fsfs_conf_file_path(path
))
866 if options
.fsfs_sharding
is not None:
867 def transform_line(line
):
868 if line
.startswith('layout '):
869 if options
.fsfs_sharding
> 0:
870 line
= 'layout sharded %d' % options
.fsfs_sharding
872 line
= 'layout linear'
876 format_file_path
= get_fsfs_format_file_path(path
)
877 contents
= open(format_file_path
, 'rb').read()
880 new_contents
= "".join([transform_line(line
) + "\n"
881 for line
in contents
.split("\n")])
882 if new_contents
[-1] == "\n":
883 # we don't currently allow empty lines (\n\n) in the format file.
884 new_contents
= new_contents
[:-1]
887 os
.chmod(format_file_path
, 0666)
888 file_write(format_file_path
, new_contents
, 'wb')
891 # Note that some tests (currently only commit_tests) create their own
892 # post-commit hooks, which would override this one. :-(
893 if options
.fsfs_packing
:
895 abs_path
= os
.path
.abspath(path
)
896 create_python_hook_script(get_post_commit_hook_path(abs_path
),
897 "import subprocess\n"
900 "sys.exit(subprocess.Popen(command).wait())\n"
901 % repr([svnadmin_binary
, 'pack', abs_path
]))
903 # make the repos world-writeable, for mod_dav_svn's sake.
904 chmod_tree(path
, 0666, 0666)
906 # For copying a repository
907 def copy_repos(src_path
, dst_path
, head_revision
, ignore_uuid
= 1,
908 minor_version
= None):
909 "Copy the repository SRC_PATH, with head revision HEAD_REVISION, to DST_PATH"
911 # Save any previous value of SVN_DBG_QUIET
912 saved_quiet
= os
.environ
.get('SVN_DBG_QUIET')
913 os
.environ
['SVN_DBG_QUIET'] = 'y'
915 # Do an svnadmin dump|svnadmin load cycle. Print a fake pipe command so that
916 # the displayed CMDs can be run by hand
917 create_repos(dst_path
, minor_version
)
918 dump_args
= ['dump', src_path
]
919 load_args
= ['load', dst_path
]
922 load_args
= load_args
+ ['--ignore-uuid']
924 logger
.info('CMD: %s %s | %s %s' %
925 (os
.path
.basename(svnadmin_binary
), ' '.join(dump_args
),
926 os
.path
.basename(svnadmin_binary
), ' '.join(load_args
)))
929 dump_in
, dump_out
, dump_err
, dump_kid
= open_pipe(
930 [svnadmin_binary
] + dump_args
)
931 load_in
, load_out
, load_err
, load_kid
= open_pipe(
932 [svnadmin_binary
] + load_args
,
933 stdin
=dump_out
) # Attached to dump_kid
935 load_stdout
, load_stderr
, load_exit_code
= wait_on_pipe(load_kid
, True)
936 dump_stdout
, dump_stderr
, dump_exit_code
= wait_on_pipe(dump_kid
, True)
941 #load_in is dump_out so it's already closed.
946 logger
.info('<TIME = %.6f>' % (stop
- start
))
948 if saved_quiet
is None:
949 del os
.environ
['SVN_DBG_QUIET']
951 os
.environ
['SVN_DBG_QUIET'] = saved_quiet
953 dump_re
= re
.compile(r
'^\* Dumped revision (\d+)\.\r?$')
955 for dump_line
in dump_stderr
:
956 match
= dump_re
.match(dump_line
)
957 if not match
or match
.group(1) != str(expect_revision
):
958 logger
.warn('ERROR: dump failed: %s', dump_line
.strip())
959 raise SVNRepositoryCopyFailure
961 if expect_revision
!= head_revision
+ 1:
962 logger
.warn('ERROR: dump failed; did not see revision %s', head_revision
)
963 raise SVNRepositoryCopyFailure
965 load_re
= re
.compile(r
'^------- Committed revision (\d+) >>>\r?$')
967 for load_line
in filter_dbg(load_stdout
):
968 match
= load_re
.match(load_line
)
970 if match
.group(1) != str(expect_revision
):
971 logger
.warn('ERROR: load failed: %s', load_line
.strip())
972 raise SVNRepositoryCopyFailure
974 if expect_revision
!= head_revision
+ 1:
975 logger
.warn('ERROR: load failed; did not see revision %s', head_revision
)
976 raise SVNRepositoryCopyFailure
979 def canonicalize_url(input):
980 "Canonicalize the url, if the scheme is unknown, returns intact input"
982 m
= re
.match(r
"^((file://)|((svn|svn\+ssh|http|https)(://)))", input)
985 return scheme
+ re
.sub(r
'//*', '/', input[len(scheme
):])
990 def create_python_hook_script(hook_path
, hook_script_code
,
991 cmd_alternative
=None):
992 """Create a Python hook script at HOOK_PATH with the specified
996 if cmd_alternative
is not None:
997 file_write("%s.bat" % hook_path
,
1000 # Use an absolute path since the working directory is not guaranteed
1001 hook_path
= os
.path
.abspath(hook_path
)
1002 # Fill the python file.
1003 file_write("%s.py" % hook_path
, hook_script_code
)
1004 # Fill the batch wrapper file.
1005 file_write("%s.bat" % hook_path
,
1006 "@\"%s\" %s.py %%*\n" % (sys
.executable
, hook_path
))
1008 # For all other platforms
1009 file_write(hook_path
, "#!%s\n%s" % (sys
.executable
, hook_script_code
))
1010 os
.chmod(hook_path
, 0755)
1012 def write_restrictive_svnserve_conf(repo_dir
, anon_access
="none"):
1013 "Create a restrictive authz file ( no anynomous access )."
1015 fp
= open(get_svnserve_conf_file_path(repo_dir
), 'w')
1016 fp
.write("[general]\nanon-access = %s\nauth-access = write\n"
1017 "authz-db = authz\n" % anon_access
)
1018 if options
.enable_sasl
:
1019 fp
.write("realm = svntest\n[sasl]\nuse-sasl = true\n");
1021 fp
.write("password-db = passwd\n")
1024 # Warning: because mod_dav_svn uses one shared authz file for all
1025 # repositories, you *cannot* use write_authz_file in any test that
1026 # might be run in parallel.
1028 # write_authz_file can *only* be used in test suites which disable
1029 # parallel execution at the bottom like so
1030 # if __name__ == '__main__':
1031 # svntest.main.run_tests(test_list, serial_only = True)
1032 def write_authz_file(sbox
, rules
, sections
=None):
1033 """Write an authz file to SBOX, appropriate for the RA method used,
1034 with authorizations rules RULES mapping paths to strings containing
1035 the rules. You can add sections SECTIONS (ex. groups, aliases...) with
1036 an appropriate list of mappings.
1038 fp
= open(sbox
.authz_file
, 'w')
1040 # When the sandbox repository is read only it's name will be different from
1041 # the repository name.
1042 repo_name
= sbox
.repo_dir
1043 while repo_name
[-1] == '/':
1044 repo_name
= repo_name
[:-1]
1045 repo_name
= os
.path
.basename(repo_name
)
1047 if sbox
.repo_url
.startswith("http"):
1048 prefix
= repo_name
+ ":"
1052 for p
, r
in sections
.items():
1053 fp
.write("[%s]\n%s\n" % (p
, r
))
1055 for p
, r
in rules
.items():
1056 fp
.write("[%s%s]\n%s\n" % (prefix
, p
, r
))
1059 def use_editor(func
):
1060 os
.environ
['SVN_EDITOR'] = svneditor_script
1061 os
.environ
['SVN_MERGE'] = svneditor_script
1062 os
.environ
['SVNTEST_EDITOR_FUNC'] = func
1063 os
.environ
['SVN_TEST_PYTHON'] = sys
.executable
1065 def mergeinfo_notify_line(revstart
, revend
, target
=None):
1066 """Return an expected output line that describes the beginning of a
1067 mergeinfo recording notification on revisions REVSTART through REVEND."""
1069 target_re
= re
.escape(target
)
1072 if (revend
is None):
1074 revstart
= abs(revstart
)
1075 return "--- Recording mergeinfo for reverse merge of r%ld into '%s':\n" \
1076 % (revstart
, target_re
)
1078 return "--- Recording mergeinfo for merge of r%ld into '%s':\n" \
1079 % (revstart
, target_re
)
1080 elif (revstart
< revend
):
1081 return "--- Recording mergeinfo for merge of r%ld through r%ld into '%s':\n" \
1082 % (revstart
, revend
, target_re
)
1084 return "--- Recording mergeinfo for reverse merge of r%ld through " \
1085 "r%ld into '%s':\n" % (revstart
, revend
, target_re
)
1087 def merge_notify_line(revstart
=None, revend
=None, same_URL
=True,
1088 foreign
=False, target
=None):
1089 """Return an expected output line that describes the beginning of a
1090 merge operation on revisions REVSTART through REVEND. Omit both
1091 REVSTART and REVEND for the case where the left and right sides of
1092 the merge are from different URLs."""
1093 from_foreign_phrase
= foreign
and "\(from foreign repository\) " or ""
1095 target_re
= re
.escape(target
)
1099 return "--- Merging differences between %srepository URLs into '%s':\n" \
1100 % (foreign
and "foreign " or "", target_re
)
1102 if revstart
is None:
1103 # The left and right sides of the merge are from different URLs.
1104 return "--- Merging differences between %srepository URLs into '%s':\n" \
1105 % (foreign
and "foreign " or "", target_re
)
1107 return "--- Reverse-merging %sr%ld into '%s':\n" \
1108 % (from_foreign_phrase
, abs(revstart
), target_re
)
1110 return "--- Merging %sr%ld into '%s':\n" \
1111 % (from_foreign_phrase
, revstart
, target_re
)
1113 if revstart
> revend
:
1114 return "--- Reverse-merging %sr%ld through r%ld into '%s':\n" \
1115 % (from_foreign_phrase
, revstart
, revend
, target_re
)
1117 return "--- Merging %sr%ld through r%ld into '%s':\n" \
1118 % (from_foreign_phrase
, revstart
, revend
, target_re
)
1122 "Conjure up a log message based on the calling test."
1124 for idx
in range(1, 100):
1125 frame
= sys
._getframe
(idx
)
1127 # If this frame isn't from a function in *_tests.py, then skip it.
1128 filename
= frame
.f_code
.co_filename
1129 if not filename
.endswith('_tests.py'):
1132 # There should be a test_list in this module.
1133 test_list
= frame
.f_globals
.get('test_list')
1134 if test_list
is None:
1137 # If the function is not in the test_list, then skip it.
1138 func_name
= frame
.f_code
.co_name
1139 func_ob
= frame
.f_globals
.get(func_name
)
1140 if func_ob
not in test_list
:
1143 # Make the log message look like a line from a traceback.
1144 # Well...close. We use single quotes to avoid interfering with the
1145 # double-quote quoting performed on Windows
1146 return "File '%s', line %d, in %s" % (filename
, frame
.f_lineno
, func_name
)
1149 ######################################################################
1150 # Functions which check the test configuration
1151 # (useful for conditional XFails)
1153 def is_ra_type_dav():
1154 return options
.test_area_url
.startswith('http')
1156 def is_ra_type_dav_neon():
1157 """Return True iff running tests over RA-Neon.
1158 CAUTION: Result is only valid if svn was built to support both."""
1159 return options
.test_area_url
.startswith('http') and \
1160 (options
.http_library
== "neon")
1162 def is_ra_type_dav_serf():
1163 """Return True iff running tests over RA-Serf.
1164 CAUTION: Result is only valid if svn was built to support both."""
1165 return options
.test_area_url
.startswith('http') and \
1166 (options
.http_library
== "serf")
1168 def is_ra_type_svn():
1169 """Return True iff running tests over RA-svn."""
1170 return options
.test_area_url
.startswith('svn')
1172 def is_ra_type_file():
1173 """Return True iff running tests over RA-local."""
1174 return options
.test_area_url
.startswith('file')
1176 def is_fs_type_fsfs():
1177 # This assumes that fsfs is the default fs implementation.
1178 return options
.fs_type
== 'fsfs' or options
.fs_type
is None
1180 def is_fs_type_bdb():
1181 return options
.fs_type
== 'bdb'
1183 def is_os_windows():
1184 return os
.name
== 'nt'
1186 def is_windows_type_dav():
1187 return is_os_windows() and is_ra_type_dav()
1190 return os
.name
== 'posix'
1193 return sys
.platform
== 'darwin'
1195 def is_fs_case_insensitive():
1196 return (is_os_darwin() or is_os_windows())
1198 def is_threaded_python():
1201 def server_has_mergeinfo():
1202 return options
.server_minor_version
>= 5
1204 def server_has_revprop_commit():
1205 return options
.server_minor_version
>= 5
1207 def server_authz_has_aliases():
1208 return options
.server_minor_version
>= 5
1210 def server_gets_client_capabilities():
1211 return options
.server_minor_version
>= 5
1213 def server_has_partial_replay():
1214 return options
.server_minor_version
>= 5
1216 def server_enforces_UTF8_fspaths_in_verify():
1217 return options
.server_minor_version
>= 6
1219 def server_enforces_date_syntax():
1220 return options
.server_minor_version
>= 5
1222 def server_has_atomic_revprop():
1223 return options
.server_minor_version
>= 7
1225 ######################################################################
1228 class TestSpawningThread(threading
.Thread
):
1229 """A thread that runs test cases in their own processes.
1230 Receives test numbers to run from the queue, and saves results into
1231 the results field."""
1232 def __init__(self
, queue
, progress_func
, tests_total
):
1233 threading
.Thread
.__init
__(self
)
1236 self
.progress_func
= progress_func
1237 self
.tests_total
= tests_total
1242 next_index
= self
.queue
.get_nowait()
1246 self
.run_one(next_index
)
1249 if self
.progress_func
:
1250 self
.progress_func(self
.tests_total
- self
.queue
.qsize(),
1253 def run_one(self
, index
):
1254 command
= os
.path
.abspath(sys
.argv
[0])
1257 args
.append(str(index
))
1259 # add some startup arguments from this process
1261 args
.append('--fs-type=' + options
.fs_type
)
1262 if options
.test_area_url
:
1263 args
.append('--url=' + options
.test_area_url
)
1264 if logger
.getEffectiveLevel() <= logging
.DEBUG
:
1267 args
.append('--cleanup')
1268 if options
.enable_sasl
:
1269 args
.append('--enable-sasl')
1270 if options
.http_library
:
1271 args
.append('--http-library=' + options
.http_library
)
1272 if options
.server_minor_version
:
1273 args
.append('--server-minor-version=' + str(options
.server_minor_version
))
1274 if options
.mode_filter
:
1275 args
.append('--mode-filter=' + options
.mode_filter
)
1276 if options
.milestone_filter
:
1277 args
.append('--milestone-filter=' + options
.milestone_filter
)
1278 if options
.ssl_cert
:
1279 args
.append('--ssl-cert=' + options
.ssl_cert
)
1281 result
, stdout_lines
, stderr_lines
= spawn_process(command
, 0, 0, None,
1283 self
.results
.append((index
, result
, stdout_lines
, stderr_lines
))
1286 sys
.stdout
.write('.')
1288 sys
.stdout
.write('F')
1293 """Encapsulate a single test case (predicate), including logic for
1294 runing the test and test list output."""
1296 def __init__(self
, func
, index
):
1297 self
.pred
= svntest
.testcase
.create_test_case(func
)
1300 def list(self
, milestones_dict
=None):
1301 """Print test doc strings. MILESTONES_DICT is an optional mapping
1302 of issue numbers to target milestones."""
1303 if options
.mode_filter
.upper() == 'ALL' \
1304 or options
.mode_filter
.upper() == self
.pred
.list_mode().upper() \
1305 or (options
.mode_filter
.upper() == 'PASS' \
1306 and self
.pred
.list_mode() == ''):
1309 if self
.pred
.issues
:
1310 if not options
.milestone_filter
or milestones_dict
is None:
1311 issues
= self
.pred
.issues
1312 else: # Limit listing by requested target milestone(s).
1314 matches_filter
= False
1316 # Get the milestones for all the issues associated with this test.
1317 # If any one of them matches the MILESTONE_FILTER then we'll print
1319 for issue
in self
.pred
.issues
:
1320 # A safe starting assumption.
1321 milestone
= 'unknown'
1323 if milestones_dict
.has_key(str(issue
)):
1324 milestone
= milestones_dict
[str(issue
)]
1326 filter_issues
.append(str(issue
) + '(' + milestone
+ ')')
1327 pattern
= re
.compile(options
.milestone_filter
)
1328 if pattern
.match(milestone
):
1329 matches_filter
= True
1331 # Did at least one of the associated issues meet our filter?
1333 issues
= filter_issues
1335 tail
+= " [%s]" % ','.join(['#%s' % str(i
) for i
in issues
])
1337 # If there is no filter or this test made if through
1338 # the filter then print it!
1339 if options
.milestone_filter
is None or len(issues
):
1340 if self
.pred
.inprogress
:
1341 tail
+= " [[%s]]" % self
.pred
.inprogress
1343 print(" %3d %-5s %s%s" % (self
.index
,
1344 self
.pred
.list_mode(),
1345 self
.pred
.description
,
1350 return self
.pred
.list_mode()
1352 def get_issues(self
):
1353 return self
.pred
.issues
1355 def get_function_name(self
):
1356 return self
.pred
.get_function_name()
1358 def _print_name(self
, prefix
):
1359 if self
.pred
.inprogress
:
1360 print("%s %s %s: %s [[WIMP: %s]]" % (prefix
,
1361 os
.path
.basename(sys
.argv
[0]),
1363 self
.pred
.description
,
1364 self
.pred
.inprogress
))
1366 print("%s %s %s: %s" % (prefix
,
1367 os
.path
.basename(sys
.argv
[0]),
1369 self
.pred
.description
))
1373 """Run self.pred and return the result. The return value is
1374 - 0 if the test was successful
1375 - 1 if it errored in a way that indicates test failure
1376 - 2 if the test skipped
1378 sbox_name
= self
.pred
.get_sandbox_name()
1380 sandbox
= svntest
.sandbox
.Sandbox(sbox_name
, self
.index
)
1384 # Explicitly set this so that commands that commit but don't supply a
1385 # log message will fail rather than invoke an editor.
1386 # Tests that want to use an editor should invoke svntest.main.use_editor.
1387 os
.environ
['SVN_EDITOR'] = ''
1388 os
.environ
['SVNTEST_EDITOR_FUNC'] = ''
1390 if options
.use_jsvn
:
1391 # Set this SVNKit specific variable to the current test (test name plus
1392 # its index) being run so that SVNKit daemon could use this test name
1393 # for its separate log file
1394 os
.environ
['SVN_CURRENT_TEST'] = os
.path
.basename(sys
.argv
[0]) + "_" + \
1397 svntest
.actions
.no_sleep_for_timestamps()
1398 svntest
.actions
.do_relocate_validation()
1400 saved_dir
= os
.getcwd()
1402 rc
= self
.pred
.run(sandbox
)
1404 self
._print
_name
('STYLE ERROR in')
1405 print('Test driver returned a status code.')
1407 result
= svntest
.testcase
.RESULT_OK
1409 result
= svntest
.testcase
.RESULT_SKIP
1411 result
= svntest
.testcase
.RESULT_FAIL
1413 # We captured Failure and its subclasses. We don't want to print
1414 # anything for plain old Failure since that just indicates test
1415 # failure, rather than relevant information. However, if there
1416 # *is* information in the exception's arguments, then print it.
1417 if ex
.__class
__ != Failure
or ex
.args
:
1419 logger
.warn('CWD: %s' % os
.getcwd())
1421 msg
= 'EXCEPTION: %s: %s' % (ex
.__class
__.__name
__, ex_args
)
1423 msg
= 'EXCEPTION: %s' % ex
.__class
__.__name
__
1424 logger
.warn(msg
, exc_info
=True)
1425 except KeyboardInterrupt:
1426 logger
.error('Interrupted')
1428 except SystemExit, ex
:
1429 logger
.error('EXCEPTION: SystemExit(%d), skipping cleanup' % ex
.code
)
1430 self
._print
_name
(ex
.code
and 'FAIL: ' or 'PASS: ')
1433 result
= svntest
.testcase
.RESULT_FAIL
1434 logger
.warn('CWD: %s' % os
.getcwd(), exc_info
=True)
1437 exit_code
, result_text
, result_benignity
= self
.pred
.results(result
)
1438 if not (options
.quiet
and result_benignity
):
1439 self
._print
_name
(result_text
)
1440 if sandbox
is not None and exit_code
!= 1 and options
.cleanup
:
1441 sandbox
.cleanup_test_paths()
1444 ######################################################################
1445 # Main testing functions
1447 # These two functions each take a TEST_LIST as input. The TEST_LIST
1448 # should be a list of test functions; each test function should take
1449 # no arguments and return a 0 on success, non-zero on failure.
1450 # Ideally, each test should also have a short, one-line docstring (so
1451 # it can be displayed by the 'list' command.)
1453 # Func to run one test in the list.
1454 def run_one_test(n
, test_list
, finished_tests
= None):
1455 """Run the Nth client test in TEST_LIST, return the result.
1457 If we're running the tests in parallel spawn the test in a new process.
1460 # allow N to be negative, so './basic_tests.py -- -1' works
1461 num_tests
= len(test_list
) - 1
1462 if (n
== 0) or (abs(n
) > num_tests
):
1463 print("There is no test %s.\n" % n
)
1468 test_mode
= TestRunner(test_list
[n
], n
).get_mode().upper()
1469 if options
.mode_filter
.upper() == 'ALL' \
1470 or options
.mode_filter
.upper() == test_mode \
1471 or (options
.mode_filter
.upper() == 'PASS' and test_mode
== ''):
1473 exit_code
= TestRunner(test_list
[n
], n
).run()
1478 def _internal_run_tests(test_list
, testnums
, parallel
, srcdir
, progress_func
):
1479 """Run the tests from TEST_LIST whose indices are listed in TESTNUMS.
1481 If we're running the tests in parallel spawn as much parallel processes
1482 as requested and gather the results in a temp. buffer when a child
1483 process is finished.
1490 # Some of the tests use sys.argv[0] to locate their test data
1491 # directory. Perhaps we should just be passing srcdir to the tests?
1493 sys
.argv
[0] = os
.path
.join(srcdir
, 'subversion', 'tests', 'cmdline',
1497 for i
, testnum
in enumerate(testnums
):
1499 if run_one_test(testnum
, test_list
) == 1:
1503 progress_func(i
+1, len(testnums
))
1505 number_queue
= queue
.Queue()
1506 for num
in testnums
:
1507 number_queue
.put(num
)
1509 threads
= [ TestSpawningThread(number_queue
, progress_func
,
1510 len(testnums
)) for i
in range(parallel
) ]
1517 # list of (index, result, stdout, stderr)
1520 results
+= t
.results
1523 # terminate the line of dots
1526 # all tests are finished, find out the result and print the logs.
1527 for (index
, result
, stdout_lines
, stderr_lines
) in results
:
1529 for line
in stdout_lines
:
1530 sys
.stdout
.write(line
)
1532 for line
in stderr_lines
:
1533 sys
.stdout
.write(line
)
1537 svntest
.sandbox
.cleanup_deferred_test_paths()
1541 def create_default_options():
1542 """Set the global options to the defaults, as provided by the argument
1547 def _create_parser():
1548 """Return a parser for our test suite."""
1549 def set_log_level(option
, opt
, value
, parser
, level
=None):
1551 # called from --verbose
1552 logger
.setLevel(level
)
1554 # called from --set-log-level
1555 logger
.setLevel(getattr(logging
, value
, None) or int(value
))
1558 _default_http_library
= 'serf'
1559 usage
= 'usage: %prog [options] [<test> ...]'
1560 parser
= optparse
.OptionParser(usage
=usage
)
1561 parser
.add_option('-l', '--list', action
='store_true', dest
='list_tests',
1562 help='Print test doc strings instead of running them')
1563 parser
.add_option('--milestone-filter', action
='store', dest
='milestone_filter',
1564 help='Limit --list to those with target milestone specified')
1565 parser
.add_option('-v', '--verbose', action
='callback',
1566 callback
=set_log_level
, callback_args
=(logging
.DEBUG
, ),
1567 help='Print binary command-lines (same as ' +
1568 '"--set-log-level logging.DEBUG")')
1569 parser
.add_option('-q', '--quiet', action
='store_true',
1570 help='Print only unexpected results (not with --verbose)')
1571 parser
.add_option('-p', '--parallel', action
='store_const',
1572 const
=default_num_threads
, dest
='parallel',
1573 help='Run the tests in parallel')
1574 parser
.add_option('-c', action
='store_true', dest
='is_child_process',
1575 help='Flag if we are running this python test as a ' +
1577 parser
.add_option('--mode-filter', action
='store', dest
='mode_filter',
1579 help='Limit tests to those with type specified (e.g. XFAIL)')
1580 parser
.add_option('--url', action
='store',
1581 help='Base url to the repos (e.g. svn://localhost)')
1582 parser
.add_option('--fs-type', action
='store',
1583 help='Subversion file system type (fsfs or bdb)')
1584 parser
.add_option('--cleanup', action
='store_true',
1585 help='Whether to clean up')
1586 parser
.add_option('--enable-sasl', action
='store_true',
1587 help='Whether to enable SASL authentication')
1588 parser
.add_option('--bin', action
='store', dest
='svn_bin',
1589 help='Use the svn binaries installed in this path')
1590 parser
.add_option('--use-jsvn', action
='store_true',
1591 help="Use the jsvn (SVNKit based) binaries. Can be " +
1592 "combined with --bin to point to a specific path")
1593 parser
.add_option('--http-library', action
='store',
1594 help="Make svn use this DAV library (neon or serf) if " +
1595 "it supports both, else assume it's using this " +
1596 "one; the default is " + _default_http_library
)
1597 parser
.add_option('--server-minor-version', type='int', action
='store',
1598 help="Set the minor version for the server ('3'..'%d')."
1600 parser
.add_option('--fsfs-packing', action
='store_true',
1601 help="Run 'svnadmin pack' automatically")
1602 parser
.add_option('--fsfs-sharding', action
='store', type='int',
1603 help='Default shard size (for fsfs)')
1604 parser
.add_option('--config-file', action
='store',
1605 help="Configuration file for tests.")
1606 parser
.add_option('--set-log-level', action
='callback', type='str',
1607 callback
=set_log_level
,
1608 help="Set log level (numerically or symbolically). " +
1609 "Symbolic levels are: CRITICAL, ERROR, WARNING, " +
1611 parser
.add_option('--log-with-timestamps', action
='store_true',
1612 help="Show timestamps in test log.")
1613 parser
.add_option('--keep-local-tmp', action
='store_true',
1614 help="Don't remove svn-test-work/local_tmp after test " +
1615 "run is complete. Useful for debugging failures.")
1616 parser
.add_option('--development', action
='store_true',
1617 help='Test development mode: provides more detailed ' +
1618 'test output and ignores all exceptions in the ' +
1619 'run_and_verify* functions. This option is only ' +
1620 'useful during test development!')
1621 parser
.add_option('--srcdir', action
='store', dest
='srcdir',
1622 help='Source directory.')
1623 parser
.add_option('--ssl-cert', action
='store',
1624 help='Path to SSL server certificate.')
1626 # most of the defaults are None, but some are other values, set them here
1627 parser
.set_defaults(
1628 server_minor_version
=SVN_VER_MINOR
,
1629 url
=file_scheme_prefix
+ \
1630 urllib
.pathname2url(os
.path
.abspath(os
.getcwd())),
1631 http_library
=_default_http_library
)
1636 def _parse_options(arglist
=sys
.argv
[1:]):
1637 """Parse the arguments in arg_list, and set the global options object with
1642 parser
= _create_parser()
1643 (options
, args
) = parser
.parse_args(arglist
)
1645 # some sanity checking
1646 if options
.fsfs_packing
and not options
.fsfs_sharding
:
1647 parser
.error("--fsfs-packing requires --fsfs-sharding")
1649 # If you change the below condition then change
1650 # ../../../../build/run_tests.py too.
1651 if options
.server_minor_version
not in range(3, SVN_VER_MINOR
+1):
1652 parser
.error("test harness only supports server minor versions 3-%d"
1656 if options
.url
[-1:] == '/': # Normalize url to have no trailing slash
1657 options
.test_area_url
= options
.url
[:-1]
1659 options
.test_area_url
= options
.url
1661 return (parser
, args
)
1664 def run_tests(test_list
, serial_only
= False):
1665 """Main routine to run all tests in TEST_LIST.
1667 NOTE: this function does not return. It does a sys.exit() with the
1668 appropriate exit code.
1671 sys
.exit(execute_tests(test_list
, serial_only
))
1673 def get_target_milestones_for_issues(issue_numbers
):
1674 xml_url
= "http://subversion.tigris.org/issues/xml.cgi?id="
1677 if isinstance(issue_numbers
, int):
1678 issue_numbers
= [str(issue_numbers
)]
1679 elif isinstance(issue_numbers
, str):
1680 issue_numbers
= [issue_numbers
]
1682 if issue_numbers
is None or len(issue_numbers
) == 0:
1685 for num
in issue_numbers
:
1686 xml_url
+= str(num
) + ','
1687 issue_dict
[str(num
)] = 'unknown'
1690 # Parse the xml for ISSUE_NO from the issue tracker into a Document.
1691 issue_xml_f
= urllib
.urlopen(xml_url
)
1693 print "WARNING: Unable to contact issue tracker; " \
1694 "milestones defaulting to 'unknown'."
1698 xmldoc
= xml
.dom
.minidom
.parse(issue_xml_f
)
1701 # Get the target milestone for each issue.
1702 issue_element
= xmldoc
.getElementsByTagName('issue')
1703 for i
in issue_element
:
1704 issue_id_element
= i
.getElementsByTagName('issue_id')
1705 issue_id
= issue_id_element
[0].childNodes
[0].nodeValue
1706 milestone_element
= i
.getElementsByTagName('target_milestone')
1707 milestone
= milestone_element
[0].childNodes
[0].nodeValue
1708 issue_dict
[issue_id
] = milestone
1710 print "ERROR: Unable to parse target milestones from issue tracker"
1716 class AbbreviatedFormatter(logging
.Formatter
):
1717 """A formatter with abbreviated loglevel indicators in the output.
1719 Use %(levelshort)s in the format string to get a single character
1720 representing the loglevel..
1724 logging
.CRITICAL
: 'C',
1725 logging
.ERROR
: 'E',
1726 logging
.WARNING
: 'W',
1728 logging
.DEBUG
: 'D',
1729 logging
.NOTSET
: '-',
1732 def format(self
, record
):
1733 record
.levelshort
= self
._level
_short
[record
.levelno
]
1734 return logging
.Formatter
.format(self
, record
)
1737 # Main func. This is the "entry point" that all the test scripts call
1738 # to run their list of tests.
1740 # This routine parses sys.argv to decide what to do.
1741 def execute_tests(test_list
, serial_only
= False, test_name
= None,
1742 progress_func
= None, test_selection
= []):
1743 """Similar to run_tests(), but just returns the exit code, rather than
1744 exiting the process. This function can be used when a caller doesn't
1745 want the process to die."""
1749 global pristine_greek_repos_url
1751 global svnadmin_binary
1752 global svnlook_binary
1753 global svnsync_binary
1754 global svndumpfilter_binary
1755 global svnversion_binary
1756 global svnmucc_binary
1760 sys
.argv
[0] = test_name
1764 # Initialize the LOGGER global variable so the option parsing can set
1765 # its loglevel, as appropriate.
1766 logger
= logging
.getLogger()
1768 # Did some chucklehead log something before we configured it? If they
1769 # did, then a default handler/formatter would get installed. We want
1770 # to be the one to install the first (and only) handler.
1771 for handler
in logger
.handlers
:
1772 if not isinstance(handler
.formatter
, AbbreviatedFormatter
):
1773 raise Exception('Logging occurred before configuration. Some code'
1774 ' path needs to be fixed. Examine the log output'
1775 ' to find what/where logged something.')
1778 # Override which tests to run from the commandline
1779 (parser
, args
) = _parse_options()
1780 test_selection
= args
1782 parser
= _create_parser()
1784 # If there are no handlers registered yet, then install our own with
1785 # our custom formatter. (anything currently installed *is* our handler
1787 if not logger
.handlers
:
1788 # Now that we have some options, let's get the logger configured before
1789 # doing anything more
1790 if options
.log_with_timestamps
:
1791 formatter
= AbbreviatedFormatter('%(levelshort)s:'
1792 ' [%(asctime)s] %(message)s',
1793 datefmt
='%Y-%m-%d %H:%M:%S')
1795 formatter
= AbbreviatedFormatter('%(levelshort)s: %(message)s')
1796 handler
= logging
.StreamHandler(sys
.stdout
)
1797 handler
.setFormatter(formatter
)
1798 logger
.addHandler(handler
)
1800 # parse the positional arguments (test nums, names)
1801 for arg
in test_selection
:
1804 testnums
.append(int(arg
))
1807 # Do nothing for now.
1812 # Check if the argument is a range
1813 numberstrings
= arg
.split(':');
1814 if len(numberstrings
) != 2:
1815 numberstrings
= arg
.split('-');
1816 if len(numberstrings
) != 2:
1818 left
= int(numberstrings
[0])
1819 right
= int(numberstrings
[1])
1823 for nr
in range(left
,right
+1):
1832 # Check if the argument is a function name, and translate
1833 # it to a number if possible
1834 for testnum
in list(range(1, len(test_list
))):
1835 test_case
= TestRunner(test_list
[testnum
], testnum
)
1836 if test_case
.get_function_name() == str(arg
):
1837 testnums
.append(testnum
)
1844 parser
.error("invalid test number, range of numbers, " +
1845 "or function '%s'\n" % arg
)
1847 # Calculate pristine_greek_repos_url from test_area_url.
1848 pristine_greek_repos_url
= options
.test_area_url
+ '/' + \
1849 urllib
.pathname2url(pristine_greek_repos_dir
)
1851 if options
.use_jsvn
:
1852 if options
.svn_bin
is None:
1853 options
.svn_bin
= ''
1854 svn_binary
= os
.path
.join(options
.svn_bin
, 'jsvn' + _bat
)
1855 svnadmin_binary
= os
.path
.join(options
.svn_bin
, 'jsvnadmin' + _bat
)
1856 svnlook_binary
= os
.path
.join(options
.svn_bin
, 'jsvnlook' + _bat
)
1857 svnsync_binary
= os
.path
.join(options
.svn_bin
, 'jsvnsync' + _bat
)
1858 svndumpfilter_binary
= os
.path
.join(options
.svn_bin
,
1859 'jsvndumpfilter' + _bat
)
1860 svnversion_binary
= os
.path
.join(options
.svn_bin
,
1861 'jsvnversion' + _bat
)
1862 svnmucc_binary
= os
.path
.join(options
.svn_bin
, 'jsvnmucc' + _bat
)
1865 svn_binary
= os
.path
.join(options
.svn_bin
, 'svn' + _exe
)
1866 svnadmin_binary
= os
.path
.join(options
.svn_bin
, 'svnadmin' + _exe
)
1867 svnlook_binary
= os
.path
.join(options
.svn_bin
, 'svnlook' + _exe
)
1868 svnsync_binary
= os
.path
.join(options
.svn_bin
, 'svnsync' + _exe
)
1869 svndumpfilter_binary
= os
.path
.join(options
.svn_bin
,
1870 'svndumpfilter' + _exe
)
1871 svnversion_binary
= os
.path
.join(options
.svn_bin
, 'svnversion' + _exe
)
1872 svnmucc_binary
= os
.path
.join(options
.svn_bin
, 'svnmucc' + _exe
)
1874 ######################################################################
1876 # Cleanup: if a previous run crashed or interrupted the python
1877 # interpreter, then `temp_dir' was never removed. This can cause wonkiness.
1878 if not options
.is_child_process
:
1879 safe_rmtree(temp_dir
, 1)
1882 # If no test numbers were listed explicitly, include all of them:
1883 testnums
= list(range(1, len(test_list
)))
1885 if options
.list_tests
:
1887 # If we want to list the target milestones, then get all the issues
1888 # associated with all the individual tests.
1889 milestones_dict
= None
1890 if options
.milestone_filter
:
1892 for testnum
in testnums
:
1893 issues
= TestRunner(test_list
[testnum
], testnum
).get_issues()
1894 test_mode
= TestRunner(test_list
[testnum
], testnum
).get_mode().upper()
1896 for issue
in issues
:
1897 if (options
.mode_filter
.upper() == 'ALL' or
1898 options
.mode_filter
.upper() == test_mode
or
1899 (options
.mode_filter
.upper() == 'PASS' and test_mode
== '')):
1900 issues_dict
[issue
]=issue
1901 milestones_dict
= get_target_milestones_for_issues(issues_dict
.keys())
1903 header
= "Test # Mode Test Description\n" \
1904 "------ ----- ----------------"
1905 printed_header
= False
1906 for testnum
in testnums
:
1907 test_mode
= TestRunner(test_list
[testnum
], testnum
).get_mode().upper()
1908 if options
.mode_filter
.upper() == 'ALL' \
1909 or options
.mode_filter
.upper() == test_mode \
1910 or (options
.mode_filter
.upper() == 'PASS' and test_mode
== ''):
1911 if not printed_header
:
1913 printed_header
= True
1914 TestRunner(test_list
[testnum
], testnum
).list(milestones_dict
)
1915 # We are simply listing the tests so always exit with success.
1918 # don't run tests in parallel when the tests don't support it or there
1919 # are only a few tests to run.
1920 if serial_only
or len(testnums
) < 2:
1921 options
.parallel
= 0
1923 if not options
.is_child_process
:
1924 # Build out the default configuration directory
1925 create_config_dir(default_config_dir
,
1926 ssl_cert
=options
.ssl_cert
,
1927 ssl_url
=options
.test_area_url
)
1929 # Setup the pristine repository
1930 svntest
.actions
.setup_pristine_greek_repository()
1933 exit_code
= _internal_run_tests(test_list
, testnums
, options
.parallel
,
1934 options
.srcdir
, progress_func
)
1936 # Remove all scratchwork: the 'pristine' repository, greek tree, etc.
1937 # This ensures that an 'import' will happen the next time we run.
1938 if not options
.is_child_process
and not options
.keep_local_tmp
:
1939 safe_rmtree(temp_dir
, 1)
1941 # Cleanup after ourselves.
1942 svntest
.sandbox
.cleanup_deferred_test_paths()
1944 # Return the appropriate exit code from the tests.