svntest/sandbox.py

   1 #
   2 # sandbox.py :  tools for manipulating a test's working area ("a sandbox")
   3 #
   4 # ====================================================================
   5 #    Licensed to the Apache Software Foundation (ASF) under one
   6 #    or more contributor license agreements.  See the NOTICE file
   7 #    distributed with this work for additional information
   8 #    regarding copyright ownership.  The ASF licenses this file
   9 #    to you under the Apache License, Version 2.0 (the
  10 #    "License"); you may not use this file except in compliance
  11 #    with the License.  You may obtain a copy of the License at
  12 #
  13 #      http://www.apache.org/licenses/LICENSE-2.0
  14 #
  15 #    Unless required by applicable law or agreed to in writing,
  16 #    software distributed under the License is distributed on an
  17 #    "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
  18 #    KIND, either express or implied.  See the License for the
  19 #    specific language governing permissions and limitations
  20 #    under the License.
  21 # ====================================================================
  22 #
  23
  24 import os
  25 import shutil
  26 import copy
  27 import urllib
  28 import logging
  29
  30 import svntest
  31
  32 logger = logging.getLogger()
  33
  34
  35 class Sandbox:
  36   """Manages a sandbox (one or more repository/working copy pairs) for
  37   a test to operate within."""
  38
  39   dependents = None
  40
  41   def __init__(self, module, idx):
  42     self.test_paths = []
  43
  44     self._set_name("%s-%d" % (module, idx))
  45     # This flag is set to True by build() and returned by is_built()
  46     self._is_built = False
  47
  48     # Create an empty directory for temporary files
  49     self.tmp_dir = self.add_wc_path('tmp', remove=True)
  50     os.mkdir(self.tmp_dir)
  51
  52   def _set_name(self, name, read_only=False):
  53     """A convenience method for renaming a sandbox, useful when
  54     working with multiple repositories in the same unit test."""
  55     if not name is None:
  56       self.name = name
  57     self.read_only = read_only
  58     self.wc_dir = os.path.join(svntest.main.general_wc_dir, self.name)
  59     self.add_test_path(self.wc_dir)
  60     if not read_only:
  61       self.repo_dir = os.path.join(svntest.main.general_repo_dir, self.name)
  62       self.repo_url = (svntest.main.options.test_area_url + '/'
  63                        + urllib.pathname2url(self.repo_dir))
  64       self.add_test_path(self.repo_dir)
  65     else:
  66       self.repo_dir = svntest.main.pristine_greek_repos_dir
  67       self.repo_url = svntest.main.pristine_greek_repos_url
  68
  69     ### TODO: Move this into to the build() method
  70     # For dav tests we need a single authz file which must be present,
  71     # so we recreate it each time a sandbox is created with some default
  72     # contents, making sure that an empty file is never present
  73     if self.repo_url.startswith("http"):
  74       # this dir doesn't exist out of the box, so we may have to make it
  75       if not os.path.exists(svntest.main.work_dir):
  76         os.makedirs(svntest.main.work_dir)
  77       self.authz_file = os.path.join(svntest.main.work_dir, "authz")
  78       tmp_authz_file = os.path.join(svntest.main.work_dir, "authz-" + self.name)
  79       open(tmp_authz_file, 'w').write("[/]\n* = rw\n")
  80       shutil.move(tmp_authz_file, self.authz_file)
  81
  82     # For svnserve tests we have a per-repository authz file, and it
  83     # doesn't need to be there in order for things to work, so we don't
  84     # have any default contents.
  85     elif self.repo_url.startswith("svn"):
  86       self.authz_file = os.path.join(self.repo_dir, "conf", "authz")
  87
  88   def clone_dependent(self, copy_wc=False):
  89     """A convenience method for creating a near-duplicate of this
  90     sandbox, useful when working with multiple repositories in the
  91     same unit test.  If COPY_WC is true, make an exact copy of this
  92     sandbox's working copy at the new sandbox's working copy
  93     directory.  Any necessary cleanup operations are triggered by
  94     cleanup of the original sandbox."""
  95
  96     if not self.dependents:
  97       self.dependents = []
  98     clone = copy.deepcopy(self)
  99     self.dependents.append(clone)
 100     clone._set_name("%s-%d" % (self.name, len(self.dependents)))
 101     if copy_wc:
 102       self.add_test_path(clone.wc_dir)
 103       shutil.copytree(self.wc_dir, clone.wc_dir, symlinks=True)
 104     return clone
 105
 106   def build(self, name=None, create_wc=True, read_only=False,
 107             minor_version=None):
 108     """Make a 'Greek Tree' repo (or refer to the central one if READ_ONLY),
 109        and check out a WC from it (unless CREATE_WC is false). Change the
 110        sandbox's name to NAME. See actions.make_repo_and_wc() for details."""
 111     self._set_name(name, read_only)
 112     svntest.actions.make_repo_and_wc(self, create_wc, read_only, minor_version)
 113     self._is_built = True
 114
 115   def authz_name(self, repo_dir=None):
 116     "return this sandbox's name for use in an authz file"
 117     repo_dir = repo_dir or self.repo_dir
 118     if self.repo_url.startswith("http"):
 119       return os.path.basename(repo_dir)
 120     else:
 121       return repo_dir.replace('\\', '/')
 122
 123   def add_test_path(self, path, remove=True):
 124     self.test_paths.append(path)
 125     if remove:
 126       svntest.main.safe_rmtree(path)
 127
 128   def add_repo_path(self, suffix, remove=True):
 129     """Generate a path, under the general repositories directory, with
 130        a name that ends in SUFFIX, e.g. suffix="2" -> ".../basic_tests.2".
 131        If REMOVE is true, remove anything currently on disk at that path.
 132        Remember that path so that the automatic clean-up mechanism can
 133        delete it at the end of the test. Generate a repository URL to
 134        refer to a repository at that path. Do not create a repository.
 135        Return (REPOS-PATH, REPOS-URL)."""
 136     path = (os.path.join(svntest.main.general_repo_dir, self.name)
 137             + '.' + suffix)
 138     url = svntest.main.options.test_area_url + \
 139                                         '/' + urllib.pathname2url(path)
 140     self.add_test_path(path, remove)
 141     return path, url
 142
 143   def add_wc_path(self, suffix, remove=True):
 144     """Generate a path, under the general working copies directory, with
 145        a name that ends in SUFFIX, e.g. suffix="2" -> ".../basic_tests.2".
 146        If REMOVE is true, remove anything currently on disk at that path.
 147        Remember that path so that the automatic clean-up mechanism can
 148        delete it at the end of the test. Do not create a working copy.
 149        Return the generated WC-PATH."""
 150     path = self.wc_dir + '.' + suffix
 151     self.add_test_path(path, remove)
 152     return path
 153
 154   tempname_offs = 0 # Counter for get_tempname
 155
 156   def get_tempname(self, prefix='tmp'):
 157     """Get a stable name for a temporary file that will be removed after
 158        running the test"""
 159
 160     self.tempname_offs = self.tempname_offs + 1
 161
 162     return os.path.join(self.tmp_dir, '%s-%s' % (prefix, self.tempname_offs))
 163
 164   def cleanup_test_paths(self):
 165     "Clean up detritus from this sandbox, and any dependents."
 166     if self.dependents:
 167       # Recursively cleanup any dependent sandboxes.
 168       for sbox in self.dependents:
 169         sbox.cleanup_test_paths()
 170     # cleanup all test specific working copies and repositories
 171     for path in self.test_paths:
 172       if not path is svntest.main.pristine_greek_repos_dir:
 173         _cleanup_test_path(path)
 174
 175   def is_built(self):
 176     "Returns True when build() has been called on this instance."
 177     return self._is_built
 178
 179   def ospath(self, relpath, wc_dir=None):
 180     """Return RELPATH converted to an OS-style path relative to the WC dir
 181        of this sbox, or relative to OS-style path WC_DIR if supplied."""
 182     if wc_dir is None:
 183       wc_dir = self.wc_dir
 184     return os.path.join(wc_dir, svntest.wc.to_ospath(relpath))
 185
 186   def ospaths(self, relpaths, wc_dir=None):
 187     """Return a list of RELPATHS but with each path converted to an OS-style
 188        path relative to the WC dir of this sbox, or relative to OS-style
 189        path WC_DIR if supplied."""
 190     return [self.ospath(rp, wc_dir) for rp in relpaths]
 191
 192   def redirected_root_url(self, temporary=False):
 193     """If TEMPORARY is set, return the URL which should be configured
 194        to temporarily redirect to the root of this repository;
 195        otherwise, return the URL which should be configured to
 196        permanent redirect there.  (Assumes that the sandbox is not
 197        read-only.)"""
 198     assert not self.read_only
 199     assert self.repo_url.startswith("http")
 200     parts = self.repo_url.rsplit('/', 1)
 201     return '%s/REDIRECT-%s-%s' % (parts[0],
 202                                   temporary and 'TEMP' or 'PERM',
 203                                   parts[1])
 204
 205   def simple_update(self, target=None, revision='HEAD'):
 206     """Update the WC or TARGET.
 207        TARGET is a relpath relative to the WC."""
 208     if target is None:
 209       target = self.wc_dir
 210     else:
 211       target = self.ospath(target)
 212     svntest.main.run_svn(False, 'update', target, '-r', revision)
 213
 214   def simple_switch(self, url, target=None):
 215     """Switch the WC or TARGET to URL.
 216        TARGET is a relpath relative to the WC."""
 217     if target is None:
 218       target = self.wc_dir
 219     else:
 220       target = self.ospath(target)
 221     svntest.main.run_svn(False, 'switch', url, target, '--ignore-ancestry')
 222
 223   def simple_commit(self, target=None, message=None):
 224     """Commit the WC or TARGET, with a default or supplied log message.
 225        Raise if the exit code is non-zero or there is output on stderr.
 226        TARGET is a relpath relative to the WC."""
 227     assert not self.read_only
 228     if target is None:
 229       target = self.wc_dir
 230     else:
 231       target = self.ospath(target)
 232     if message is None:
 233       message = svntest.main.make_log_msg()
 234     svntest.main.run_svn(False, 'commit', '-m', message,
 235                          target)
 236
 237   def simple_rm(self, *targets):
 238     """Schedule TARGETS for deletion.
 239        TARGETS are relpaths relative to the WC."""
 240     assert len(targets) > 0
 241     targets = self.ospaths(targets)
 242     svntest.main.run_svn(False, 'rm', *targets)
 243
 244   def simple_mkdir(self, *targets):
 245     """Create TARGETS as directories scheduled for addition.
 246        TARGETS are relpaths relative to the WC."""
 247     assert len(targets) > 0
 248     targets = self.ospaths(targets)
 249     svntest.main.run_svn(False, 'mkdir', *targets)
 250
 251   def simple_add(self, *targets):
 252     """Schedule TARGETS for addition.
 253        TARGETS are relpaths relative to the WC."""
 254     assert len(targets) > 0
 255     targets = self.ospaths(targets)
 256     svntest.main.run_svn(False, 'add', *targets)
 257
 258   def simple_revert(self, *targets):
 259     """Revert TARGETS.
 260        TARGETS are relpaths relative to the WC."""
 261     assert len(targets) > 0
 262     targets = self.ospaths(targets)
 263     svntest.main.run_svn(False, 'revert', *targets)
 264
 265   def simple_propset(self, name, value, *targets):
 266     """Set property NAME to VALUE on TARGETS.
 267        TARGETS are relpaths relative to the WC."""
 268     assert len(targets) > 0
 269     targets = self.ospaths(targets)
 270     svntest.main.run_svn(False, 'propset', name, value, *targets)
 271
 272   def simple_propdel(self, name, *targets):
 273     """Delete property NAME from TARGETS.
 274        TARGETS are relpaths relative to the WC."""
 275     assert len(targets) > 0
 276     targets = self.ospaths(targets)
 277     svntest.main.run_svn(False, 'propdel', name, *targets)
 278
 279   def simple_propget(self, name, target):
 280     """Return the value of the property NAME on TARGET.
 281        TARGET is a relpath relative to the WC."""
 282     target = self.ospath(target)
 283     exit, out, err = svntest.main.run_svn(False, 'propget',
 284                                           '--strict', name, target)
 285     return ''.join(out)
 286
 287   def simple_proplist(self, target):
 288     """Return a dictionary mapping property name to property value, of the
 289        properties on TARGET.
 290        TARGET is a relpath relative to the WC."""
 291     target = self.ospath(target)
 292     exit, out, err = svntest.main.run_svn(False, 'proplist',
 293                                           '--verbose', '--quiet', target)
 294     props = {}
 295     for line in out:
 296       line = line.rstrip('\r\n')
 297       if line[2] != ' ':  # property name
 298         name = line[2:]
 299         val = None
 300       elif line.startswith('    '):  # property value
 301         if val is None:
 302           val = line[4:]
 303         else:
 304           val += '\n' + line[4:]
 305         props[name] = val
 306       else:
 307         raise Exception("Unexpected line '" + line + "' in proplist output" + str(out))
 308     return props
 309
 310   def simple_copy(self, source, dest):
 311     """Copy SOURCE to DEST in the WC.
 312        SOURCE and DEST are relpaths relative to the WC."""
 313     source = self.ospath(source)
 314     dest = self.ospath(dest)
 315     svntest.main.run_svn(False, 'copy', source, dest)
 316
 317   def simple_move(self, source, dest):
 318     """Move SOURCE to DEST in the WC.
 319        SOURCE and DEST are relpaths relative to the WC."""
 320     source = self.ospath(source)
 321     dest = self.ospath(dest)
 322     svntest.main.run_svn(False, 'move', source, dest)
 323
 324   def simple_repo_copy(self, source, dest):
 325     """Copy SOURCE to DEST in the repository, committing the result with a
 326        default log message.
 327        SOURCE and DEST are relpaths relative to the repo root."""
 328     svntest.main.run_svn(False, 'copy', '-m', svntest.main.make_log_msg(),
 329                          self.repo_url + '/' + source,
 330                          self.repo_url + '/' + dest)
 331
 332   def simple_append(self, dest, contents, truncate=False):
 333     """Append CONTENTS to file DEST, optionally truncating it first.
 334        DEST is a relpath relative to the WC."""
 335     open(self.ospath(dest), truncate and 'w' or 'a').write(contents)
 336
 337
 338 def is_url(target):
 339   return (target.startswith('^/')
 340           or target.startswith('file://')
 341           or target.startswith('http://')
 342           or target.startswith('https://')
 343           or target.startswith('svn://')
 344           or target.startswith('svn+ssh://'))
 345
 346
 347 _deferred_test_paths = []
 348
 349 def cleanup_deferred_test_paths():
 350   global _deferred_test_paths
 351   test_paths = _deferred_test_paths
 352   _deferred_test_paths = []
 353   for path in test_paths:
 354     _cleanup_test_path(path, True)
 355
 356
 357 def _cleanup_test_path(path, retrying=False):
 358   if retrying:
 359     logger.info("CLEANUP: RETRY: %s", path)
 360   else:
 361     logger.info("CLEANUP: %s", path)
 362
 363   try:
 364     svntest.main.safe_rmtree(path)
 365   except:
 366     logger.info("WARNING: cleanup failed, will try again later")
 367     _deferred_test_paths.append(path)