buildbot/steps/source.py

   1 # -*- test-case-name: buildbot.test.test_vc -*-
   2
   3 import warnings
   4 from email.Utils import formatdate
   5 from twisted.python import log
   6 from buildbot.process.buildstep import LoggingBuildStep, LoggedRemoteCommand
   7 from buildbot.interfaces import BuildSlaveTooOldError
   8 from buildbot.status.builder import SKIPPED
   9
  10
  11 class Source(LoggingBuildStep):
  12     """This is a base class to generate a source tree in the buildslave.
  13     Each version control system has a specialized subclass, and is expected
  14     to override __init__ and implement computeSourceRevision() and
  15     startVC(). The class as a whole builds up the self.args dictionary, then
  16     starts a LoggedRemoteCommand with those arguments.
  17     """
  18
  19     # if the checkout fails, there's no point in doing anything else
  20     haltOnFailure = True
  21     notReally = False
  22
  23     branch = None # the default branch, should be set in __init__
  24
  25     def __init__(self, workdir, mode='update', alwaysUseLatest=False,
  26                  timeout=20*60, retry=None, **kwargs):
  27         """
  28         @type  workdir: string
  29         @param workdir: local directory (relative to the Builder's root)
  30                         where the tree should be placed
  31
  32         @type  mode: string
  33         @param mode: the kind of VC operation that is desired:
  34            - 'update': specifies that the checkout/update should be
  35              performed directly into the workdir. Each build is performed
  36              in the same directory, allowing for incremental builds. This
  37              minimizes disk space, bandwidth, and CPU time. However, it
  38              may encounter problems if the build process does not handle
  39              dependencies properly (if you must sometimes do a 'clean
  40              build' to make sure everything gets compiled), or if source
  41              files are deleted but generated files can influence test
  42              behavior (e.g. python's .pyc files), or when source
  43              directories are deleted but generated files prevent CVS from
  44              removing them.
  45
  46            - 'copy': specifies that the source-controlled workspace
  47              should be maintained in a separate directory (called the
  48              'copydir'), using checkout or update as necessary. For each
  49              build, a new workdir is created with a copy of the source
  50              tree (rm -rf workdir; cp -r copydir workdir). This doubles
  51              the disk space required, but keeps the bandwidth low
  52              (update instead of a full checkout). A full 'clean' build
  53              is performed each time.  This avoids any generated-file
  54              build problems, but is still occasionally vulnerable to
  55              problems such as a CVS repository being manually rearranged
  56              (causing CVS errors on update) which are not an issue with
  57              a full checkout.
  58
  59            - 'clobber': specifies that the working directory should be
  60              deleted each time, necessitating a full checkout for each
  61              build. This insures a clean build off a complete checkout,
  62              avoiding any of the problems described above, but is
  63              bandwidth intensive, as the whole source tree must be
  64              pulled down for each build.
  65
  66            - 'export': is like 'clobber', except that e.g. the 'cvs
  67              export' command is used to create the working directory.
  68              This command removes all VC metadata files (the
  69              CVS/.svn/{arch} directories) from the tree, which is
  70              sometimes useful for creating source tarballs (to avoid
  71              including the metadata in the tar file). Not all VC systems
  72              support export.
  73
  74         @type  alwaysUseLatest: boolean
  75         @param alwaysUseLatest: whether to always update to the most
  76         recent available sources for this build.
  77
  78         Normally the Source step asks its Build for a list of all
  79         Changes that are supposed to go into the build, then computes a
  80         'source stamp' (revision number or timestamp) that will cause
  81         exactly that set of changes to be present in the checked out
  82         tree. This is turned into, e.g., 'cvs update -D timestamp', or
  83         'svn update -r revnum'. If alwaysUseLatest=True, bypass this
  84         computation and always update to the latest available sources
  85         for each build.
  86
  87         The source stamp helps avoid a race condition in which someone
  88         commits a change after the master has decided to start a build
  89         but before the slave finishes checking out the sources. At best
  90         this results in a build which contains more changes than the
  91         buildmaster thinks it has (possibly resulting in the wrong
  92         person taking the blame for any problems that result), at worst
  93         is can result in an incoherent set of sources (splitting a
  94         non-atomic commit) which may not build at all.
  95
  96         @type  retry: tuple of ints (delay, repeats) (or None)
  97         @param retry: if provided, VC update failures are re-attempted up
  98                       to REPEATS times, with DELAY seconds between each
  99                       attempt. Some users have slaves with poor connectivity
 100                       to their VC repository, and they say that up to 80% of
 101                       their build failures are due to transient network
 102                       failures that could be handled by simply retrying a
 103                       couple times.
 104
 105         """
 106
 107         LoggingBuildStep.__init__(self, **kwargs)
 108
 109         assert mode in ("update", "copy", "clobber", "export")
 110         if retry:
 111             delay, repeats = retry
 112             assert isinstance(repeats, int)
 113             assert repeats > 0
 114         self.args = {'mode': mode,
 115                      'workdir': workdir,
 116                      'timeout': timeout,
 117                      'retry': retry,
 118                      'patch': None, # set during .start
 119                      }
 120         self.alwaysUseLatest = alwaysUseLatest
 121
 122         # Compute defaults for descriptions:
 123         description = ["updating"]
 124         descriptionDone = ["update"]
 125         if mode == "clobber":
 126             description = ["checkout"]
 127             # because checkingouting takes too much space
 128             descriptionDone = ["checkout"]
 129         elif mode == "export":
 130             description = ["exporting"]
 131             descriptionDone = ["export"]
 132         self.description = description
 133         self.descriptionDone = descriptionDone
 134
 135     def describe(self, done=False):
 136         if done:
 137             return self.descriptionDone
 138         return self.description
 139
 140     def computeSourceRevision(self, changes):
 141         """Each subclass must implement this method to do something more
 142         precise than -rHEAD every time. For version control systems that use
 143         repository-wide change numbers (SVN, P4), this can simply take the
 144         maximum such number from all the changes involved in this build. For
 145         systems that do not (CVS), it needs to create a timestamp based upon
 146         the latest Change, the Build's treeStableTimer, and an optional
 147         self.checkoutDelay value."""
 148         return None
 149
 150     def start(self):
 151         if self.notReally:
 152             log.msg("faking %s checkout/update" % self.name)
 153             self.step_status.setColor("green")
 154             self.step_status.setText(["fake", self.name, "successful"])
 155             self.addCompleteLog("log",
 156                                 "Faked %s checkout/update 'successful'\n" \
 157                                 % self.name)
 158             return SKIPPED
 159
 160         # what source stamp would this build like to use?
 161         s = self.build.getSourceStamp()
 162         # if branch is None, then use the Step's "default" branch
 163         branch = s.branch or self.branch
 164         # if revision is None, use the latest sources (-rHEAD)
 165         revision = s.revision
 166         if not revision and not self.alwaysUseLatest:
 167             revision = self.computeSourceRevision(s.changes)
 168         # if patch is None, then do not patch the tree after checkout
 169
 170         # 'patch' is None or a tuple of (patchlevel, diff)
 171         patch = s.patch
 172         if patch:
 173             self.addCompleteLog("patch", patch[1])
 174
 175         self.startVC(branch, revision, patch)
 176
 177     def commandComplete(self, cmd):
 178         got_revision = None
 179         if cmd.updates.has_key("got_revision"):
 180             got_revision = cmd.updates["got_revision"][-1]
 181         self.setProperty("got_revision", got_revision)
 182
 183
 184
 185 class CVS(Source):
 186     """I do CVS checkout/update operations.
 187
 188     Note: if you are doing anonymous/pserver CVS operations, you will need
 189     to manually do a 'cvs login' on each buildslave before the slave has any
 190     hope of success. XXX: fix then, take a cvs password as an argument and
 191     figure out how to do a 'cvs login' on each build
 192     """
 193
 194     name = "cvs"
 195
 196     #progressMetrics = ('output',)
 197     #
 198     # additional things to track: update gives one stderr line per directory
 199     # (starting with 'cvs server: Updating ') (and is fairly stable if files
 200     # is empty), export gives one line per directory (starting with 'cvs
 201     # export: Updating ') and another line per file (starting with U). Would
 202     # be nice to track these, requires grepping LogFile data for lines,
 203     # parsing each line. Might be handy to have a hook in LogFile that gets
 204     # called with each complete line.
 205
 206     def __init__(self, cvsroot, cvsmodule,
 207                  global_options=[], branch=None, checkoutDelay=None,
 208                  login=None,
 209                  clobber=0, export=0, copydir=None,
 210                  **kwargs):
 211
 212         """
 213         @type  cvsroot: string
 214         @param cvsroot: CVS Repository from which the source tree should
 215                         be obtained. '/home/warner/Repository' for local
 216                         or NFS-reachable repositories,
 217                         ':pserver:anon@foo.com:/cvs' for anonymous CVS,
 218                         'user@host.com:/cvs' for non-anonymous CVS or
 219                         CVS over ssh. Lots of possibilities, check the
 220                         CVS documentation for more.
 221
 222         @type  cvsmodule: string
 223         @param cvsmodule: subdirectory of CVS repository that should be
 224                           retrieved
 225
 226         @type  login: string or None
 227         @param login: if not None, a string which will be provided as a
 228                       password to the 'cvs login' command, used when a
 229                       :pserver: method is used to access the repository.
 230                       This login is only needed once, but must be run
 231                       each time (just before the CVS operation) because
 232                       there is no way for the buildslave to tell whether
 233                       it was previously performed or not.
 234
 235         @type  branch: string
 236         @param branch: the default branch name, will be used in a '-r'
 237                        argument to specify which branch of the source tree
 238                        should be used for this checkout. Defaults to None,
 239                        which means to use 'HEAD'.
 240
 241         @type  checkoutDelay: int or None
 242         @param checkoutDelay: if not None, the number of seconds to put
 243                               between the last known Change and the
 244                               timestamp given to the -D argument. This
 245                               defaults to exactly half of the parent
 246                               Build's .treeStableTimer, but it could be
 247                               set to something else if your CVS change
 248                               notification has particularly weird
 249                               latency characteristics.
 250
 251         @type  global_options: list of strings
 252         @param global_options: these arguments are inserted in the cvs
 253                                command line, before the
 254                                'checkout'/'update' command word. See
 255                                'cvs --help-options' for a list of what
 256                                may be accepted here.  ['-r'] will make
 257                                the checked out files read only. ['-r',
 258                                '-R'] will also assume the repository is
 259                                read-only (I assume this means it won't
 260                                use locks to insure atomic access to the
 261                                ,v files)."""
 262
 263         self.checkoutDelay = checkoutDelay
 264         self.branch = branch
 265
 266         if not kwargs.has_key('mode') and (clobber or export or copydir):
 267             # deal with old configs
 268             warnings.warn("Please use mode=, not clobber/export/copydir",
 269                           DeprecationWarning)
 270             if export:
 271                 kwargs['mode'] = "export"
 272             elif clobber:
 273                 kwargs['mode'] = "clobber"
 274             elif copydir:
 275                 kwargs['mode'] = "copy"
 276             else:
 277                 kwargs['mode'] = "update"
 278
 279         Source.__init__(self, **kwargs)
 280
 281         self.args.update({'cvsroot': cvsroot,
 282                           'cvsmodule': cvsmodule,
 283                           'global_options': global_options,
 284                           'login': login,
 285                           })
 286
 287     def computeSourceRevision(self, changes):
 288         if not changes:
 289             return None
 290         lastChange = max([c.when for c in changes])
 291         if self.checkoutDelay is not None:
 292             when = lastChange + self.checkoutDelay
 293         else:
 294             lastSubmit = max([r.submittedAt for r in self.build.requests])
 295             when = (lastChange + lastSubmit) / 2
 296         return formatdate(when)
 297
 298     def startVC(self, branch, revision, patch):
 299         if self.slaveVersionIsOlderThan("cvs", "1.39"):
 300             # the slave doesn't know to avoid re-using the same sourcedir
 301             # when the branch changes. We have no way of knowing which branch
 302             # the last build used, so if we're using a non-default branch and
 303             # either 'update' or 'copy' modes, it is safer to refuse to
 304             # build, and tell the user they need to upgrade the buildslave.
 305             if (branch != self.branch
 306                 and self.args['mode'] in ("update", "copy")):
 307                 m = ("This buildslave (%s) does not know about multiple "
 308                      "branches, and using mode=%s would probably build the "
 309                      "wrong tree. "
 310                      "Refusing to build. Please upgrade the buildslave to "
 311                      "buildbot-0.7.0 or newer." % (self.build.slavename,
 312                                                    self.args['mode']))
 313                 log.msg(m)
 314                 raise BuildSlaveTooOldError(m)
 315
 316         if branch is None:
 317             branch = "HEAD"
 318         self.args['branch'] = branch
 319         self.args['revision'] = revision
 320         self.args['patch'] = patch
 321
 322         if self.args['branch'] == "HEAD" and self.args['revision']:
 323             # special case. 'cvs update -r HEAD -D today' gives no files
 324             # TODO: figure out why, see if it applies to -r BRANCH
 325             self.args['branch'] = None
 326
 327         # deal with old slaves
 328         warnings = []
 329         slavever = self.slaveVersion("cvs", "old")
 330
 331         if slavever == "old":
 332             # 0.5.0
 333             if self.args['mode'] == "export":
 334                 self.args['export'] = 1
 335             elif self.args['mode'] == "clobber":
 336                 self.args['clobber'] = 1
 337             elif self.args['mode'] == "copy":
 338                 self.args['copydir'] = "source"
 339             self.args['tag'] = self.args['branch']
 340             assert not self.args['patch'] # 0.5.0 slave can't do patch
 341
 342         cmd = LoggedRemoteCommand("cvs", self.args)
 343         self.startCommand(cmd, warnings)
 344
 345
 346 class SVN(Source):
 347     """I perform Subversion checkout/update operations."""
 348
 349     name = 'svn'
 350
 351     def __init__(self, svnurl=None, baseURL=None, defaultBranch=None,
 352                  directory=None, **kwargs):
 353         """
 354         @type  svnurl: string
 355         @param svnurl: the URL which points to the Subversion server,
 356                        combining the access method (HTTP, ssh, local file),
 357                        the repository host/port, the repository path, the
 358                        sub-tree within the repository, and the branch to
 359                        check out. Using C{svnurl} does not enable builds of
 360                        alternate branches: use C{baseURL} to enable this.
 361                        Use exactly one of C{svnurl} and C{baseURL}.
 362
 363         @param baseURL: if branches are enabled, this is the base URL to
 364                         which a branch name will be appended. It should
 365                         probably end in a slash. Use exactly one of
 366                         C{svnurl} and C{baseURL}.
 367
 368         @param defaultBranch: if branches are enabled, this is the branch
 369                               to use if the Build does not specify one
 370                               explicitly. It will simply be appended
 371                               to C{baseURL} and the result handed to
 372                               the SVN command.
 373         """
 374
 375         if not kwargs.has_key('workdir') and directory is not None:
 376             # deal with old configs
 377             warnings.warn("Please use workdir=, not directory=",
 378                           DeprecationWarning)
 379             kwargs['workdir'] = directory
 380
 381         self.svnurl = svnurl
 382         self.baseURL = baseURL
 383         self.branch = defaultBranch
 384
 385         Source.__init__(self, **kwargs)
 386
 387         if not svnurl and not baseURL:
 388             raise ValueError("you must use exactly one of svnurl and baseURL")
 389
 390
 391     def computeSourceRevision(self, changes):
 392         if not changes:
 393             return None
 394         lastChange = max([int(c.revision) for c in changes])
 395         return lastChange
 396
 397     def startVC(self, branch, revision, patch):
 398
 399         # handle old slaves
 400         warnings = []
 401         slavever = self.slaveVersion("svn", "old")
 402         if not slavever:
 403             m = "slave does not have the 'svn' command"
 404             raise BuildSlaveTooOldError(m)
 405
 406         if self.slaveVersionIsOlderThan("svn", "1.39"):
 407             # the slave doesn't know to avoid re-using the same sourcedir
 408             # when the branch changes. We have no way of knowing which branch
 409             # the last build used, so if we're using a non-default branch and
 410             # either 'update' or 'copy' modes, it is safer to refuse to
 411             # build, and tell the user they need to upgrade the buildslave.
 412             if (branch != self.branch
 413                 and self.args['mode'] in ("update", "copy")):
 414                 m = ("This buildslave (%s) does not know about multiple "
 415                      "branches, and using mode=%s would probably build the "
 416                      "wrong tree. "
 417                      "Refusing to build. Please upgrade the buildslave to "
 418                      "buildbot-0.7.0 or newer." % (self.build.slavename,
 419                                                    self.args['mode']))
 420                 raise BuildSlaveTooOldError(m)
 421
 422         if slavever == "old":
 423             # 0.5.0 compatibility
 424             if self.args['mode'] in ("clobber", "copy"):
 425                 # TODO: use some shell commands to make up for the
 426                 # deficiency, by blowing away the old directory first (thus
 427                 # forcing a full checkout)
 428                 warnings.append("WARNING: this slave can only do SVN updates"
 429                                 ", not mode=%s\n" % self.args['mode'])
 430                 log.msg("WARNING: this slave only does mode=update")
 431             if self.args['mode'] == "export":
 432                 raise BuildSlaveTooOldError("old slave does not have "
 433                                             "mode=export")
 434             self.args['directory'] = self.args['workdir']
 435             if revision is not None:
 436                 # 0.5.0 can only do HEAD. We have no way of knowing whether
 437                 # the requested revision is HEAD or not, and for
 438                 # slowly-changing trees this will probably do the right
 439                 # thing, so let it pass with a warning
 440                 m = ("WARNING: old slave can only update to HEAD, not "
 441                      "revision=%s" % revision)
 442                 log.msg(m)
 443                 warnings.append(m + "\n")
 444             revision = "HEAD" # interprets this key differently
 445             if patch:
 446                 raise BuildSlaveTooOldError("old slave can't do patch")
 447
 448         if self.svnurl:
 449             assert not branch # we need baseURL= to use branches
 450             self.args['svnurl'] = self.svnurl
 451         else:
 452             self.args['svnurl'] = self.baseURL + branch
 453         self.args['revision'] = revision
 454         self.args['patch'] = patch
 455
 456         revstuff = []
 457         if branch is not None and branch != self.branch:
 458             revstuff.append("[branch]")
 459         if revision is not None:
 460             revstuff.append("r%s" % revision)
 461         self.description.extend(revstuff)
 462         self.descriptionDone.extend(revstuff)
 463
 464         cmd = LoggedRemoteCommand("svn", self.args)
 465         self.startCommand(cmd, warnings)
 466
 467
 468 class Darcs(Source):
 469     """Check out a source tree from a Darcs repository at 'repourl'.
 470
 471     To the best of my knowledge, Darcs has no concept of file modes. This
 472     means the eXecute-bit will be cleared on all source files. As a result,
 473     you may need to invoke configuration scripts with something like:
 474
 475     C{s(step.Configure, command=['/bin/sh', './configure'])}
 476     """
 477
 478     name = "darcs"
 479
 480     def __init__(self, repourl=None, baseURL=None, defaultBranch=None,
 481                  **kwargs):
 482         """
 483         @type  repourl: string
 484         @param repourl: the URL which points at the Darcs repository. This
 485                         is used as the default branch. Using C{repourl} does
 486                         not enable builds of alternate branches: use
 487                         C{baseURL} to enable this. Use either C{repourl} or
 488                         C{baseURL}, not both.
 489
 490         @param baseURL: if branches are enabled, this is the base URL to
 491                         which a branch name will be appended. It should
 492                         probably end in a slash. Use exactly one of
 493                         C{repourl} and C{baseURL}.
 494
 495         @param defaultBranch: if branches are enabled, this is the branch
 496                               to use if the Build does not specify one
 497                               explicitly. It will simply be appended to
 498                               C{baseURL} and the result handed to the
 499                               'darcs pull' command.
 500         """
 501         self.repourl = repourl
 502         self.baseURL = baseURL
 503         self.branch = defaultBranch
 504         Source.__init__(self, **kwargs)
 505         assert kwargs['mode'] != "export", \
 506                "Darcs does not have an 'export' mode"
 507         if (not repourl and not baseURL) or (repourl and baseURL):
 508             raise ValueError("you must provide exactly one of repourl and"
 509                              " baseURL")
 510
 511     def startVC(self, branch, revision, patch):
 512         slavever = self.slaveVersion("darcs")
 513         if not slavever:
 514             m = "slave is too old, does not know about darcs"
 515             raise BuildSlaveTooOldError(m)
 516
 517         if self.slaveVersionIsOlderThan("darcs", "1.39"):
 518             if revision:
 519                 # TODO: revisit this once we implement computeSourceRevision
 520                 m = "0.6.6 slaves can't handle args['revision']"
 521                 raise BuildSlaveTooOldError(m)
 522
 523             # the slave doesn't know to avoid re-using the same sourcedir
 524             # when the branch changes. We have no way of knowing which branch
 525             # the last build used, so if we're using a non-default branch and
 526             # either 'update' or 'copy' modes, it is safer to refuse to
 527             # build, and tell the user they need to upgrade the buildslave.
 528             if (branch != self.branch
 529                 and self.args['mode'] in ("update", "copy")):
 530                 m = ("This buildslave (%s) does not know about multiple "
 531                      "branches, and using mode=%s would probably build the "
 532                      "wrong tree. "
 533                      "Refusing to build. Please upgrade the buildslave to "
 534                      "buildbot-0.7.0 or newer." % (self.build.slavename,
 535                                                    self.args['mode']))
 536                 raise BuildSlaveTooOldError(m)
 537
 538         if self.repourl:
 539             assert not branch # we need baseURL= to use branches
 540             self.args['repourl'] = self.repourl
 541         else:
 542             self.args['repourl'] = self.baseURL + branch
 543         self.args['revision'] = revision
 544         self.args['patch'] = patch
 545
 546         revstuff = []
 547         if branch is not None and branch != self.branch:
 548             revstuff.append("[branch]")
 549         self.description.extend(revstuff)
 550         self.descriptionDone.extend(revstuff)
 551
 552         cmd = LoggedRemoteCommand("darcs", self.args)
 553         self.startCommand(cmd)
 554
 555
 556 class Git(Source):
 557     """Check out a source tree from a git repository 'repourl'."""
 558
 559     name = "git"
 560
 561     def __init__(self, repourl, **kwargs):
 562         """
 563         @type  repourl: string
 564         @param repourl: the URL which points at the git repository
 565         """
 566         self.branch = None # TODO
 567         Source.__init__(self, **kwargs)
 568         self.args['repourl'] = repourl
 569
 570     def startVC(self, branch, revision, patch):
 571         self.args['branch'] = branch
 572         self.args['revision'] = revision
 573         self.args['patch'] = patch
 574         slavever = self.slaveVersion("git")
 575         if not slavever:
 576             raise BuildSlaveTooOldError("slave is too old, does not know "
 577                                         "about git")
 578         cmd = LoggedRemoteCommand("git", self.args)
 579         self.startCommand(cmd)
 580
 581
 582 class Arch(Source):
 583     """Check out a source tree from an Arch repository named 'archive'
 584     available at 'url'. 'version' specifies which version number (development
 585     line) will be used for the checkout: this is mostly equivalent to a
 586     branch name. This version uses the 'tla' tool to do the checkout, to use
 587     'baz' see L{Bazaar} instead.
 588     """
 589
 590     name = "arch"
 591     # TODO: slaves >0.6.6 will accept args['build-config'], so use it
 592
 593     def __init__(self, url, version, archive=None, **kwargs):
 594         """
 595         @type  url: string
 596         @param url: the Arch coordinates of the repository. This is
 597                     typically an http:// URL, but could also be the absolute
 598                     pathname of a local directory instead.
 599
 600         @type  version: string
 601         @param version: the category--branch--version to check out. This is
 602                         the default branch. If a build specifies a different
 603                         branch, it will be used instead of this.
 604
 605         @type  archive: string
 606         @param archive: The archive name. If provided, it must match the one
 607                         that comes from the repository. If not, the
 608                         repository's default will be used.
 609         """
 610         self.branch = version
 611         Source.__init__(self, **kwargs)
 612         self.args.update({'url': url,
 613                           'archive': archive,
 614                           })
 615
 616     def computeSourceRevision(self, changes):
 617         # in Arch, fully-qualified revision numbers look like:
 618         #  arch@buildbot.sourceforge.net--2004/buildbot--dev--0--patch-104
 619         # For any given builder, all of this is fixed except the patch-104.
 620         # The Change might have any part of the fully-qualified string, so we
 621         # just look for the last part. We return the "patch-NN" string.
 622         if not changes:
 623             return None
 624         lastChange = None
 625         for c in changes:
 626             if not c.revision:
 627                 continue
 628             if c.revision.endswith("--base-0"):
 629                 rev = 0
 630             else:
 631                 i = c.revision.rindex("patch")
 632                 rev = int(c.revision[i+len("patch-"):])
 633             lastChange = max(lastChange, rev)
 634         if lastChange is None:
 635             return None
 636         if lastChange == 0:
 637             return "base-0"
 638         return "patch-%d" % lastChange
 639
 640     def checkSlaveVersion(self, cmd, branch):
 641         warnings = []
 642         slavever = self.slaveVersion(cmd)
 643         if not slavever:
 644             m = "slave is too old, does not know about %s" % cmd
 645             raise BuildSlaveTooOldError(m)
 646
 647         # slave 1.28 and later understand 'revision'
 648         if self.slaveVersionIsOlderThan(cmd, "1.28"):
 649             if not self.alwaysUseLatest:
 650                 # we don't know whether our requested revision is the latest
 651                 # or not. If the tree does not change very quickly, this will
 652                 # probably build the right thing, so emit a warning rather
 653                 # than refuse to build at all
 654                 m = "WARNING, buildslave is too old to use a revision"
 655                 log.msg(m)
 656                 warnings.append(m + "\n")
 657
 658         if self.slaveVersionIsOlderThan(cmd, "1.39"):
 659             # the slave doesn't know to avoid re-using the same sourcedir
 660             # when the branch changes. We have no way of knowing which branch
 661             # the last build used, so if we're using a non-default branch and
 662             # either 'update' or 'copy' modes, it is safer to refuse to
 663             # build, and tell the user they need to upgrade the buildslave.
 664             if (branch != self.branch
 665                 and self.args['mode'] in ("update", "copy")):
 666                 m = ("This buildslave (%s) does not know about multiple "
 667                      "branches, and using mode=%s would probably build the "
 668                      "wrong tree. "
 669                      "Refusing to build. Please upgrade the buildslave to "
 670                      "buildbot-0.7.0 or newer." % (self.build.slavename,
 671                                                    self.args['mode']))
 672                 log.msg(m)
 673                 raise BuildSlaveTooOldError(m)
 674
 675         return warnings
 676
 677     def startVC(self, branch, revision, patch):
 678         self.args['version'] = branch
 679         self.args['revision'] = revision
 680         self.args['patch'] = patch
 681         warnings = self.checkSlaveVersion("arch", branch)
 682
 683         revstuff = []
 684         if branch is not None and branch != self.branch:
 685             revstuff.append("[branch]")
 686         if revision is not None:
 687             revstuff.append("patch%s" % revision)
 688         self.description.extend(revstuff)
 689         self.descriptionDone.extend(revstuff)
 690
 691         cmd = LoggedRemoteCommand("arch", self.args)
 692         self.startCommand(cmd, warnings)
 693
 694
 695 class Bazaar(Arch):
 696     """Bazaar is an alternative client for Arch repositories. baz is mostly
 697     compatible with tla, but archive registration is slightly different."""
 698
 699     # TODO: slaves >0.6.6 will accept args['build-config'], so use it
 700
 701     def __init__(self, url, version, archive, **kwargs):
 702         """
 703         @type  url: string
 704         @param url: the Arch coordinates of the repository. This is
 705                     typically an http:// URL, but could also be the absolute
 706                     pathname of a local directory instead.
 707
 708         @type  version: string
 709         @param version: the category--branch--version to check out
 710
 711         @type  archive: string
 712         @param archive: The archive name (required). This must always match
 713                         the one that comes from the repository, otherwise the
 714                         buildslave will attempt to get sources from the wrong
 715                         archive.
 716         """
 717         self.branch = version
 718         Source.__init__(self, **kwargs)
 719         self.args.update({'url': url,
 720                           'archive': archive,
 721                           })
 722
 723     def startVC(self, branch, revision, patch):
 724         self.args['version'] = branch
 725         self.args['revision'] = revision
 726         self.args['patch'] = patch
 727         warnings = self.checkSlaveVersion("bazaar", branch)
 728
 729         revstuff = []
 730         if branch is not None and branch != self.branch:
 731             revstuff.append("[branch]")
 732         if revision is not None:
 733             revstuff.append("patch%s" % revision)
 734         self.description.extend(revstuff)
 735         self.descriptionDone.extend(revstuff)
 736
 737         cmd = LoggedRemoteCommand("bazaar", self.args)
 738         self.startCommand(cmd, warnings)
 739
 740 class Bzr(Source):
 741     """Check out a source tree from a bzr (Bazaar) repository at 'repourl'.
 742
 743     """
 744
 745     name = "bzr"
 746
 747     def __init__(self, repourl=None, baseURL=None, defaultBranch=None,
 748                  **kwargs):
 749         """
 750         @type  repourl: string
 751         @param repourl: the URL which points at the bzr repository. This
 752                         is used as the default branch. Using C{repourl} does
 753                         not enable builds of alternate branches: use
 754                         C{baseURL} to enable this. Use either C{repourl} or
 755                         C{baseURL}, not both.
 756
 757         @param baseURL: if branches are enabled, this is the base URL to
 758                         which a branch name will be appended. It should
 759                         probably end in a slash. Use exactly one of
 760                         C{repourl} and C{baseURL}.
 761
 762         @param defaultBranch: if branches are enabled, this is the branch
 763                               to use if the Build does not specify one
 764                               explicitly. It will simply be appended to
 765                               C{baseURL} and the result handed to the
 766                               'bzr checkout pull' command.
 767         """
 768         self.repourl = repourl
 769         self.baseURL = baseURL
 770         self.branch = defaultBranch
 771         Source.__init__(self, **kwargs)
 772         if (not repourl and not baseURL) or (repourl and baseURL):
 773             raise ValueError("you must provide exactly one of repourl and"
 774                              " baseURL")
 775
 776     def computeSourceRevision(self, changes):
 777         if not changes:
 778             return None
 779         lastChange = max([int(c.revision) for c in changes])
 780         return lastChange
 781
 782     def startVC(self, branch, revision, patch):
 783         slavever = self.slaveVersion("bzr")
 784         if not slavever:
 785             m = "slave is too old, does not know about bzr"
 786             raise BuildSlaveTooOldError(m)
 787
 788         if self.repourl:
 789             assert not branch # we need baseURL= to use branches
 790             self.args['repourl'] = self.repourl
 791         else:
 792             self.args['repourl'] = self.baseURL + branch
 793         self.args['revision'] = revision
 794         self.args['patch'] = patch
 795
 796         revstuff = []
 797         if branch is not None and branch != self.branch:
 798             revstuff.append("[branch]")
 799         self.description.extend(revstuff)
 800         self.descriptionDone.extend(revstuff)
 801
 802         cmd = LoggedRemoteCommand("bzr", self.args)
 803         self.startCommand(cmd)
 804
 805
 806 class Mercurial(Source):
 807     """Check out a source tree from a mercurial repository 'repourl'."""
 808
 809     name = "hg"
 810
 811     def __init__(self, repourl=None, baseURL=None, defaultBranch=None,
 812                  **kwargs):
 813         """
 814         @type  repourl: string
 815         @param repourl: the URL which points at the Mercurial repository.
 816                         This is used as the default branch. Using C{repourl}
 817                         does not enable builds of alternate branches: use
 818                         C{baseURL} to enable this. Use either C{repourl} or
 819                         C{baseURL}, not both.
 820
 821         @param baseURL: if branches are enabled, this is the base URL to
 822                         which a branch name will be appended. It should
 823                         probably end in a slash. Use exactly one of
 824                         C{repourl} and C{baseURL}.
 825
 826         @param defaultBranch: if branches are enabled, this is the branch
 827                               to use if the Build does not specify one
 828                               explicitly. It will simply be appended to
 829                               C{baseURL} and the result handed to the
 830                               'hg clone' command.
 831         """
 832         self.repourl = repourl
 833         self.baseURL = baseURL
 834         self.branch = defaultBranch
 835         Source.__init__(self, **kwargs)
 836         if (not repourl and not baseURL) or (repourl and baseURL):
 837             raise ValueError("you must provide exactly one of repourl and"
 838                              " baseURL")
 839
 840     def startVC(self, branch, revision, patch):
 841         slavever = self.slaveVersion("hg")
 842         if not slavever:
 843             raise BuildSlaveTooOldError("slave is too old, does not know "
 844                                         "about hg")
 845
 846         if self.repourl:
 847             assert not branch # we need baseURL= to use branches
 848             self.args['repourl'] = self.repourl
 849         else:
 850             self.args['repourl'] = self.baseURL + branch
 851         self.args['revision'] = revision
 852         self.args['patch'] = patch
 853
 854         revstuff = []
 855         if branch is not None and branch != self.branch:
 856             revstuff.append("[branch]")
 857         self.description.extend(revstuff)
 858         self.descriptionDone.extend(revstuff)
 859
 860         cmd = LoggedRemoteCommand("hg", self.args)
 861         self.startCommand(cmd)
 862
 863
 864 class P4(Source):
 865     """ P4 is a class for accessing perforce revision control"""
 866     name = "p4"
 867
 868     def __init__(self, p4base, defaultBranch=None, p4port=None, p4user=None,
 869                  p4passwd=None, p4extra_views=[],
 870                  p4client='buildbot_%(slave)s_%(builder)s', **kwargs):
 871         """
 872         @type  p4base: string
 873         @param p4base: A view into a perforce depot, typically
 874                        "//depot/proj/"
 875
 876         @type  defaultBranch: string
 877         @param defaultBranch: Identify a branch to build by default. Perforce
 878                               is a view based branching system. So, the branch
 879                               is normally the name after the base. For example,
 880                               branch=1.0 is view=//depot/proj/1.0/...
 881                               branch=1.1 is view=//depot/proj/1.1/...
 882
 883         @type  p4port: string
 884         @param p4port: Specify the perforce server to connection in the format
 885                        <host>:<port>. Example "perforce.example.com:1666"
 886
 887         @type  p4user: string
 888         @param p4user: The perforce user to run the command as.
 889
 890         @type  p4passwd: string
 891         @param p4passwd: The password for the perforce user.
 892
 893         @type  p4extra_views: list of tuples
 894         @param p4extra_views: Extra views to be added to
 895                               the client that is being used.
 896
 897         @type  p4client: string
 898         @param p4client: The perforce client to use for this buildslave.
 899         """
 900
 901         self.branch = defaultBranch
 902         Source.__init__(self, **kwargs)
 903         self.args['p4port'] = p4port
 904         self.args['p4user'] = p4user
 905         self.args['p4passwd'] = p4passwd
 906         self.args['p4base'] = p4base
 907         self.args['p4extra_views'] = p4extra_views
 908         self.args['p4client'] = p4client % {
 909             'slave': self.build.slavename,
 910             'builder': self.build.builder.name,
 911         }
 912
 913     def computeSourceRevision(self, changes):
 914         if not changes:
 915             return None
 916         lastChange = max([int(c.revision) for c in changes])
 917         return lastChange
 918
 919     def startVC(self, branch, revision, patch):
 920         slavever = self.slaveVersion("p4")
 921         assert slavever, "slave is too old, does not know about p4"
 922         args = dict(self.args)
 923         args['branch'] = branch or self.branch
 924         args['revision'] = revision
 925         args['patch'] = patch
 926         cmd = LoggedRemoteCommand("p4", args)
 927         self.startCommand(cmd)
 928
 929 class P4Sync(Source):
 930     """This is a partial solution for using a P4 source repository. You are
 931     required to manually set up each build slave with a useful P4
 932     environment, which means setting various per-slave environment variables,
 933     and creating a P4 client specification which maps the right files into
 934     the slave's working directory. Once you have done that, this step merely
 935     performs a 'p4 sync' to update that workspace with the newest files.
 936
 937     Each slave needs the following environment:
 938
 939      - PATH: the 'p4' binary must be on the slave's PATH
 940      - P4USER: each slave needs a distinct user account
 941      - P4CLIENT: each slave needs a distinct client specification
 942
 943     You should use 'p4 client' (?) to set up a client view spec which maps
 944     the desired files into $SLAVEBASE/$BUILDERBASE/source .
 945     """
 946
 947     name = "p4sync"
 948
 949     def __init__(self, p4port, p4user, p4passwd, p4client, **kwargs):
 950         assert kwargs['mode'] == "copy", "P4Sync can only be used in mode=copy"
 951         self.branch = None
 952         Source.__init__(self, **kwargs)
 953         self.args['p4port'] = p4port
 954         self.args['p4user'] = p4user
 955         self.args['p4passwd'] = p4passwd
 956         self.args['p4client'] = p4client
 957
 958     def computeSourceRevision(self, changes):
 959         if not changes:
 960             return None
 961         lastChange = max([int(c.revision) for c in changes])
 962         return lastChange
 963
 964     def startVC(self, branch, revision, patch):
 965         slavever = self.slaveVersion("p4sync")
 966         assert slavever, "slave is too old, does not know about p4"
 967         cmd = LoggedRemoteCommand("p4sync", self.args)
 968         self.startCommand(cmd)
 969
 970 class Monotone(Source):
 971     """Check out a revision from a monotone server at 'server_addr',
 972     branch 'branch'.  'revision' specifies which revision id to check
 973     out.
 974
 975     This step will first create a local database, if necessary, and then pull
 976     the contents of the server into the database.  Then it will do the
 977     checkout/update from this database."""
 978
 979     name = "monotone"
 980
 981     def __init__(self, server_addr, branch, db_path="monotone.db",
 982                  monotone="monotone",
 983                  **kwargs):
 984         Source.__init__(self, **kwargs)
 985         self.args.update({"server_addr": server_addr,
 986                           "branch": branch,
 987                           "db_path": db_path,
 988                           "monotone": monotone})
 989
 990     def computeSourceRevision(self, changes):
 991         if not changes:
 992             return None
 993         return changes[-1].revision
 994
 995     def startVC(self):
 996         slavever = self.slaveVersion("monotone")
 997         assert slavever, "slave is too old, does not know about monotone"
 998         cmd = LoggedRemoteCommand("monotone", self.args)
 999         self.startCommand(cmd)
1000