buildbot/steps/source.py

   1 # -*- test-case-name: buildbot.test.test_vc -*-
   2
   3 import warnings
   4 from email.Utils import formatdate
   5 from twisted.python import log
   6 from buildbot.process.buildstep import LoggingBuildStep, LoggedRemoteCommand
   7 from buildbot.interfaces import BuildSlaveTooOldError
   8 from buildbot.status.builder import SKIPPED
   9
  10
  11 class Source(LoggingBuildStep):
  12     """This is a base class to generate a source tree in the buildslave.
  13     Each version control system has a specialized subclass, and is expected
  14     to override __init__ and implement computeSourceRevision() and
  15     startVC(). The class as a whole builds up the self.args dictionary, then
  16     starts a LoggedRemoteCommand with those arguments.
  17     """
  18
  19     # if the checkout fails, there's no point in doing anything else
  20     haltOnFailure = True
  21     notReally = False
  22
  23     branch = None # the default branch, should be set in __init__
  24
  25     def __init__(self, workdir, mode='update', alwaysUseLatest=False,
  26                  timeout=20*60, retry=None, **kwargs):
  27         """
  28         @type  workdir: string
  29         @param workdir: local directory (relative to the Builder's root)
  30                         where the tree should be placed
  31
  32         @type  mode: string
  33         @param mode: the kind of VC operation that is desired:
  34            - 'update': specifies that the checkout/update should be
  35              performed directly into the workdir. Each build is performed
  36              in the same directory, allowing for incremental builds. This
  37              minimizes disk space, bandwidth, and CPU time. However, it
  38              may encounter problems if the build process does not handle
  39              dependencies properly (if you must sometimes do a 'clean
  40              build' to make sure everything gets compiled), or if source
  41              files are deleted but generated files can influence test
  42              behavior (e.g. python's .pyc files), or when source
  43              directories are deleted but generated files prevent CVS from
  44              removing them.
  45
  46            - 'copy': specifies that the source-controlled workspace
  47              should be maintained in a separate directory (called the
  48              'copydir'), using checkout or update as necessary. For each
  49              build, a new workdir is created with a copy of the source
  50              tree (rm -rf workdir; cp -r copydir workdir). This doubles
  51              the disk space required, but keeps the bandwidth low
  52              (update instead of a full checkout). A full 'clean' build
  53              is performed each time.  This avoids any generated-file
  54              build problems, but is still occasionally vulnerable to
  55              problems such as a CVS repository being manually rearranged
  56              (causing CVS errors on update) which are not an issue with
  57              a full checkout.
  58
  59            - 'clobber': specifies that the working directory should be
  60              deleted each time, necessitating a full checkout for each
  61              build. This insures a clean build off a complete checkout,
  62              avoiding any of the problems described above, but is
  63              bandwidth intensive, as the whole source tree must be
  64              pulled down for each build.
  65
  66            - 'export': is like 'clobber', except that e.g. the 'cvs
  67              export' command is used to create the working directory.
  68              This command removes all VC metadata files (the
  69              CVS/.svn/{arch} directories) from the tree, which is
  70              sometimes useful for creating source tarballs (to avoid
  71              including the metadata in the tar file). Not all VC systems
  72              support export.
  73
  74         @type  alwaysUseLatest: boolean
  75         @param alwaysUseLatest: whether to always update to the most
  76         recent available sources for this build.
  77
  78         Normally the Source step asks its Build for a list of all
  79         Changes that are supposed to go into the build, then computes a
  80         'source stamp' (revision number or timestamp) that will cause
  81         exactly that set of changes to be present in the checked out
  82         tree. This is turned into, e.g., 'cvs update -D timestamp', or
  83         'svn update -r revnum'. If alwaysUseLatest=True, bypass this
  84         computation and always update to the latest available sources
  85         for each build.
  86
  87         The source stamp helps avoid a race condition in which someone
  88         commits a change after the master has decided to start a build
  89         but before the slave finishes checking out the sources. At best
  90         this results in a build which contains more changes than the
  91         buildmaster thinks it has (possibly resulting in the wrong
  92         person taking the blame for any problems that result), at worst
  93         is can result in an incoherent set of sources (splitting a
  94         non-atomic commit) which may not build at all.
  95
  96         @type  retry: tuple of ints (delay, repeats) (or None)
  97         @param retry: if provided, VC update failures are re-attempted up
  98                       to REPEATS times, with DELAY seconds between each
  99                       attempt. Some users have slaves with poor connectivity
 100                       to their VC repository, and they say that up to 80% of
 101                       their build failures are due to transient network
 102                       failures that could be handled by simply retrying a
 103                       couple times.
 104
 105         """
 106
 107         LoggingBuildStep.__init__(self, **kwargs)
 108
 109         assert mode in ("update", "copy", "clobber", "export")
 110         if retry:
 111             delay, repeats = retry
 112             assert isinstance(repeats, int)
 113             assert repeats > 0
 114         self.args = {'mode': mode,
 115                      'workdir': workdir,
 116                      'timeout': timeout,
 117                      'retry': retry,
 118                      'patch': None, # set during .start
 119                      }
 120         self.alwaysUseLatest = alwaysUseLatest
 121
 122         # Compute defaults for descriptions:
 123         description = ["updating"]
 124         descriptionDone = ["update"]
 125         if mode == "clobber":
 126             description = ["checkout"]
 127             # because checkingouting takes too much space
 128             descriptionDone = ["checkout"]
 129         elif mode == "export":
 130             description = ["exporting"]
 131             descriptionDone = ["export"]
 132         self.description = description
 133         self.descriptionDone = descriptionDone
 134
 135     def describe(self, done=False):
 136         if done:
 137             return self.descriptionDone
 138         return self.description
 139
 140     def computeSourceRevision(self, changes):
 141         """Each subclass must implement this method to do something more
 142         precise than -rHEAD every time. For version control systems that use
 143         repository-wide change numbers (SVN, P4), this can simply take the
 144         maximum such number from all the changes involved in this build. For
 145         systems that do not (CVS), it needs to create a timestamp based upon
 146         the latest Change, the Build's treeStableTimer, and an optional
 147         self.checkoutDelay value."""
 148         return None
 149
 150     def start(self):
 151         if self.notReally:
 152             log.msg("faking %s checkout/update" % self.name)
 153             self.step_status.setColor("green")
 154             self.step_status.setText(["fake", self.name, "successful"])
 155             self.addCompleteLog("log",
 156                                 "Faked %s checkout/update 'successful'\n" \
 157                                 % self.name)
 158             return SKIPPED
 159
 160         # what source stamp would this build like to use?
 161         s = self.build.getSourceStamp()
 162         # if branch is None, then use the Step's "default" branch
 163         branch = s.branch or self.branch
 164         # if revision is None, use the latest sources (-rHEAD)
 165         revision = s.revision
 166         if not revision and not self.alwaysUseLatest:
 167             revision = self.computeSourceRevision(s.changes)
 168         # if patch is None, then do not patch the tree after checkout
 169
 170         # 'patch' is None or a tuple of (patchlevel, diff)
 171         patch = s.patch
 172
 173         self.startVC(branch, revision, patch)
 174
 175     def commandComplete(self, cmd):
 176         got_revision = None
 177         if cmd.updates.has_key("got_revision"):
 178             got_revision = cmd.updates["got_revision"][-1]
 179         self.setProperty("got_revision", got_revision)
 180
 181
 182
 183 class CVS(Source):
 184     """I do CVS checkout/update operations.
 185
 186     Note: if you are doing anonymous/pserver CVS operations, you will need
 187     to manually do a 'cvs login' on each buildslave before the slave has any
 188     hope of success. XXX: fix then, take a cvs password as an argument and
 189     figure out how to do a 'cvs login' on each build
 190     """
 191
 192     name = "cvs"
 193
 194     #progressMetrics = ('output',)
 195     #
 196     # additional things to track: update gives one stderr line per directory
 197     # (starting with 'cvs server: Updating ') (and is fairly stable if files
 198     # is empty), export gives one line per directory (starting with 'cvs
 199     # export: Updating ') and another line per file (starting with U). Would
 200     # be nice to track these, requires grepping LogFile data for lines,
 201     # parsing each line. Might be handy to have a hook in LogFile that gets
 202     # called with each complete line.
 203
 204     def __init__(self, cvsroot, cvsmodule,
 205                  global_options=[], branch=None, checkoutDelay=None,
 206                  login=None,
 207                  clobber=0, export=0, copydir=None,
 208                  **kwargs):
 209
 210         """
 211         @type  cvsroot: string
 212         @param cvsroot: CVS Repository from which the source tree should
 213                         be obtained. '/home/warner/Repository' for local
 214                         or NFS-reachable repositories,
 215                         ':pserver:anon@foo.com:/cvs' for anonymous CVS,
 216                         'user@host.com:/cvs' for non-anonymous CVS or
 217                         CVS over ssh. Lots of possibilities, check the
 218                         CVS documentation for more.
 219
 220         @type  cvsmodule: string
 221         @param cvsmodule: subdirectory of CVS repository that should be
 222                           retrieved
 223
 224         @type  login: string or None
 225         @param login: if not None, a string which will be provided as a
 226                       password to the 'cvs login' command, used when a
 227                       :pserver: method is used to access the repository.
 228                       This login is only needed once, but must be run
 229                       each time (just before the CVS operation) because
 230                       there is no way for the buildslave to tell whether
 231                       it was previously performed or not.
 232
 233         @type  branch: string
 234         @param branch: the default branch name, will be used in a '-r'
 235                        argument to specify which branch of the source tree
 236                        should be used for this checkout. Defaults to None,
 237                        which means to use 'HEAD'.
 238
 239         @type  checkoutDelay: int or None
 240         @param checkoutDelay: if not None, the number of seconds to put
 241                               between the last known Change and the
 242                               timestamp given to the -D argument. This
 243                               defaults to exactly half of the parent
 244                               Build's .treeStableTimer, but it could be
 245                               set to something else if your CVS change
 246                               notification has particularly weird
 247                               latency characteristics.
 248
 249         @type  global_options: list of strings
 250         @param global_options: these arguments are inserted in the cvs
 251                                command line, before the
 252                                'checkout'/'update' command word. See
 253                                'cvs --help-options' for a list of what
 254                                may be accepted here.  ['-r'] will make
 255                                the checked out files read only. ['-r',
 256                                '-R'] will also assume the repository is
 257                                read-only (I assume this means it won't
 258                                use locks to insure atomic access to the
 259                                ,v files)."""
 260
 261         self.checkoutDelay = checkoutDelay
 262         self.branch = branch
 263
 264         if not kwargs.has_key('mode') and (clobber or export or copydir):
 265             # deal with old configs
 266             warnings.warn("Please use mode=, not clobber/export/copydir",
 267                           DeprecationWarning)
 268             if export:
 269                 kwargs['mode'] = "export"
 270             elif clobber:
 271                 kwargs['mode'] = "clobber"
 272             elif copydir:
 273                 kwargs['mode'] = "copy"
 274             else:
 275                 kwargs['mode'] = "update"
 276
 277         Source.__init__(self, **kwargs)
 278
 279         self.args.update({'cvsroot': cvsroot,
 280                           'cvsmodule': cvsmodule,
 281                           'global_options': global_options,
 282                           'login': login,
 283                           })
 284
 285     def computeSourceRevision(self, changes):
 286         if not changes:
 287             return None
 288         lastChange = max([c.when for c in changes])
 289         if self.checkoutDelay is not None:
 290             when = lastChange + self.checkoutDelay
 291         else:
 292             lastSubmit = max([r.submittedAt for r in self.build.requests])
 293             when = (lastChange + lastSubmit) / 2
 294         return formatdate(when)
 295
 296     def startVC(self, branch, revision, patch):
 297         if self.slaveVersionIsOlderThan("cvs", "1.39"):
 298             # the slave doesn't know to avoid re-using the same sourcedir
 299             # when the branch changes. We have no way of knowing which branch
 300             # the last build used, so if we're using a non-default branch and
 301             # either 'update' or 'copy' modes, it is safer to refuse to
 302             # build, and tell the user they need to upgrade the buildslave.
 303             if (branch != self.branch
 304                 and self.args['mode'] in ("update", "copy")):
 305                 m = ("This buildslave (%s) does not know about multiple "
 306                      "branches, and using mode=%s would probably build the "
 307                      "wrong tree. "
 308                      "Refusing to build. Please upgrade the buildslave to "
 309                      "buildbot-0.7.0 or newer." % (self.build.slavename,
 310                                                    self.args['mode']))
 311                 log.msg(m)
 312                 raise BuildSlaveTooOldError(m)
 313
 314         if branch is None:
 315             branch = "HEAD"
 316         self.args['branch'] = branch
 317         self.args['revision'] = revision
 318         self.args['patch'] = patch
 319
 320         if self.args['branch'] == "HEAD" and self.args['revision']:
 321             # special case. 'cvs update -r HEAD -D today' gives no files
 322             # TODO: figure out why, see if it applies to -r BRANCH
 323             self.args['branch'] = None
 324
 325         # deal with old slaves
 326         warnings = []
 327         slavever = self.slaveVersion("cvs", "old")
 328
 329         if slavever == "old":
 330             # 0.5.0
 331             if self.args['mode'] == "export":
 332                 self.args['export'] = 1
 333             elif self.args['mode'] == "clobber":
 334                 self.args['clobber'] = 1
 335             elif self.args['mode'] == "copy":
 336                 self.args['copydir'] = "source"
 337             self.args['tag'] = self.args['branch']
 338             assert not self.args['patch'] # 0.5.0 slave can't do patch
 339
 340         cmd = LoggedRemoteCommand("cvs", self.args)
 341         self.startCommand(cmd, warnings)
 342
 343
 344 class SVN(Source):
 345     """I perform Subversion checkout/update operations."""
 346
 347     name = 'svn'
 348
 349     def __init__(self, svnurl=None, baseURL=None, defaultBranch=None,
 350                  directory=None, **kwargs):
 351         """
 352         @type  svnurl: string
 353         @param svnurl: the URL which points to the Subversion server,
 354                        combining the access method (HTTP, ssh, local file),
 355                        the repository host/port, the repository path, the
 356                        sub-tree within the repository, and the branch to
 357                        check out. Using C{svnurl} does not enable builds of
 358                        alternate branches: use C{baseURL} to enable this.
 359                        Use exactly one of C{svnurl} and C{baseURL}.
 360
 361         @param baseURL: if branches are enabled, this is the base URL to
 362                         which a branch name will be appended. It should
 363                         probably end in a slash. Use exactly one of
 364                         C{svnurl} and C{baseURL}.
 365
 366         @param defaultBranch: if branches are enabled, this is the branch
 367                               to use if the Build does not specify one
 368                               explicitly. It will simply be appended
 369                               to C{baseURL} and the result handed to
 370                               the SVN command.
 371         """
 372
 373         if not kwargs.has_key('workdir') and directory is not None:
 374             # deal with old configs
 375             warnings.warn("Please use workdir=, not directory=",
 376                           DeprecationWarning)
 377             kwargs['workdir'] = directory
 378
 379         self.svnurl = svnurl
 380         self.baseURL = baseURL
 381         self.branch = defaultBranch
 382
 383         Source.__init__(self, **kwargs)
 384
 385         if not svnurl and not baseURL:
 386             raise ValueError("you must use exactly one of svnurl and baseURL")
 387
 388
 389     def computeSourceRevision(self, changes):
 390         if not changes:
 391             return None
 392         lastChange = max([int(c.revision) for c in changes])
 393         return lastChange
 394
 395     def startVC(self, branch, revision, patch):
 396
 397         # handle old slaves
 398         warnings = []
 399         slavever = self.slaveVersion("svn", "old")
 400         if not slavever:
 401             m = "slave does not have the 'svn' command"
 402             raise BuildSlaveTooOldError(m)
 403
 404         if self.slaveVersionIsOlderThan("svn", "1.39"):
 405             # the slave doesn't know to avoid re-using the same sourcedir
 406             # when the branch changes. We have no way of knowing which branch
 407             # the last build used, so if we're using a non-default branch and
 408             # either 'update' or 'copy' modes, it is safer to refuse to
 409             # build, and tell the user they need to upgrade the buildslave.
 410             if (branch != self.branch
 411                 and self.args['mode'] in ("update", "copy")):
 412                 m = ("This buildslave (%s) does not know about multiple "
 413                      "branches, and using mode=%s would probably build the "
 414                      "wrong tree. "
 415                      "Refusing to build. Please upgrade the buildslave to "
 416                      "buildbot-0.7.0 or newer." % (self.build.slavename,
 417                                                    self.args['mode']))
 418                 raise BuildSlaveTooOldError(m)
 419
 420         if slavever == "old":
 421             # 0.5.0 compatibility
 422             if self.args['mode'] in ("clobber", "copy"):
 423                 # TODO: use some shell commands to make up for the
 424                 # deficiency, by blowing away the old directory first (thus
 425                 # forcing a full checkout)
 426                 warnings.append("WARNING: this slave can only do SVN updates"
 427                                 ", not mode=%s\n" % self.args['mode'])
 428                 log.msg("WARNING: this slave only does mode=update")
 429             if self.args['mode'] == "export":
 430                 raise BuildSlaveTooOldError("old slave does not have "
 431                                             "mode=export")
 432             self.args['directory'] = self.args['workdir']
 433             if revision is not None:
 434                 # 0.5.0 can only do HEAD. We have no way of knowing whether
 435                 # the requested revision is HEAD or not, and for
 436                 # slowly-changing trees this will probably do the right
 437                 # thing, so let it pass with a warning
 438                 m = ("WARNING: old slave can only update to HEAD, not "
 439                      "revision=%s" % revision)
 440                 log.msg(m)
 441                 warnings.append(m + "\n")
 442             revision = "HEAD" # interprets this key differently
 443             if patch:
 444                 raise BuildSlaveTooOldError("old slave can't do patch")
 445
 446         if self.svnurl:
 447             assert not branch # we need baseURL= to use branches
 448             self.args['svnurl'] = self.svnurl
 449         else:
 450             self.args['svnurl'] = self.baseURL + branch
 451         self.args['revision'] = revision
 452         self.args['patch'] = patch
 453
 454         revstuff = []
 455         if branch is not None and branch != self.branch:
 456             revstuff.append("[branch]")
 457         if revision is not None:
 458             revstuff.append("r%s" % revision)
 459         self.description.extend(revstuff)
 460         self.descriptionDone.extend(revstuff)
 461
 462         cmd = LoggedRemoteCommand("svn", self.args)
 463         self.startCommand(cmd, warnings)
 464
 465
 466 class Darcs(Source):
 467     """Check out a source tree from a Darcs repository at 'repourl'.
 468
 469     To the best of my knowledge, Darcs has no concept of file modes. This
 470     means the eXecute-bit will be cleared on all source files. As a result,
 471     you may need to invoke configuration scripts with something like:
 472
 473     C{s(step.Configure, command=['/bin/sh', './configure'])}
 474     """
 475
 476     name = "darcs"
 477
 478     def __init__(self, repourl=None, baseURL=None, defaultBranch=None,
 479                  **kwargs):
 480         """
 481         @type  repourl: string
 482         @param repourl: the URL which points at the Darcs repository. This
 483                         is used as the default branch. Using C{repourl} does
 484                         not enable builds of alternate branches: use
 485                         C{baseURL} to enable this. Use either C{repourl} or
 486                         C{baseURL}, not both.
 487
 488         @param baseURL: if branches are enabled, this is the base URL to
 489                         which a branch name will be appended. It should
 490                         probably end in a slash. Use exactly one of
 491                         C{repourl} and C{baseURL}.
 492
 493         @param defaultBranch: if branches are enabled, this is the branch
 494                               to use if the Build does not specify one
 495                               explicitly. It will simply be appended to
 496                               C{baseURL} and the result handed to the
 497                               'darcs pull' command.
 498         """
 499         self.repourl = repourl
 500         self.baseURL = baseURL
 501         self.branch = defaultBranch
 502         Source.__init__(self, **kwargs)
 503         assert kwargs['mode'] != "export", \
 504                "Darcs does not have an 'export' mode"
 505         if (not repourl and not baseURL) or (repourl and baseURL):
 506             raise ValueError("you must provide exactly one of repourl and"
 507                              " baseURL")
 508
 509     def startVC(self, branch, revision, patch):
 510         slavever = self.slaveVersion("darcs")
 511         if not slavever:
 512             m = "slave is too old, does not know about darcs"
 513             raise BuildSlaveTooOldError(m)
 514
 515         if self.slaveVersionIsOlderThan("darcs", "1.39"):
 516             if revision:
 517                 # TODO: revisit this once we implement computeSourceRevision
 518                 m = "0.6.6 slaves can't handle args['revision']"
 519                 raise BuildSlaveTooOldError(m)
 520
 521             # the slave doesn't know to avoid re-using the same sourcedir
 522             # when the branch changes. We have no way of knowing which branch
 523             # the last build used, so if we're using a non-default branch and
 524             # either 'update' or 'copy' modes, it is safer to refuse to
 525             # build, and tell the user they need to upgrade the buildslave.
 526             if (branch != self.branch
 527                 and self.args['mode'] in ("update", "copy")):
 528                 m = ("This buildslave (%s) does not know about multiple "
 529                      "branches, and using mode=%s would probably build the "
 530                      "wrong tree. "
 531                      "Refusing to build. Please upgrade the buildslave to "
 532                      "buildbot-0.7.0 or newer." % (self.build.slavename,
 533                                                    self.args['mode']))
 534                 raise BuildSlaveTooOldError(m)
 535
 536         if self.repourl:
 537             assert not branch # we need baseURL= to use branches
 538             self.args['repourl'] = self.repourl
 539         else:
 540             self.args['repourl'] = self.baseURL + branch
 541         self.args['revision'] = revision
 542         self.args['patch'] = patch
 543
 544         revstuff = []
 545         if branch is not None and branch != self.branch:
 546             revstuff.append("[branch]")
 547         self.description.extend(revstuff)
 548         self.descriptionDone.extend(revstuff)
 549
 550         cmd = LoggedRemoteCommand("darcs", self.args)
 551         self.startCommand(cmd)
 552
 553
 554 class Git(Source):
 555     """Check out a source tree from a git repository 'repourl'."""
 556
 557     name = "git"
 558
 559     def __init__(self, repourl, **kwargs):
 560         """
 561         @type  repourl: string
 562         @param repourl: the URL which points at the git repository
 563         """
 564         self.branch = None # TODO
 565         Source.__init__(self, **kwargs)
 566         self.args['repourl'] = repourl
 567
 568     def startVC(self, branch, revision, patch):
 569         self.args['branch'] = branch
 570         self.args['revision'] = revision
 571         self.args['patch'] = patch
 572         slavever = self.slaveVersion("git")
 573         if not slavever:
 574             raise BuildSlaveTooOldError("slave is too old, does not know "
 575                                         "about git")
 576         cmd = LoggedRemoteCommand("git", self.args)
 577         self.startCommand(cmd)
 578
 579
 580 class Arch(Source):
 581     """Check out a source tree from an Arch repository named 'archive'
 582     available at 'url'. 'version' specifies which version number (development
 583     line) will be used for the checkout: this is mostly equivalent to a
 584     branch name. This version uses the 'tla' tool to do the checkout, to use
 585     'baz' see L{Bazaar} instead.
 586     """
 587
 588     name = "arch"
 589     # TODO: slaves >0.6.6 will accept args['build-config'], so use it
 590
 591     def __init__(self, url, version, archive=None, **kwargs):
 592         """
 593         @type  url: string
 594         @param url: the Arch coordinates of the repository. This is
 595                     typically an http:// URL, but could also be the absolute
 596                     pathname of a local directory instead.
 597
 598         @type  version: string
 599         @param version: the category--branch--version to check out. This is
 600                         the default branch. If a build specifies a different
 601                         branch, it will be used instead of this.
 602
 603         @type  archive: string
 604         @param archive: The archive name. If provided, it must match the one
 605                         that comes from the repository. If not, the
 606                         repository's default will be used.
 607         """
 608         self.branch = version
 609         Source.__init__(self, **kwargs)
 610         self.args.update({'url': url,
 611                           'archive': archive,
 612                           })
 613
 614     def computeSourceRevision(self, changes):
 615         # in Arch, fully-qualified revision numbers look like:
 616         #  arch@buildbot.sourceforge.net--2004/buildbot--dev--0--patch-104
 617         # For any given builder, all of this is fixed except the patch-104.
 618         # The Change might have any part of the fully-qualified string, so we
 619         # just look for the last part. We return the "patch-NN" string.
 620         if not changes:
 621             return None
 622         lastChange = None
 623         for c in changes:
 624             if not c.revision:
 625                 continue
 626             if c.revision.endswith("--base-0"):
 627                 rev = 0
 628             else:
 629                 i = c.revision.rindex("patch")
 630                 rev = int(c.revision[i+len("patch-"):])
 631             lastChange = max(lastChange, rev)
 632         if lastChange is None:
 633             return None
 634         if lastChange == 0:
 635             return "base-0"
 636         return "patch-%d" % lastChange
 637
 638     def checkSlaveVersion(self, cmd, branch):
 639         warnings = []
 640         slavever = self.slaveVersion(cmd)
 641         if not slavever:
 642             m = "slave is too old, does not know about %s" % cmd
 643             raise BuildSlaveTooOldError(m)
 644
 645         # slave 1.28 and later understand 'revision'
 646         if self.slaveVersionIsOlderThan(cmd, "1.28"):
 647             if not self.alwaysUseLatest:
 648                 # we don't know whether our requested revision is the latest
 649                 # or not. If the tree does not change very quickly, this will
 650                 # probably build the right thing, so emit a warning rather
 651                 # than refuse to build at all
 652                 m = "WARNING, buildslave is too old to use a revision"
 653                 log.msg(m)
 654                 warnings.append(m + "\n")
 655
 656         if self.slaveVersionIsOlderThan(cmd, "1.39"):
 657             # the slave doesn't know to avoid re-using the same sourcedir
 658             # when the branch changes. We have no way of knowing which branch
 659             # the last build used, so if we're using a non-default branch and
 660             # either 'update' or 'copy' modes, it is safer to refuse to
 661             # build, and tell the user they need to upgrade the buildslave.
 662             if (branch != self.branch
 663                 and self.args['mode'] in ("update", "copy")):
 664                 m = ("This buildslave (%s) does not know about multiple "
 665                      "branches, and using mode=%s would probably build the "
 666                      "wrong tree. "
 667                      "Refusing to build. Please upgrade the buildslave to "
 668                      "buildbot-0.7.0 or newer." % (self.build.slavename,
 669                                                    self.args['mode']))
 670                 log.msg(m)
 671                 raise BuildSlaveTooOldError(m)
 672
 673         return warnings
 674
 675     def startVC(self, branch, revision, patch):
 676         self.args['version'] = branch
 677         self.args['revision'] = revision
 678         self.args['patch'] = patch
 679         warnings = self.checkSlaveVersion("arch", branch)
 680
 681         revstuff = []
 682         if branch is not None and branch != self.branch:
 683             revstuff.append("[branch]")
 684         if revision is not None:
 685             revstuff.append("patch%s" % revision)
 686         self.description.extend(revstuff)
 687         self.descriptionDone.extend(revstuff)
 688
 689         cmd = LoggedRemoteCommand("arch", self.args)
 690         self.startCommand(cmd, warnings)
 691
 692
 693 class Bazaar(Arch):
 694     """Bazaar is an alternative client for Arch repositories. baz is mostly
 695     compatible with tla, but archive registration is slightly different."""
 696
 697     # TODO: slaves >0.6.6 will accept args['build-config'], so use it
 698
 699     def __init__(self, url, version, archive, **kwargs):
 700         """
 701         @type  url: string
 702         @param url: the Arch coordinates of the repository. This is
 703                     typically an http:// URL, but could also be the absolute
 704                     pathname of a local directory instead.
 705
 706         @type  version: string
 707         @param version: the category--branch--version to check out
 708
 709         @type  archive: string
 710         @param archive: The archive name (required). This must always match
 711                         the one that comes from the repository, otherwise the
 712                         buildslave will attempt to get sources from the wrong
 713                         archive.
 714         """
 715         self.branch = version
 716         Source.__init__(self, **kwargs)
 717         self.args.update({'url': url,
 718                           'archive': archive,
 719                           })
 720
 721     def startVC(self, branch, revision, patch):
 722         self.args['version'] = branch
 723         self.args['revision'] = revision
 724         self.args['patch'] = patch
 725         warnings = self.checkSlaveVersion("bazaar", branch)
 726
 727         revstuff = []
 728         if branch is not None and branch != self.branch:
 729             revstuff.append("[branch]")
 730         if revision is not None:
 731             revstuff.append("patch%s" % revision)
 732         self.description.extend(revstuff)
 733         self.descriptionDone.extend(revstuff)
 734
 735         cmd = LoggedRemoteCommand("bazaar", self.args)
 736         self.startCommand(cmd, warnings)
 737
 738 class Mercurial(Source):
 739     """Check out a source tree from a mercurial repository 'repourl'."""
 740
 741     name = "hg"
 742
 743     def __init__(self, repourl=None, baseURL=None, defaultBranch=None,
 744                  **kwargs):
 745         """
 746         @type  repourl: string
 747         @param repourl: the URL which points at the Mercurial repository.
 748                         This is used as the default branch. Using C{repourl}
 749                         does not enable builds of alternate branches: use
 750                         C{baseURL} to enable this. Use either C{repourl} or
 751                         C{baseURL}, not both.
 752
 753         @param baseURL: if branches are enabled, this is the base URL to
 754                         which a branch name will be appended. It should
 755                         probably end in a slash. Use exactly one of
 756                         C{repourl} and C{baseURL}.
 757
 758         @param defaultBranch: if branches are enabled, this is the branch
 759                               to use if the Build does not specify one
 760                               explicitly. It will simply be appended to
 761                               C{baseURL} and the result handed to the
 762                               'hg clone' command.
 763         """
 764         self.repourl = repourl
 765         self.baseURL = baseURL
 766         self.branch = defaultBranch
 767         Source.__init__(self, **kwargs)
 768         if (not repourl and not baseURL) or (repourl and baseURL):
 769             raise ValueError("you must provide exactly one of repourl and"
 770                              " baseURL")
 771
 772     def startVC(self, branch, revision, patch):
 773         slavever = self.slaveVersion("hg")
 774         if not slavever:
 775             raise BuildSlaveTooOldError("slave is too old, does not know "
 776                                         "about hg")
 777
 778         if self.repourl:
 779             assert not branch # we need baseURL= to use branches
 780             self.args['repourl'] = self.repourl
 781         else:
 782             self.args['repourl'] = self.baseURL + branch
 783         self.args['revision'] = revision
 784         self.args['patch'] = patch
 785
 786         revstuff = []
 787         if branch is not None and branch != self.branch:
 788             revstuff.append("[branch]")
 789         self.description.extend(revstuff)
 790         self.descriptionDone.extend(revstuff)
 791
 792         cmd = LoggedRemoteCommand("hg", self.args)
 793         self.startCommand(cmd)
 794
 795
 796 class P4(Source):
 797     """ P4 is a class for accessing perforce revision control"""
 798     name = "p4"
 799
 800     def __init__(self, p4base, defaultBranch=None, p4port=None, p4user=None,
 801                  p4passwd=None, p4extra_views=[],
 802                  p4client='buildbot_%(slave)s_%(builder)s', **kwargs):
 803         """
 804         @type  p4base: string
 805         @param p4base: A view into a perforce depot, typically
 806                        "//depot/proj/"
 807
 808         @type  defaultBranch: string
 809         @param defaultBranch: Identify a branch to build by default. Perforce
 810                               is a view based branching system. So, the branch
 811                               is normally the name after the base. For example,
 812                               branch=1.0 is view=//depot/proj/1.0/...
 813                               branch=1.1 is view=//depot/proj/1.1/...
 814
 815         @type  p4port: string
 816         @param p4port: Specify the perforce server to connection in the format
 817                        <host>:<port>. Example "perforce.example.com:1666"
 818
 819         @type  p4user: string
 820         @param p4user: The perforce user to run the command as.
 821
 822         @type  p4passwd: string
 823         @param p4passwd: The password for the perforce user.
 824
 825         @type  p4extra_views: list of tuples
 826         @param p4extra_views: Extra views to be added to
 827                               the client that is being used.
 828
 829         @type  p4client: string
 830         @param p4client: The perforce client to use for this buildslave.
 831         """
 832
 833         self.branch = defaultBranch
 834         Source.__init__(self, **kwargs)
 835         self.args['p4port'] = p4port
 836         self.args['p4user'] = p4user
 837         self.args['p4passwd'] = p4passwd
 838         self.args['p4base'] = p4base
 839         self.args['p4extra_views'] = p4extra_views
 840         self.args['p4client'] = p4client % {
 841             'slave': self.build.slavename,
 842             'builder': self.build.builder.name,
 843         }
 844
 845     def computeSourceRevision(self, changes):
 846         if not changes:
 847             return None
 848         lastChange = max([int(c.revision) for c in changes])
 849         return lastChange
 850
 851     def startVC(self, branch, revision, patch):
 852         slavever = self.slaveVersion("p4")
 853         assert slavever, "slave is too old, does not know about p4"
 854         args = dict(self.args)
 855         args['branch'] = branch or self.branch
 856         args['revision'] = revision
 857         args['patch'] = patch
 858         cmd = LoggedRemoteCommand("p4", args)
 859         self.startCommand(cmd)
 860
 861 class P4Sync(Source):
 862     """This is a partial solution for using a P4 source repository. You are
 863     required to manually set up each build slave with a useful P4
 864     environment, which means setting various per-slave environment variables,
 865     and creating a P4 client specification which maps the right files into
 866     the slave's working directory. Once you have done that, this step merely
 867     performs a 'p4 sync' to update that workspace with the newest files.
 868
 869     Each slave needs the following environment:
 870
 871      - PATH: the 'p4' binary must be on the slave's PATH
 872      - P4USER: each slave needs a distinct user account
 873      - P4CLIENT: each slave needs a distinct client specification
 874
 875     You should use 'p4 client' (?) to set up a client view spec which maps
 876     the desired files into $SLAVEBASE/$BUILDERBASE/source .
 877     """
 878
 879     name = "p4sync"
 880
 881     def __init__(self, p4port, p4user, p4passwd, p4client, **kwargs):
 882         assert kwargs['mode'] == "copy", "P4Sync can only be used in mode=copy"
 883         self.branch = None
 884         Source.__init__(self, **kwargs)
 885         self.args['p4port'] = p4port
 886         self.args['p4user'] = p4user
 887         self.args['p4passwd'] = p4passwd
 888         self.args['p4client'] = p4client
 889
 890     def computeSourceRevision(self, changes):
 891         if not changes:
 892             return None
 893         lastChange = max([int(c.revision) for c in changes])
 894         return lastChange
 895
 896     def startVC(self, branch, revision, patch):
 897         slavever = self.slaveVersion("p4sync")
 898         assert slavever, "slave is too old, does not know about p4"
 899         cmd = LoggedRemoteCommand("p4sync", self.args)
 900         self.startCommand(cmd)
 901
 902 class Monotone(Source):
 903     """Check out a revision from a monotone server at 'server_addr',
 904     branch 'branch'.  'revision' specifies which revision id to check
 905     out.
 906
 907     This step will first create a local database, if necessary, and then pull
 908     the contents of the server into the database.  Then it will do the
 909     checkout/update from this database."""
 910
 911     name = "monotone"
 912
 913     def __init__(self, server_addr, branch, db_path="monotone.db",
 914                  monotone="monotone",
 915                  **kwargs):
 916         Source.__init__(self, **kwargs)
 917         self.args.update({"server_addr": server_addr,
 918                           "branch": branch,
 919                           "db_path": db_path,
 920                           "monotone": monotone})
 921
 922     def computeSourceRevision(self, changes):
 923         if not changes:
 924             return None
 925         return changes[-1].revision
 926
 927     def startVC(self):
 928         slavever = self.slaveVersion("monotone")
 929         assert slavever, "slave is too old, does not know about monotone"
 930         cmd = LoggedRemoteCommand("monotone", self.args)
 931         self.startCommand(cmd)
 932