buildbot/steps/source.py

   1 # -*- test-case-name: buildbot.test.test_vc -*-
   2
   3 from warnings import warn
   4 from email.Utils import formatdate
   5 from twisted.python import log
   6 from buildbot.process.buildstep import LoggingBuildStep, LoggedRemoteCommand
   7 from buildbot.interfaces import BuildSlaveTooOldError
   8 from buildbot.status.builder import SKIPPED
   9
  10
  11 class Source(LoggingBuildStep):
  12     """This is a base class to generate a source tree in the buildslave.
  13     Each version control system has a specialized subclass, and is expected
  14     to override __init__ and implement computeSourceRevision() and
  15     startVC(). The class as a whole builds up the self.args dictionary, then
  16     starts a LoggedRemoteCommand with those arguments.
  17     """
  18
  19     # if the checkout fails, there's no point in doing anything else
  20     haltOnFailure = True
  21     flunkOnFailure = True
  22     notReally = False
  23
  24     branch = None # the default branch, should be set in __init__
  25
  26     def __init__(self, workdir=None, mode='update', alwaysUseLatest=False,
  27                  timeout=20*60, retry=None, **kwargs):
  28         """
  29         @type  workdir: string
  30         @param workdir: local directory (relative to the Builder's root)
  31                         where the tree should be placed
  32
  33         @type  mode: string
  34         @param mode: the kind of VC operation that is desired:
  35            - 'update': specifies that the checkout/update should be
  36              performed directly into the workdir. Each build is performed
  37              in the same directory, allowing for incremental builds. This
  38              minimizes disk space, bandwidth, and CPU time. However, it
  39              may encounter problems if the build process does not handle
  40              dependencies properly (if you must sometimes do a 'clean
  41              build' to make sure everything gets compiled), or if source
  42              files are deleted but generated files can influence test
  43              behavior (e.g. python's .pyc files), or when source
  44              directories are deleted but generated files prevent CVS from
  45              removing them.
  46
  47            - 'copy': specifies that the source-controlled workspace
  48              should be maintained in a separate directory (called the
  49              'copydir'), using checkout or update as necessary. For each
  50              build, a new workdir is created with a copy of the source
  51              tree (rm -rf workdir; cp -R -P -p copydir workdir). This
  52              doubles the disk space required, but keeps the bandwidth low
  53              (update instead of a full checkout). A full 'clean' build
  54              is performed each time.  This avoids any generated-file
  55              build problems, but is still occasionally vulnerable to
  56              problems such as a CVS repository being manually rearranged
  57              (causing CVS errors on update) which are not an issue with
  58              a full checkout.
  59
  60            - 'clobber': specifies that the working directory should be
  61              deleted each time, necessitating a full checkout for each
  62              build. This insures a clean build off a complete checkout,
  63              avoiding any of the problems described above, but is
  64              bandwidth intensive, as the whole source tree must be
  65              pulled down for each build.
  66
  67            - 'export': is like 'clobber', except that e.g. the 'cvs
  68              export' command is used to create the working directory.
  69              This command removes all VC metadata files (the
  70              CVS/.svn/{arch} directories) from the tree, which is
  71              sometimes useful for creating source tarballs (to avoid
  72              including the metadata in the tar file). Not all VC systems
  73              support export.
  74
  75         @type  alwaysUseLatest: boolean
  76         @param alwaysUseLatest: whether to always update to the most
  77         recent available sources for this build.
  78
  79         Normally the Source step asks its Build for a list of all
  80         Changes that are supposed to go into the build, then computes a
  81         'source stamp' (revision number or timestamp) that will cause
  82         exactly that set of changes to be present in the checked out
  83         tree. This is turned into, e.g., 'cvs update -D timestamp', or
  84         'svn update -r revnum'. If alwaysUseLatest=True, bypass this
  85         computation and always update to the latest available sources
  86         for each build.
  87
  88         The source stamp helps avoid a race condition in which someone
  89         commits a change after the master has decided to start a build
  90         but before the slave finishes checking out the sources. At best
  91         this results in a build which contains more changes than the
  92         buildmaster thinks it has (possibly resulting in the wrong
  93         person taking the blame for any problems that result), at worst
  94         is can result in an incoherent set of sources (splitting a
  95         non-atomic commit) which may not build at all.
  96
  97         @type  retry: tuple of ints (delay, repeats) (or None)
  98         @param retry: if provided, VC update failures are re-attempted up
  99                       to REPEATS times, with DELAY seconds between each
 100                       attempt. Some users have slaves with poor connectivity
 101                       to their VC repository, and they say that up to 80% of
 102                       their build failures are due to transient network
 103                       failures that could be handled by simply retrying a
 104                       couple times.
 105
 106         """
 107
 108         LoggingBuildStep.__init__(self, **kwargs)
 109         self.addFactoryArguments(workdir=workdir,
 110                                  mode=mode,
 111                                  alwaysUseLatest=alwaysUseLatest,
 112                                  timeout=timeout,
 113                                  retry=retry,
 114                                  )
 115
 116         assert mode in ("update", "copy", "clobber", "export")
 117         if retry:
 118             delay, repeats = retry
 119             assert isinstance(repeats, int)
 120             assert repeats > 0
 121         self.args = {'mode': mode,
 122                      'workdir': workdir,
 123                      'timeout': timeout,
 124                      'retry': retry,
 125                      'patch': None, # set during .start
 126                      }
 127         self.alwaysUseLatest = alwaysUseLatest
 128
 129         # Compute defaults for descriptions:
 130         description = ["updating"]
 131         descriptionDone = ["update"]
 132         if mode == "clobber":
 133             description = ["checkout"]
 134             # because checkingouting takes too much space
 135             descriptionDone = ["checkout"]
 136         elif mode == "export":
 137             description = ["exporting"]
 138             descriptionDone = ["export"]
 139         self.description = description
 140         self.descriptionDone = descriptionDone
 141
 142     def setDefaultWorkdir(self, workdir):
 143         self.args['workdir'] = self.args['workdir'] or workdir
 144
 145     def describe(self, done=False):
 146         if done:
 147             return self.descriptionDone
 148         return self.description
 149
 150     def computeSourceRevision(self, changes):
 151         """Each subclass must implement this method to do something more
 152         precise than -rHEAD every time. For version control systems that use
 153         repository-wide change numbers (SVN, P4), this can simply take the
 154         maximum such number from all the changes involved in this build. For
 155         systems that do not (CVS), it needs to create a timestamp based upon
 156         the latest Change, the Build's treeStableTimer, and an optional
 157         self.checkoutDelay value."""
 158         return None
 159
 160     def start(self):
 161         if self.notReally:
 162             log.msg("faking %s checkout/update" % self.name)
 163             self.step_status.setText(["fake", self.name, "successful"])
 164             self.addCompleteLog("log",
 165                                 "Faked %s checkout/update 'successful'\n" \
 166                                 % self.name)
 167             return SKIPPED
 168
 169         # what source stamp would this build like to use?
 170         s = self.build.getSourceStamp()
 171         # if branch is None, then use the Step's "default" branch
 172         branch = s.branch or self.branch
 173         # if revision is None, use the latest sources (-rHEAD)
 174         revision = s.revision
 175         if not revision and not self.alwaysUseLatest:
 176             revision = self.computeSourceRevision(s.changes)
 177         # if patch is None, then do not patch the tree after checkout
 178
 179         # 'patch' is None or a tuple of (patchlevel, diff)
 180         patch = s.patch
 181         if patch:
 182             self.addCompleteLog("patch", patch[1])
 183
 184         self.startVC(branch, revision, patch)
 185
 186     def commandComplete(self, cmd):
 187         if cmd.updates.has_key("got_revision"):
 188             got_revision = cmd.updates["got_revision"][-1]
 189             if got_revision is not None:
 190                 self.setProperty("got_revision", str(got_revision), "Source")
 191
 192
 193
 194 class CVS(Source):
 195     """I do CVS checkout/update operations.
 196
 197     Note: if you are doing anonymous/pserver CVS operations, you will need
 198     to manually do a 'cvs login' on each buildslave before the slave has any
 199     hope of success. XXX: fix then, take a cvs password as an argument and
 200     figure out how to do a 'cvs login' on each build
 201     """
 202
 203     name = "cvs"
 204
 205     #progressMetrics = ('output',)
 206     #
 207     # additional things to track: update gives one stderr line per directory
 208     # (starting with 'cvs server: Updating ') (and is fairly stable if files
 209     # is empty), export gives one line per directory (starting with 'cvs
 210     # export: Updating ') and another line per file (starting with U). Would
 211     # be nice to track these, requires grepping LogFile data for lines,
 212     # parsing each line. Might be handy to have a hook in LogFile that gets
 213     # called with each complete line.
 214
 215     def __init__(self, cvsroot, cvsmodule,
 216                  global_options=[], branch=None, checkoutDelay=None,
 217                  login=None,
 218                  **kwargs):
 219
 220         """
 221         @type  cvsroot: string
 222         @param cvsroot: CVS Repository from which the source tree should
 223                         be obtained. '/home/warner/Repository' for local
 224                         or NFS-reachable repositories,
 225                         ':pserver:anon@foo.com:/cvs' for anonymous CVS,
 226                         'user@host.com:/cvs' for non-anonymous CVS or
 227                         CVS over ssh. Lots of possibilities, check the
 228                         CVS documentation for more.
 229
 230         @type  cvsmodule: string
 231         @param cvsmodule: subdirectory of CVS repository that should be
 232                           retrieved
 233
 234         @type  login: string or None
 235         @param login: if not None, a string which will be provided as a
 236                       password to the 'cvs login' command, used when a
 237                       :pserver: method is used to access the repository.
 238                       This login is only needed once, but must be run
 239                       each time (just before the CVS operation) because
 240                       there is no way for the buildslave to tell whether
 241                       it was previously performed or not.
 242
 243         @type  branch: string
 244         @param branch: the default branch name, will be used in a '-r'
 245                        argument to specify which branch of the source tree
 246                        should be used for this checkout. Defaults to None,
 247                        which means to use 'HEAD'.
 248
 249         @type  checkoutDelay: int or None
 250         @param checkoutDelay: if not None, the number of seconds to put
 251                               between the last known Change and the
 252                               timestamp given to the -D argument. This
 253                               defaults to exactly half of the parent
 254                               Build's .treeStableTimer, but it could be
 255                               set to something else if your CVS change
 256                               notification has particularly weird
 257                               latency characteristics.
 258
 259         @type  global_options: list of strings
 260         @param global_options: these arguments are inserted in the cvs
 261                                command line, before the
 262                                'checkout'/'update' command word. See
 263                                'cvs --help-options' for a list of what
 264                                may be accepted here.  ['-r'] will make
 265                                the checked out files read only. ['-r',
 266                                '-R'] will also assume the repository is
 267                                read-only (I assume this means it won't
 268                                use locks to insure atomic access to the
 269                                ,v files)."""
 270
 271         self.checkoutDelay = checkoutDelay
 272         self.branch = branch
 273
 274         Source.__init__(self, **kwargs)
 275         self.addFactoryArguments(cvsroot=cvsroot,
 276                                  cvsmodule=cvsmodule,
 277                                  global_options=global_options,
 278                                  branch=branch,
 279                                  checkoutDelay=checkoutDelay,
 280                                  login=login,
 281                                  )
 282
 283         self.args.update({'cvsroot': cvsroot,
 284                           'cvsmodule': cvsmodule,
 285                           'global_options': global_options,
 286                           'login': login,
 287                           })
 288
 289     def computeSourceRevision(self, changes):
 290         if not changes:
 291             return None
 292         lastChange = max([c.when for c in changes])
 293         if self.checkoutDelay is not None:
 294             when = lastChange + self.checkoutDelay
 295         else:
 296             lastSubmit = max([r.submittedAt for r in self.build.requests])
 297             when = (lastChange + lastSubmit) / 2
 298         return formatdate(when)
 299
 300     def startVC(self, branch, revision, patch):
 301         if self.slaveVersionIsOlderThan("cvs", "1.39"):
 302             # the slave doesn't know to avoid re-using the same sourcedir
 303             # when the branch changes. We have no way of knowing which branch
 304             # the last build used, so if we're using a non-default branch and
 305             # either 'update' or 'copy' modes, it is safer to refuse to
 306             # build, and tell the user they need to upgrade the buildslave.
 307             if (branch != self.branch
 308                 and self.args['mode'] in ("update", "copy")):
 309                 m = ("This buildslave (%s) does not know about multiple "
 310                      "branches, and using mode=%s would probably build the "
 311                      "wrong tree. "
 312                      "Refusing to build. Please upgrade the buildslave to "
 313                      "buildbot-0.7.0 or newer." % (self.build.slavename,
 314                                                    self.args['mode']))
 315                 log.msg(m)
 316                 raise BuildSlaveTooOldError(m)
 317
 318         if branch is None:
 319             branch = "HEAD"
 320         self.args['branch'] = branch
 321         self.args['revision'] = revision
 322         self.args['patch'] = patch
 323
 324         if self.args['branch'] == "HEAD" and self.args['revision']:
 325             # special case. 'cvs update -r HEAD -D today' gives no files
 326             # TODO: figure out why, see if it applies to -r BRANCH
 327             self.args['branch'] = None
 328
 329         # deal with old slaves
 330         warnings = []
 331         slavever = self.slaveVersion("cvs", "old")
 332
 333         if slavever == "old":
 334             # 0.5.0
 335             if self.args['mode'] == "export":
 336                 self.args['export'] = 1
 337             elif self.args['mode'] == "clobber":
 338                 self.args['clobber'] = 1
 339             elif self.args['mode'] == "copy":
 340                 self.args['copydir'] = "source"
 341             self.args['tag'] = self.args['branch']
 342             assert not self.args['patch'] # 0.5.0 slave can't do patch
 343
 344         cmd = LoggedRemoteCommand("cvs", self.args)
 345         self.startCommand(cmd, warnings)
 346
 347
 348 class SVN(Source):
 349     """I perform Subversion checkout/update operations."""
 350
 351     name = 'svn'
 352
 353     def __init__(self, svnurl=None, baseURL=None, defaultBranch=None,
 354                  directory=None, username=None, password=None, **kwargs):
 355         """
 356         @type  svnurl: string
 357         @param svnurl: the URL which points to the Subversion server,
 358                        combining the access method (HTTP, ssh, local file),
 359                        the repository host/port, the repository path, the
 360                        sub-tree within the repository, and the branch to
 361                        check out. Using C{svnurl} does not enable builds of
 362                        alternate branches: use C{baseURL} to enable this.
 363                        Use exactly one of C{svnurl} and C{baseURL}.
 364
 365         @param baseURL: if branches are enabled, this is the base URL to
 366                         which a branch name will be appended. It should
 367                         probably end in a slash. Use exactly one of
 368                         C{svnurl} and C{baseURL}.
 369
 370         @param defaultBranch: if branches are enabled, this is the branch
 371                               to use if the Build does not specify one
 372                               explicitly. It will simply be appended
 373                               to C{baseURL} and the result handed to
 374                               the SVN command.
 375
 376         @param username: username to pass to svn's --username
 377         @param password: username to pass to svn's --password
 378         """
 379
 380         if not kwargs.has_key('workdir') and directory is not None:
 381             # deal with old configs
 382             warn("Please use workdir=, not directory=", DeprecationWarning)
 383             kwargs['workdir'] = directory
 384
 385         self.svnurl = svnurl
 386         self.baseURL = baseURL
 387         self.branch = defaultBranch
 388         self.username = username
 389         self.password = password
 390
 391         Source.__init__(self, **kwargs)
 392         self.addFactoryArguments(svnurl=svnurl,
 393                                  baseURL=baseURL,
 394                                  defaultBranch=defaultBranch,
 395                                  directory=directory,
 396                                  username=username,
 397                                  password=password,
 398                                  )
 399
 400         if not svnurl and not baseURL:
 401             raise ValueError("you must use exactly one of svnurl and baseURL")
 402
 403
 404     def computeSourceRevision(self, changes):
 405         if not changes or None in [c.revision for c in changes]:
 406             return None
 407         lastChange = max([int(c.revision) for c in changes])
 408         return lastChange
 409
 410     def startVC(self, branch, revision, patch):
 411
 412         # handle old slaves
 413         warnings = []
 414         slavever = self.slaveVersion("svn", "old")
 415         if not slavever:
 416             m = "slave does not have the 'svn' command"
 417             raise BuildSlaveTooOldError(m)
 418
 419         if self.slaveVersionIsOlderThan("svn", "1.39"):
 420             # the slave doesn't know to avoid re-using the same sourcedir
 421             # when the branch changes. We have no way of knowing which branch
 422             # the last build used, so if we're using a non-default branch and
 423             # either 'update' or 'copy' modes, it is safer to refuse to
 424             # build, and tell the user they need to upgrade the buildslave.
 425             if (branch != self.branch
 426                 and self.args['mode'] in ("update", "copy")):
 427                 m = ("This buildslave (%s) does not know about multiple "
 428                      "branches, and using mode=%s would probably build the "
 429                      "wrong tree. "
 430                      "Refusing to build. Please upgrade the buildslave to "
 431                      "buildbot-0.7.0 or newer." % (self.build.slavename,
 432                                                    self.args['mode']))
 433                 raise BuildSlaveTooOldError(m)
 434
 435         if slavever == "old":
 436             # 0.5.0 compatibility
 437             if self.args['mode'] in ("clobber", "copy"):
 438                 # TODO: use some shell commands to make up for the
 439                 # deficiency, by blowing away the old directory first (thus
 440                 # forcing a full checkout)
 441                 warnings.append("WARNING: this slave can only do SVN updates"
 442                                 ", not mode=%s\n" % self.args['mode'])
 443                 log.msg("WARNING: this slave only does mode=update")
 444             if self.args['mode'] == "export":
 445                 raise BuildSlaveTooOldError("old slave does not have "
 446                                             "mode=export")
 447             self.args['directory'] = self.args['workdir']
 448             if revision is not None:
 449                 # 0.5.0 can only do HEAD. We have no way of knowing whether
 450                 # the requested revision is HEAD or not, and for
 451                 # slowly-changing trees this will probably do the right
 452                 # thing, so let it pass with a warning
 453                 m = ("WARNING: old slave can only update to HEAD, not "
 454                      "revision=%s" % revision)
 455                 log.msg(m)
 456                 warnings.append(m + "\n")
 457             revision = "HEAD" # interprets this key differently
 458             if patch:
 459                 raise BuildSlaveTooOldError("old slave can't do patch")
 460
 461         if self.svnurl:
 462             assert not branch # we need baseURL= to use branches
 463             self.args['svnurl'] = self.svnurl
 464         else:
 465             self.args['svnurl'] = self.baseURL + branch
 466         self.args['revision'] = revision
 467         self.args['patch'] = patch
 468
 469         if self.username is not None or self.password is not None:
 470             if self.slaveVersionIsOlderThan("svn", "2.8"):
 471                 m = ("This buildslave (%s) does not support svn usernames "
 472                      "and passwords.  "
 473                      "Refusing to build. Please upgrade the buildslave to "
 474                      "buildbot-0.7.10 or newer." % (self.build.slavename,))
 475                 raise BuildSlaveTooOldError(m)
 476             if self.username is not None: self.args['username'] = self.username
 477             if self.password is not None: self.args['password'] = self.password
 478
 479         revstuff = []
 480         if branch is not None and branch != self.branch:
 481             revstuff.append("[branch]")
 482         if revision is not None:
 483             revstuff.append("r%s" % revision)
 484         if patch is not None:
 485             revstuff.append("[patch]")
 486         self.description.extend(revstuff)
 487         self.descriptionDone.extend(revstuff)
 488
 489         cmd = LoggedRemoteCommand("svn", self.args)
 490         self.startCommand(cmd, warnings)
 491
 492
 493 class Darcs(Source):
 494     """Check out a source tree from a Darcs repository at 'repourl'.
 495
 496     Darcs has no concept of file modes. This means the eXecute-bit will be
 497     cleared on all source files. As a result, you may need to invoke
 498     configuration scripts with something like:
 499
 500     C{s(step.Configure, command=['/bin/sh', './configure'])}
 501     """
 502
 503     name = "darcs"
 504
 505     def __init__(self, repourl=None, baseURL=None, defaultBranch=None,
 506                  **kwargs):
 507         """
 508         @type  repourl: string
 509         @param repourl: the URL which points at the Darcs repository. This
 510                         is used as the default branch. Using C{repourl} does
 511                         not enable builds of alternate branches: use
 512                         C{baseURL} to enable this. Use either C{repourl} or
 513                         C{baseURL}, not both.
 514
 515         @param baseURL: if branches are enabled, this is the base URL to
 516                         which a branch name will be appended. It should
 517                         probably end in a slash. Use exactly one of
 518                         C{repourl} and C{baseURL}.
 519
 520         @param defaultBranch: if branches are enabled, this is the branch
 521                               to use if the Build does not specify one
 522                               explicitly. It will simply be appended to
 523                               C{baseURL} and the result handed to the
 524                               'darcs pull' command.
 525         """
 526         self.repourl = repourl
 527         self.baseURL = baseURL
 528         self.branch = defaultBranch
 529         Source.__init__(self, **kwargs)
 530         self.addFactoryArguments(repourl=repourl,
 531                                  baseURL=baseURL,
 532                                  defaultBranch=defaultBranch,
 533                                  )
 534         assert self.args['mode'] != "export", \
 535                "Darcs does not have an 'export' mode"
 536         if (not repourl and not baseURL) or (repourl and baseURL):
 537             raise ValueError("you must provide exactly one of repourl and"
 538                              " baseURL")
 539
 540     def startVC(self, branch, revision, patch):
 541         slavever = self.slaveVersion("darcs")
 542         if not slavever:
 543             m = "slave is too old, does not know about darcs"
 544             raise BuildSlaveTooOldError(m)
 545
 546         if self.slaveVersionIsOlderThan("darcs", "1.39"):
 547             if revision:
 548                 # TODO: revisit this once we implement computeSourceRevision
 549                 m = "0.6.6 slaves can't handle args['revision']"
 550                 raise BuildSlaveTooOldError(m)
 551
 552             # the slave doesn't know to avoid re-using the same sourcedir
 553             # when the branch changes. We have no way of knowing which branch
 554             # the last build used, so if we're using a non-default branch and
 555             # either 'update' or 'copy' modes, it is safer to refuse to
 556             # build, and tell the user they need to upgrade the buildslave.
 557             if (branch != self.branch
 558                 and self.args['mode'] in ("update", "copy")):
 559                 m = ("This buildslave (%s) does not know about multiple "
 560                      "branches, and using mode=%s would probably build the "
 561                      "wrong tree. "
 562                      "Refusing to build. Please upgrade the buildslave to "
 563                      "buildbot-0.7.0 or newer." % (self.build.slavename,
 564                                                    self.args['mode']))
 565                 raise BuildSlaveTooOldError(m)
 566
 567         if self.repourl:
 568             assert not branch # we need baseURL= to use branches
 569             self.args['repourl'] = self.repourl
 570         else:
 571             self.args['repourl'] = self.baseURL + branch
 572         self.args['revision'] = revision
 573         self.args['patch'] = patch
 574
 575         revstuff = []
 576         if branch is not None and branch != self.branch:
 577             revstuff.append("[branch]")
 578         self.description.extend(revstuff)
 579         self.descriptionDone.extend(revstuff)
 580
 581         cmd = LoggedRemoteCommand("darcs", self.args)
 582         self.startCommand(cmd)
 583
 584
 585 class Git(Source):
 586     """Check out a source tree from a git repository 'repourl'."""
 587
 588     name = "git"
 589
 590     def __init__(self, repourl, branch="master", **kwargs):
 591         """
 592         @type  repourl: string
 593         @param repourl: the URL which points at the git repository
 594
 595         @type  branch: string
 596         @param branch: The branch or tag to check out by default. If
 597                        a build specifies a different branch, it will
 598                        be used instead of this.
 599         """
 600         Source.__init__(self, **kwargs)
 601         self.addFactoryArguments(repourl=repourl, branch=branch)
 602         self.args.update({'repourl': repourl,
 603                           'branch': branch})
 604
 605     def computeSourceRevision(self, changes):
 606         if not changes:
 607             return None
 608         return changes[-1].revision
 609
 610     def startVC(self, branch, revision, patch):
 611         if branch is not None:
 612             self.args['branch'] = branch
 613
 614         self.args['revision'] = revision
 615         self.args['patch'] = patch
 616         slavever = self.slaveVersion("git")
 617         if not slavever:
 618             raise BuildSlaveTooOldError("slave is too old, does not know "
 619                                         "about git")
 620         cmd = LoggedRemoteCommand("git", self.args)
 621         self.startCommand(cmd)
 622
 623
 624 class Arch(Source):
 625     """Check out a source tree from an Arch repository named 'archive'
 626     available at 'url'. 'version' specifies which version number (development
 627     line) will be used for the checkout: this is mostly equivalent to a
 628     branch name. This version uses the 'tla' tool to do the checkout, to use
 629     'baz' see L{Bazaar} instead.
 630     """
 631
 632     name = "arch"
 633     # TODO: slaves >0.6.6 will accept args['build-config'], so use it
 634
 635     def __init__(self, url, version, archive=None, **kwargs):
 636         """
 637         @type  url: string
 638         @param url: the Arch coordinates of the repository. This is
 639                     typically an http:// URL, but could also be the absolute
 640                     pathname of a local directory instead.
 641
 642         @type  version: string
 643         @param version: the category--branch--version to check out. This is
 644                         the default branch. If a build specifies a different
 645                         branch, it will be used instead of this.
 646
 647         @type  archive: string
 648         @param archive: The archive name. If provided, it must match the one
 649                         that comes from the repository. If not, the
 650                         repository's default will be used.
 651         """
 652         self.branch = version
 653         Source.__init__(self, **kwargs)
 654         self.addFactoryArguments(url=url,
 655                                  version=version,
 656                                  archive=archive,
 657                                  )
 658         self.args.update({'url': url,
 659                           'archive': archive,
 660                           })
 661
 662     def computeSourceRevision(self, changes):
 663         # in Arch, fully-qualified revision numbers look like:
 664         #  arch@buildbot.sourceforge.net--2004/buildbot--dev--0--patch-104
 665         # For any given builder, all of this is fixed except the patch-104.
 666         # The Change might have any part of the fully-qualified string, so we
 667         # just look for the last part. We return the "patch-NN" string.
 668         if not changes:
 669             return None
 670         lastChange = None
 671         for c in changes:
 672             if not c.revision:
 673                 continue
 674             if c.revision.endswith("--base-0"):
 675                 rev = 0
 676             else:
 677                 i = c.revision.rindex("patch")
 678                 rev = int(c.revision[i+len("patch-"):])
 679             lastChange = max(lastChange, rev)
 680         if lastChange is None:
 681             return None
 682         if lastChange == 0:
 683             return "base-0"
 684         return "patch-%d" % lastChange
 685
 686     def checkSlaveVersion(self, cmd, branch):
 687         warnings = []
 688         slavever = self.slaveVersion(cmd)
 689         if not slavever:
 690             m = "slave is too old, does not know about %s" % cmd
 691             raise BuildSlaveTooOldError(m)
 692
 693         # slave 1.28 and later understand 'revision'
 694         if self.slaveVersionIsOlderThan(cmd, "1.28"):
 695             if not self.alwaysUseLatest:
 696                 # we don't know whether our requested revision is the latest
 697                 # or not. If the tree does not change very quickly, this will
 698                 # probably build the right thing, so emit a warning rather
 699                 # than refuse to build at all
 700                 m = "WARNING, buildslave is too old to use a revision"
 701                 log.msg(m)
 702                 warnings.append(m + "\n")
 703
 704         if self.slaveVersionIsOlderThan(cmd, "1.39"):
 705             # the slave doesn't know to avoid re-using the same sourcedir
 706             # when the branch changes. We have no way of knowing which branch
 707             # the last build used, so if we're using a non-default branch and
 708             # either 'update' or 'copy' modes, it is safer to refuse to
 709             # build, and tell the user they need to upgrade the buildslave.
 710             if (branch != self.branch
 711                 and self.args['mode'] in ("update", "copy")):
 712                 m = ("This buildslave (%s) does not know about multiple "
 713                      "branches, and using mode=%s would probably build the "
 714                      "wrong tree. "
 715                      "Refusing to build. Please upgrade the buildslave to "
 716                      "buildbot-0.7.0 or newer." % (self.build.slavename,
 717                                                    self.args['mode']))
 718                 log.msg(m)
 719                 raise BuildSlaveTooOldError(m)
 720
 721         return warnings
 722
 723     def startVC(self, branch, revision, patch):
 724         self.args['version'] = branch
 725         self.args['revision'] = revision
 726         self.args['patch'] = patch
 727         warnings = self.checkSlaveVersion("arch", branch)
 728
 729         revstuff = []
 730         if branch is not None and branch != self.branch:
 731             revstuff.append("[branch]")
 732         if revision is not None:
 733             revstuff.append("patch%s" % revision)
 734         self.description.extend(revstuff)
 735         self.descriptionDone.extend(revstuff)
 736
 737         cmd = LoggedRemoteCommand("arch", self.args)
 738         self.startCommand(cmd, warnings)
 739
 740
 741 class Bazaar(Arch):
 742     """Bazaar is an alternative client for Arch repositories. baz is mostly
 743     compatible with tla, but archive registration is slightly different."""
 744
 745     # TODO: slaves >0.6.6 will accept args['build-config'], so use it
 746
 747     def __init__(self, url, version, archive, **kwargs):
 748         """
 749         @type  url: string
 750         @param url: the Arch coordinates of the repository. This is
 751                     typically an http:// URL, but could also be the absolute
 752                     pathname of a local directory instead.
 753
 754         @type  version: string
 755         @param version: the category--branch--version to check out
 756
 757         @type  archive: string
 758         @param archive: The archive name (required). This must always match
 759                         the one that comes from the repository, otherwise the
 760                         buildslave will attempt to get sources from the wrong
 761                         archive.
 762         """
 763         self.branch = version
 764         Source.__init__(self, **kwargs)
 765         self.addFactoryArguments(url=url,
 766                                  version=version,
 767                                  archive=archive,
 768                                  )
 769         self.args.update({'url': url,
 770                           'archive': archive,
 771                           })
 772
 773     def startVC(self, branch, revision, patch):
 774         self.args['version'] = branch
 775         self.args['revision'] = revision
 776         self.args['patch'] = patch
 777         warnings = self.checkSlaveVersion("bazaar", branch)
 778
 779         revstuff = []
 780         if branch is not None and branch != self.branch:
 781             revstuff.append("[branch]")
 782         if revision is not None:
 783             revstuff.append("patch%s" % revision)
 784         self.description.extend(revstuff)
 785         self.descriptionDone.extend(revstuff)
 786
 787         cmd = LoggedRemoteCommand("bazaar", self.args)
 788         self.startCommand(cmd, warnings)
 789
 790 class Bzr(Source):
 791     """Check out a source tree from a bzr (Bazaar) repository at 'repourl'.
 792
 793     """
 794
 795     name = "bzr"
 796
 797     def __init__(self, repourl=None, baseURL=None, defaultBranch=None,
 798                  **kwargs):
 799         """
 800         @type  repourl: string
 801         @param repourl: the URL which points at the bzr repository. This
 802                         is used as the default branch. Using C{repourl} does
 803                         not enable builds of alternate branches: use
 804                         C{baseURL} to enable this. Use either C{repourl} or
 805                         C{baseURL}, not both.
 806
 807         @param baseURL: if branches are enabled, this is the base URL to
 808                         which a branch name will be appended. It should
 809                         probably end in a slash. Use exactly one of
 810                         C{repourl} and C{baseURL}.
 811
 812         @param defaultBranch: if branches are enabled, this is the branch
 813                               to use if the Build does not specify one
 814                               explicitly. It will simply be appended to
 815                               C{baseURL} and the result handed to the
 816                               'bzr checkout pull' command.
 817         """
 818         self.repourl = repourl
 819         self.baseURL = baseURL
 820         self.branch = defaultBranch
 821         Source.__init__(self, **kwargs)
 822         self.addFactoryArguments(repourl=repourl,
 823                                  baseURL=baseURL,
 824                                  defaultBranch=defaultBranch,
 825                                  )
 826         if (not repourl and not baseURL) or (repourl and baseURL):
 827             raise ValueError("you must provide exactly one of repourl and"
 828                              " baseURL")
 829
 830     def computeSourceRevision(self, changes):
 831         if not changes:
 832             return None
 833         lastChange = max([int(c.revision) for c in changes])
 834         return lastChange
 835
 836     def startVC(self, branch, revision, patch):
 837         slavever = self.slaveVersion("bzr")
 838         if not slavever:
 839             m = "slave is too old, does not know about bzr"
 840             raise BuildSlaveTooOldError(m)
 841
 842         if self.repourl:
 843             assert not branch # we need baseURL= to use branches
 844             self.args['repourl'] = self.repourl
 845         else:
 846             self.args['repourl'] = self.baseURL + branch
 847         self.args['revision'] = revision
 848         self.args['patch'] = patch
 849
 850         revstuff = []
 851         if branch is not None and branch != self.branch:
 852             revstuff.append("[branch]")
 853         self.description.extend(revstuff)
 854         self.descriptionDone.extend(revstuff)
 855
 856         cmd = LoggedRemoteCommand("bzr", self.args)
 857         self.startCommand(cmd)
 858
 859
 860 class Mercurial(Source):
 861     """Check out a source tree from a mercurial repository 'repourl'."""
 862
 863     name = "hg"
 864
 865     def __init__(self, repourl=None, baseURL=None, defaultBranch=None,
 866                  branchType='dirname', **kwargs):
 867         """
 868         @type  repourl: string
 869         @param repourl: the URL which points at the Mercurial repository.
 870                         This uses the 'default' branch unless defaultBranch is
 871                         specified below and the C{branchType} is set to
 872                         'inrepo'.  It is an error to specify a branch without
 873                         setting the C{branchType} to 'inrepo'.
 874
 875         @param baseURL: if 'dirname' branches are enabled, this is the base URL
 876                         to which a branch name will be appended. It should
 877                         probably end in a slash.  Use exactly one of C{repourl}
 878                         and C{baseURL}.
 879
 880         @param defaultBranch: if branches are enabled, this is the branch
 881                               to use if the Build does not specify one
 882                               explicitly.
 883                               For 'dirname' branches, It will simply be
 884                               appended to C{baseURL} and the result handed to
 885                               the 'hg update' command.
 886                               For 'inrepo' branches, this specifies the named
 887                               revision to which the tree will update after a
 888                               clone.
 889
 890         @param branchType: either 'dirname' or 'inrepo' depending on whether
 891                            the branch name should be appended to the C{baseURL}
 892                            or the branch is a mercurial named branch and can be
 893                            found within the C{repourl}
 894         """
 895         self.repourl = repourl
 896         self.baseURL = baseURL
 897         self.branch = defaultBranch
 898         self.branchType = branchType
 899         Source.__init__(self, **kwargs)
 900         self.addFactoryArguments(repourl=repourl,
 901                                  baseURL=baseURL,
 902                                  defaultBranch=defaultBranch,
 903                                  branchType=branchType,
 904                                  )
 905         if (not repourl and not baseURL) or (repourl and baseURL):
 906             raise ValueError("you must provide exactly one of repourl and"
 907                              " baseURL")
 908
 909     def startVC(self, branch, revision, patch):
 910         slavever = self.slaveVersion("hg")
 911         if not slavever:
 912             raise BuildSlaveTooOldError("slave is too old, does not know "
 913                                         "about hg")
 914
 915         if self.repourl:
 916             # we need baseURL= to use dirname branches
 917             assert self.branchType == 'inrepo' or not branch
 918             self.args['repourl'] = self.repourl
 919             if branch:
 920                 self.args['branch'] = branch
 921         else:
 922             self.args['repourl'] = self.baseURL + branch
 923         self.args['revision'] = revision
 924         self.args['patch'] = patch
 925
 926         revstuff = []
 927         if branch is not None and branch != self.branch:
 928             revstuff.append("[branch]")
 929         self.description.extend(revstuff)
 930         self.descriptionDone.extend(revstuff)
 931
 932         cmd = LoggedRemoteCommand("hg", self.args)
 933         self.startCommand(cmd)
 934
 935     def computeSourceRevision(self, changes):
 936         if not changes:
 937             return None
 938         # without knowing the revision ancestry graph, we can't sort the
 939         # changes at all. So for now, assume they were given to us in sorted
 940         # order, and just pay attention to the last one. See ticket #103 for
 941         # more details.
 942         if len(changes) > 1:
 943             log.msg("Mercurial.computeSourceRevision: warning: "
 944                     "there are %d changes here, assuming the last one is "
 945                     "the most recent" % len(changes))
 946         return changes[-1].revision
 947
 948
 949 class P4(Source):
 950     """ P4 is a class for accessing perforce revision control"""
 951     name = "p4"
 952
 953     def __init__(self, p4base, defaultBranch=None, p4port=None, p4user=None,
 954                  p4passwd=None, p4extra_views=[],
 955                  p4client='buildbot_%(slave)s_%(builder)s', **kwargs):
 956         """
 957         @type  p4base: string
 958         @param p4base: A view into a perforce depot, typically
 959                        "//depot/proj/"
 960
 961         @type  defaultBranch: string
 962         @param defaultBranch: Identify a branch to build by default. Perforce
 963                               is a view based branching system. So, the branch
 964                               is normally the name after the base. For example,
 965                               branch=1.0 is view=//depot/proj/1.0/...
 966                               branch=1.1 is view=//depot/proj/1.1/...
 967
 968         @type  p4port: string
 969         @param p4port: Specify the perforce server to connection in the format
 970                        <host>:<port>. Example "perforce.example.com:1666"
 971
 972         @type  p4user: string
 973         @param p4user: The perforce user to run the command as.
 974
 975         @type  p4passwd: string
 976         @param p4passwd: The password for the perforce user.
 977
 978         @type  p4extra_views: list of tuples
 979         @param p4extra_views: Extra views to be added to
 980                               the client that is being used.
 981
 982         @type  p4client: string
 983         @param p4client: The perforce client to use for this buildslave.
 984         """
 985
 986         self.branch = defaultBranch
 987         Source.__init__(self, **kwargs)
 988         self.addFactoryArguments(p4base=p4base,
 989                                  defaultBranch=defaultBranch,
 990                                  p4port=p4port,
 991                                  p4user=p4user,
 992                                  p4passwd=p4passwd,
 993                                  p4extra_views=p4extra_views,
 994                                  p4client=p4client,
 995                                  )
 996         self.args['p4port'] = p4port
 997         self.args['p4user'] = p4user
 998         self.args['p4passwd'] = p4passwd
 999         self.args['p4base'] = p4base
1000         self.args['p4extra_views'] = p4extra_views
1001         self.p4client = p4client
1002
1003     def setBuild(self, build):
1004         Source.setBuild(self, build)
1005         self.args['p4client'] = self.p4client % {
1006             'slave': build.slavename,
1007             'builder': build.builder.name,
1008         }
1009
1010     def computeSourceRevision(self, changes):
1011         if not changes:
1012             return None
1013         lastChange = max([int(c.revision) for c in changes])
1014         return lastChange
1015
1016     def startVC(self, branch, revision, patch):
1017         slavever = self.slaveVersion("p4")
1018         assert slavever, "slave is too old, does not know about p4"
1019         args = dict(self.args)
1020         args['branch'] = branch or self.branch
1021         args['revision'] = revision
1022         args['patch'] = patch
1023         cmd = LoggedRemoteCommand("p4", args)
1024         self.startCommand(cmd)
1025
1026 class P4Sync(Source):
1027     """This is a partial solution for using a P4 source repository. You are
1028     required to manually set up each build slave with a useful P4
1029     environment, which means setting various per-slave environment variables,
1030     and creating a P4 client specification which maps the right files into
1031     the slave's working directory. Once you have done that, this step merely
1032     performs a 'p4 sync' to update that workspace with the newest files.
1033
1034     Each slave needs the following environment:
1035
1036      - PATH: the 'p4' binary must be on the slave's PATH
1037      - P4USER: each slave needs a distinct user account
1038      - P4CLIENT: each slave needs a distinct client specification
1039
1040     You should use 'p4 client' (?) to set up a client view spec which maps
1041     the desired files into $SLAVEBASE/$BUILDERBASE/source .
1042     """
1043
1044     name = "p4sync"
1045
1046     def __init__(self, p4port, p4user, p4passwd, p4client, **kwargs):
1047         assert kwargs['mode'] == "copy", "P4Sync can only be used in mode=copy"
1048         self.branch = None
1049         Source.__init__(self, **kwargs)
1050         self.addFactoryArguments(p4port=p4port,
1051                                  p4user=p4user,
1052                                  p4passwd=p4passwd,
1053                                  p4client=p4client,
1054                                 )
1055         self.args['p4port'] = p4port
1056         self.args['p4user'] = p4user
1057         self.args['p4passwd'] = p4passwd
1058         self.args['p4client'] = p4client
1059
1060     def computeSourceRevision(self, changes):
1061         if not changes:
1062             return None
1063         lastChange = max([int(c.revision) for c in changes])
1064         return lastChange
1065
1066     def startVC(self, branch, revision, patch):
1067         slavever = self.slaveVersion("p4sync")
1068         assert slavever, "slave is too old, does not know about p4"
1069         cmd = LoggedRemoteCommand("p4sync", self.args)
1070         self.startCommand(cmd)
1071
1072 class Monotone(Source):
1073     """Check out a revision from a monotone server at 'server_addr',
1074     branch 'branch'.  'revision' specifies which revision id to check
1075     out.
1076
1077     This step will first create a local database, if necessary, and then pull
1078     the contents of the server into the database.  Then it will do the
1079     checkout/update from this database."""
1080
1081     name = "monotone"
1082
1083     def __init__(self, server_addr, branch, db_path="monotone.db",
1084                  monotone="monotone",
1085                  **kwargs):
1086         Source.__init__(self, **kwargs)
1087         self.addFactoryArguments(server_addr=server_addr,
1088                                  branch=branch,
1089                                  db_path=db_path,
1090                                  monotone=monotone,
1091                                  )
1092         self.args.update({"server_addr": server_addr,
1093                           "branch": branch,
1094                           "db_path": db_path,
1095                           "monotone": monotone})
1096
1097     def computeSourceRevision(self, changes):
1098         if not changes:
1099             return None
1100         return changes[-1].revision
1101
1102     def startVC(self):
1103         slavever = self.slaveVersion("monotone")
1104         assert slavever, "slave is too old, does not know about monotone"
1105         cmd = LoggedRemoteCommand("monotone", self.args)
1106         self.startCommand(cmd)
1107