buildbot/changes/p4poller.py

   1 # -*- test-case-name: buildbot.test.test_p4poller -*-
   2
   3 # Many thanks to Dave Peticolas for contributing this module
   4
   5 import re
   6 import time
   7
   8 from twisted.python import log, failure
   9 from twisted.internet import defer, reactor
  10 from twisted.internet.utils import getProcessOutput
  11 from twisted.internet.task import LoopingCall
  12
  13 from buildbot import util
  14 from buildbot.changes import base, changes
  15
  16 def get_simple_split(branchfile):
  17     """Splits the branchfile argument and assuming branch is
  18        the first path component in branchfile, will return
  19        branch and file else None."""
  20
  21     index = branchfile.find('/')
  22     if index == -1: return None, None
  23     branch, file = branchfile.split('/', 1)
  24     return branch, file
  25
  26 class P4Source(base.ChangeSource, util.ComparableMixin):
  27     """This source will poll a perforce repository for changes and submit
  28     them to the change master."""
  29
  30     compare_attrs = ["p4port", "p4user", "p4passwd", "p4base",
  31                      "p4bin", "pollinterval"]
  32
  33     changes_line_re = re.compile(
  34             r"Change (?P<num>\d+) on \S+ by \S+@\S+ '.+'$")
  35     describe_header_re = re.compile(
  36             r"Change \d+ by (?P<who>\S+)@\S+ on (?P<when>.+)$")
  37     file_re = re.compile(r"^\.\.\. (?P<path>[^#]+)#\d+ \w+$")
  38     datefmt = '%Y/%m/%d %H:%M:%S'
  39
  40     parent = None # filled in when we're added
  41     last_change = None
  42     loop = None
  43     working = False
  44
  45     def __init__(self, p4port=None, p4user=None, p4passwd=None,
  46                  p4base='//', p4bin='p4',
  47                  split_file=lambda branchfile: (None, branchfile),
  48                  pollinterval=60 * 10, histmax=None):
  49         """
  50         @type  p4port:       string
  51         @param p4port:       p4 port definition (host:portno)
  52         @type  p4user:       string
  53         @param p4user:       p4 user
  54         @type  p4passwd:     string
  55         @param p4passwd:     p4 passwd
  56         @type  p4base:       string
  57         @param p4base:       p4 file specification to limit a poll to
  58                              without the trailing '...' (i.e., //)
  59         @type  p4bin:        string
  60         @param p4bin:        path to p4 binary, defaults to just 'p4'
  61         @type  split_file:   func
  62         $param split_file:   splits a filename into branch and filename.
  63         @type  pollinterval: int
  64         @param pollinterval: interval in seconds between polls
  65         @type  histmax:      int
  66         @param histmax:      (obsolete) maximum number of changes to look back through.
  67                              ignored; accepted for backwards compatibility.
  68         """
  69
  70         self.p4port = p4port
  71         self.p4user = p4user
  72         self.p4passwd = p4passwd
  73         self.p4base = p4base
  74         self.p4bin = p4bin
  75         self.split_file = split_file
  76         self.pollinterval = pollinterval
  77         self.loop = LoopingCall(self.checkp4)
  78
  79     def startService(self):
  80         base.ChangeSource.startService(self)
  81
  82         # Don't start the loop just yet because the reactor isn't running.
  83         # Give it a chance to go and install our SIGCHLD handler before
  84         # spawning processes.
  85         reactor.callLater(0, self.loop.start, self.pollinterval)
  86
  87     def stopService(self):
  88         self.loop.stop()
  89         return base.ChangeSource.stopService(self)
  90
  91     def describe(self):
  92         return "p4source %s %s" % (self.p4port, self.p4base)
  93
  94     def checkp4(self):
  95         # Our return value is only used for unit testing.
  96         if self.working:
  97             log.msg("Skipping checkp4 because last one has not finished")
  98             return defer.succeed(None)
  99         else:
 100             self.working = True
 101             d = self._get_changes()
 102             d.addCallback(self._process_changes)
 103             d.addBoth(self._finished)
 104             return d
 105
 106     def _finished(self, res):
 107         assert self.working
 108         self.working = False
 109
 110         # Again, the return value is only for unit testing.
 111         # If there's a failure, log it so it isn't lost.
 112         if isinstance(res, failure.Failure):
 113             log.msg('P4 poll failed: %s' % res)
 114             return None
 115         return res
 116
 117     def _get_changes(self):
 118         args = []
 119         if self.p4port:
 120             args.extend(['-p', self.p4port])
 121         if self.p4user:
 122             args.extend(['-u', self.p4user])
 123         if self.p4passwd:
 124             args.extend(['-P', self.p4passwd])
 125         args.extend(['changes'])
 126         if self.last_change is not None:
 127             args.extend(['%s...@%d,now' % (self.p4base, self.last_change+1)])
 128         else:
 129             args.extend(['-m', '1', '%s...' % (self.p4base,)])
 130         env = {}
 131         return getProcessOutput(self.p4bin, args, env)
 132
 133     def _process_changes(self, result):
 134         last_change = self.last_change
 135         changelists = []
 136         for line in result.split('\n'):
 137             line = line.strip()
 138             if not line: continue
 139             m = self.changes_line_re.match(line)
 140             assert m, "Unexpected 'p4 changes' output: %r" % result
 141             num = int(m.group('num'))
 142             if last_change is None:
 143                 log.msg('P4Poller: starting at change %d' % num)
 144                 self.last_change = num
 145                 return []
 146             changelists.append(num)
 147         changelists.reverse() # oldest first
 148
 149         # Retrieve each sequentially.
 150         d = defer.succeed(None)
 151         for c in changelists:
 152             d.addCallback(self._get_describe, c)
 153             d.addCallback(self._process_describe, c)
 154         return d
 155
 156     def _get_describe(self, dummy, num):
 157         args = []
 158         if self.p4port:
 159             args.extend(['-p', self.p4port])
 160         if self.p4user:
 161             args.extend(['-u', self.p4user])
 162         if self.p4passwd:
 163             args.extend(['-P', self.p4passwd])
 164         args.extend(['describe', '-s', str(num)])
 165         env = {}
 166         d = getProcessOutput(self.p4bin, args, env)
 167         return d
 168
 169     def _process_describe(self, result, num):
 170         lines = result.split('\n')
 171         # SF#1555985: Wade Brainerd reports a stray ^M at the end of the date
 172         # field. The rstrip() is intended to remove that.
 173         lines[0] = lines[0].rstrip()
 174         m = self.describe_header_re.match(lines[0])
 175         assert m, "Unexpected 'p4 describe -s' result: %r" % result
 176         who = m.group('who')
 177         when = time.mktime(time.strptime(m.group('when'), self.datefmt))
 178         comments = ''
 179         while not lines[0].startswith('Affected files'):
 180             comments += lines.pop(0) + '\n'
 181         lines.pop(0) # affected files
 182
 183         branch_files = {} # dict for branch mapped to file(s)
 184         while lines:
 185             line = lines.pop(0).strip()
 186             if not line: continue
 187             m = self.file_re.match(line)
 188             assert m, "Invalid file line: %r" % line
 189             path = m.group('path')
 190             if path.startswith(self.p4base):
 191                 branch, file = self.split_file(path[len(self.p4base):])
 192                 if (branch == None and file == None): continue
 193                 if branch_files.has_key(branch):
 194                     branch_files[branch].append(file)
 195                 else:
 196                     branch_files[branch] = [file]
 197
 198         for branch in branch_files:
 199             c = changes.Change(who=who,
 200                                files=branch_files[branch],
 201                                comments=comments,
 202                                revision=num,
 203                                when=when,
 204                                branch=branch)
 205             self.parent.addChange(c)
 206
 207         self.last_change = num