buildbot/changes/p4poller.py

   1 # -*- test-case-name: buildbot.test.test_p4poller -*-
   2
   3 # Many thanks to Dave Peticolas for contributing this module
   4
   5 import re
   6 import time
   7
   8 from twisted.python import log, failure
   9 from twisted.internet import defer, reactor
  10 from twisted.internet.utils import getProcessOutput
  11 from twisted.internet.task import LoopingCall
  12
  13 from buildbot import util
  14 from buildbot.changes import base, changes
  15
  16 def get_simple_split(branchfile):
  17     """Splits the branchfile argument and assuming branch is
  18        the first path component in branchfile, will return
  19        branch and file else None."""
  20
  21     index = branchfile.find('/')
  22     if index == -1: return None, None
  23     branch, file = branchfile.split('/', 1)
  24     return branch, file
  25
  26 class P4Source(base.ChangeSource, util.ComparableMixin):
  27     """This source will poll a perforce repository for changes and submit
  28     them to the change master."""
  29
  30     compare_attrs = ["p4port", "p4user", "p4passwd", "p4base",
  31                      "p4bin", "pollinterval", "histmax"]
  32
  33     changes_line_re = re.compile(
  34             r"Change (?P<num>\d+) on \S+ by \S+@\S+ '.+'$")
  35     describe_header_re = re.compile(
  36             r"Change \d+ by (?P<who>\S+)@\S+ on (?P<when>.+)$")
  37     file_re = re.compile(r"^\.\.\. (?P<path>[^#]+)#\d+ \w+$")
  38     datefmt = '%Y/%m/%d %H:%M:%S'
  39
  40     parent = None # filled in when we're added
  41     last_change = None
  42     loop = None
  43     volatile = ['loop']
  44     working = False
  45
  46     def __init__(self, p4port=None, p4user=None, p4passwd=None,
  47                  p4base='//', p4bin='p4',
  48                  split_file=lambda branchfile: (None, branchfile),
  49                  pollinterval=60 * 10, histmax=100):
  50         """
  51         @type  p4port:       string
  52         @param p4port:       p4 port definition (host:portno)
  53         @type  p4user:       string
  54         @param p4user:       p4 user
  55         @type  p4passwd:     string
  56         @param p4passwd:     p4 passwd
  57         @type  p4base:       string
  58         @param p4base:       p4 file specification to limit a poll to
  59                              without the trailing '...' (i.e., //)
  60         @type  p4bin:        string
  61         @param p4bin:        path to p4 binary, defaults to just 'p4'
  62         @type  split_file:   func
  63         $param split_file:   splits a filename into branch and filename.
  64         @type  pollinterval: int
  65         @param pollinterval: interval in seconds between polls
  66         @type  histmax:      int
  67         @param histmax:      maximum number of changes to look back through
  68         """
  69
  70         self.p4port = p4port
  71         self.p4user = p4user
  72         self.p4passwd = p4passwd
  73         self.p4base = p4base
  74         self.p4bin = p4bin
  75         self.split_file = split_file
  76         self.pollinterval = pollinterval
  77         self.histmax = histmax
  78
  79     def startService(self):
  80         self.loop = LoopingCall(self.checkp4)
  81         base.ChangeSource.startService(self)
  82
  83         # Don't start the loop just yet because the reactor isn't running.
  84         # Give it a chance to go and install our SIGCHLD handler before
  85         # spawning processes.
  86         reactor.callLater(0, self.loop.start, self.pollinterval)
  87
  88     def stopService(self):
  89         self.loop.stop()
  90         return base.ChangeSource.stopService(self)
  91
  92     def describe(self):
  93         return "p4source %s %s" % (self.p4port, self.p4base)
  94
  95     def checkp4(self):
  96         # Our return value is only used for unit testing.
  97         if self.working:
  98             log.msg("Skipping checkp4 because last one has not finished")
  99             return defer.succeed(None)
 100         else:
 101             self.working = True
 102             d = self._get_changes()
 103             d.addCallback(self._process_changes)
 104             d.addBoth(self._finished)
 105             return d
 106
 107     def _finished(self, res):
 108         assert self.working
 109         self.working = False
 110
 111         # Again, the return value is only for unit testing.
 112         # If there's a failure, log it so it isn't lost.
 113         if isinstance(res, failure.Failure):
 114             log.msg('P4 poll failed: %s' % res)
 115         return res
 116
 117     def _get_changes(self):
 118         args = []
 119         if self.p4port:
 120             args.extend(['-p', self.p4port])
 121         if self.p4user:
 122             args.extend(['-u', self.p4user])
 123         if self.p4passwd:
 124             args.extend(['-P', self.p4passwd])
 125         args.extend(['changes', '-m', str(self.histmax), self.p4base + '...'])
 126         env = {}
 127         return getProcessOutput(self.p4bin, args, env)
 128
 129     def _process_changes(self, result):
 130         last_change = self.last_change
 131         changelists = []
 132         for line in result.split('\n'):
 133             line = line.strip()
 134             if not line: continue
 135             m = self.changes_line_re.match(line)
 136             assert m, "Unexpected 'p4 changes' output: %r" % result
 137             num = m.group('num')
 138             if last_change is None:
 139                 log.msg('P4Poller: starting at change %s' % num)
 140                 self.last_change = num
 141                 return []
 142             if last_change == num:
 143                 break
 144             changelists.append(num)
 145         changelists.reverse() # oldest first
 146
 147         # Retrieve each sequentially.
 148         d = defer.succeed(None)
 149         for c in changelists:
 150             d.addCallback(self._get_describe, c)
 151             d.addCallback(self._process_describe, c)
 152         return d
 153
 154     def _get_describe(self, dummy, num):
 155         args = []
 156         if self.p4port:
 157             args.extend(['-p', self.p4port])
 158         if self.p4user:
 159             args.extend(['-u', self.p4user])
 160         if self.p4passwd:
 161             args.extend(['-P', self.p4passwd])
 162         args.extend(['describe', '-s', num])
 163         env = {}
 164         d = getProcessOutput(self.p4bin, args, env)
 165         return d
 166
 167     def _process_describe(self, result, num):
 168         lines = result.split('\n')
 169         m = self.describe_header_re.match(lines[0])
 170         assert m, "Unexpected 'p4 describe -s' result: %r" % result
 171         who = m.group('who')
 172         when = time.mktime(time.strptime(m.group('when'), self.datefmt))
 173         comments = ''
 174         while not lines[0].startswith('Affected files'):
 175             comments += lines.pop(0) + '\n'
 176         lines.pop(0) # affected files
 177
 178         branch_files = {} # dict for branch mapped to file(s)
 179         while lines:
 180             line = lines.pop(0).strip()
 181             if not line: continue
 182             m = self.file_re.match(line)
 183             assert m, "Invalid file line: %r" % line
 184             path = m.group('path')
 185             if path.startswith(self.p4base):
 186                 branch, file = self.split_file(path[len(self.p4base):])
 187                 if (branch == None and file == None): continue
 188                 if branch_files.has_key(branch):
 189                     branch_files[branch].append(file)
 190                 else:
 191                     branch_files[branch] = [file]
 192
 193         for branch in branch_files:
 194             c = changes.Change(who=who,
 195                                files=branch_files[branch],
 196                                comments=comments,
 197                                revision=num,
 198                                when=when,
 199                                branch=branch)
 200             self.parent.addChange(c)
 201
 202         self.last_change = num