master/contrib/windows/buildbot_service.py

   1 # Runs the build-bot as a Windows service.
   2 # To use:
   3 # * Install and configure buildbot as per normal (ie, running
   4 #  'setup.py install' from the source directory).
   5 #
   6 # * Configure any number of build-bot directories (slaves or masters), as
   7 #   per the buildbot instructions.  Test these directories normally by
   8 #   using the (possibly modified) "buildbot.bat" file and ensure everything
   9 #   is working as expected.
  10 #
  11 # * Install the buildbot service.  Execute the command:
  12 #   % python buildbot_service.py
  13 #   To see installation options.  You probably want to specify:
  14 #   + --username and --password options to specify the user to run the
  15 #   + --startup auto to have the service start at boot time.
  16 #
  17 #   For example:
  18 #   % python buildbot_service.py --user mark --password secret \
  19 #     --startup auto install
  20 #   Alternatively, you could execute:
  21 #   % python buildbot_service.py install
  22 #   to install the service with default options, then use Control Panel
  23 #   to configure it.
  24 #
  25 # * Start the service specifying the name of all buildbot directories as
  26 #   service args.  This can be done one of 2 ways:
  27 #   - Execute the command:
  28 #     % python buildbot_service.py start "dir_name1" "dir_name2"
  29 #   or:
  30 #   - Start Control Panel->Administrative Tools->Services
  31 #   - Locate the previously installed buildbot service.
  32 #   - Open the "properties" for the service.
  33 #   - Enter the directory names into the "Start Parameters" textbox.  The
  34 #     directory names must be fully qualified, and surrounded in quotes if
  35 #    they include spaces.
  36 #   - Press the "Start"button.
  37 #   Note that the service will automatically use the previously specified
  38 #   directories if no arguments are specified. This means the directories
  39 #   need only be specified when the directories to use have changed (and
  40 #   therefore also the first time buildbot is configured)
  41 #
  42 # * The service should now be running.  You should check the Windows
  43 #   event log.  If all goes well, you should see some information messages
  44 #   telling you the buildbot has successfully started.
  45 #
  46 # * If you change the buildbot configuration, you must restart the service.
  47 #   There is currently no way to ask a running buildbot to reload the
  48 #   config.  You can restart by executing:
  49 #   % python buildbot_service.py restart
  50 #
  51 # Troubleshooting:
  52 # * Check the Windows event log for any errors.
  53 # * Check the "twistd.log" file in your buildbot directories - once each
  54 #   bot has been started it just writes to this log as normal.
  55 # * Try executing:
  56 #   % python buildbot_service.py debug
  57 #   This will execute the buildbot service in "debug" mode, and allow you to
  58 #   see all messages etc generated. If the service works in debug mode but
  59 #   not as a real service, the error probably relates to the environment or
  60 #   permissions of the user configured to run the service (debug mode runs as
  61 #   the currently logged in user, not the service user)
  62 # * Ensure you have the latest pywin32 build available, at least version 206.
  63
  64 # Written by Mark Hammond, 2006.
  65
  66 import sys
  67 import os
  68 import threading
  69
  70 import pywintypes
  71 import winerror
  72 import win32con
  73 import win32api
  74 import win32event
  75 import win32file
  76 import win32pipe
  77 import win32process
  78 import win32security
  79 import win32service
  80 import win32serviceutil
  81 import servicemanager
  82
  83 # Are we running in a py2exe environment?
  84 is_frozen = hasattr(sys, "frozen")
  85
  86 # Taken from the Zope service support - each "child" is run as a sub-process
  87 # (trying to run multiple twisted apps in the same process is likely to screw
  88 # stdout redirection etc).
  89 # Note that unlike the Zope service, we do *not* attempt to detect a failed
  90 # client and perform restarts - buildbot itself does a good job
  91 # at reconnecting, and Windows itself provides restart semantics should
  92 # everything go pear-shaped.
  93
  94 # We execute a new thread that captures the tail of the output from our child
  95 # process. If the child fails, it is written to the event log.
  96 # This process is unconditional, and the output is never written to disk
  97 # (except obviously via the event log entry)
  98 # Size of the blocks we read from the child process's output.
  99 CHILDCAPTURE_BLOCK_SIZE = 80
 100 # The number of BLOCKSIZE blocks we keep as process output.
 101 CHILDCAPTURE_MAX_BLOCKS = 200
 102
 103
 104 class BBService(win32serviceutil.ServiceFramework):
 105     _svc_name_ = 'BuildBot'
 106     _svc_display_name_ = _svc_name_
 107     _svc_description_ = 'Manages local buildbot slaves and masters - ' \
 108                         'see http://buildbot.sourceforge.net'
 109
 110     def __init__(self, args):
 111         win32serviceutil.ServiceFramework.__init__(self, args)
 112
 113         # Create an event which we will use to wait on. The "service stop"
 114         # request will set this event.
 115         # * We must make it inheritable so we can pass it to the child
 116         #   process via the cmd-line
 117         # * Must be manual reset so each child process and our service
 118         #   all get woken from a single set of the event.
 119         sa = win32security.SECURITY_ATTRIBUTES()
 120         sa.bInheritHandle = True
 121         self.hWaitStop = win32event.CreateEvent(sa, True, False, None)
 122
 123         self.args = args
 124         self.dirs = None
 125         self.runner_prefix = None
 126
 127         # Patch up the service messages file in a frozen exe.
 128         # (We use the py2exe option that magically bundles the .pyd files
 129         # into the .zip file - so servicemanager.pyd doesn't exist.)
 130         if is_frozen and servicemanager.RunningAsService():
 131             msg_file = os.path.join(os.path.dirname(sys.executable),
 132                                     "buildbot.msg")
 133             if os.path.isfile(msg_file):
 134                 servicemanager.Initialize("BuildBot", msg_file)
 135             else:
 136                 self.warning("Strange - '%s' does not exist" % (msg_file, ))
 137
 138     def _checkConfig(self):
 139         # Locate our child process runner (but only when run from source)
 140         if not is_frozen:
 141             # Running from source
 142             python_exe = os.path.join(sys.prefix, "python.exe")
 143             if not os.path.isfile(python_exe):
 144                 # for ppl who build Python itself from source.
 145                 python_exe = os.path.join(sys.prefix, "PCBuild", "python.exe")
 146             if not os.path.isfile(python_exe):
 147                 self.error("Can not find python.exe to spawn subprocess")
 148                 return False
 149
 150             me = __file__
 151             if me.endswith(".pyc") or me.endswith(".pyo"):
 152                 me = me[:-1]
 153
 154             self.runner_prefix = '"%s" "%s"' % (python_exe, me)
 155         else:
 156             # Running from a py2exe built executable - our child process is
 157             # us (but with the funky cmdline args!)
 158             self.runner_prefix = '"' + sys.executable + '"'
 159
 160         # Now our arg processing - this may be better handled by a
 161         # twisted/buildbot style config file - but as of time of writing,
 162         # MarkH is clueless about such things!
 163
 164         # Note that the "arguments" you type into Control Panel for the
 165         # service do *not* persist - they apply only when you click "start"
 166         # on the service. When started by Windows, args are never presented.
 167         # Thus, it is the responsibility of the service to persist any args.
 168
 169         # so, when args are presented, we save them as a "custom option". If
 170         # they are not presented, we load them from the option.
 171         self.dirs = []
 172         if len(self.args) > 1:
 173             dir_string = os.pathsep.join(self.args[1:])
 174             save_dirs = True
 175         else:
 176             dir_string = win32serviceutil.GetServiceCustomOption(self,
 177                                                             "directories")
 178             save_dirs = False
 179
 180         if not dir_string:
 181             self.error("You must specify the buildbot directories as "
 182                        "parameters to the service.\nStopping the service.")
 183             return False
 184
 185         dirs = dir_string.split(os.pathsep)
 186         for d in dirs:
 187             d = os.path.abspath(d)
 188             sentinal = os.path.join(d, "buildbot.tac")
 189             if os.path.isfile(sentinal):
 190                 self.dirs.append(d)
 191             else:
 192                 msg = "Directory '%s' is not a buildbot dir - ignoring" \
 193                       % (d, )
 194                 self.warning(msg)
 195         if not self.dirs:
 196             self.error("No valid buildbot directories were specified.\n"
 197                        "Stopping the service.")
 198             return False
 199         if save_dirs:
 200             dir_string = os.pathsep.join(self.dirs).encode("mbcs")
 201             win32serviceutil.SetServiceCustomOption(self, "directories",
 202                                                     dir_string)
 203         return True
 204
 205     def SvcStop(self):
 206         # Tell the SCM we are starting the stop process.
 207         self.ReportServiceStatus(win32service.SERVICE_STOP_PENDING)
 208         # Set the stop event - the main loop takes care of termination.
 209         win32event.SetEvent(self.hWaitStop)
 210
 211     # SvcStop only gets triggered when the user explictly stops (or restarts)
 212     # the service.  To shut the service down cleanly when Windows is shutting
 213     # down, we also need to hook SvcShutdown.
 214     SvcShutdown = SvcStop
 215
 216     def SvcDoRun(self):
 217         if not self._checkConfig():
 218             # stopped status set by caller.
 219             return
 220
 221         self.logmsg(servicemanager.PYS_SERVICE_STARTED)
 222
 223         child_infos = []
 224
 225         for bbdir in self.dirs:
 226             self.info("Starting BuildBot in directory '%s'" % (bbdir, ))
 227             hstop = self.hWaitStop
 228
 229             cmd = '%s --spawn %d start %s' % (self.runner_prefix, hstop, bbdir)
 230             #print "cmd is", cmd
 231             h, t, output = self.createProcess(cmd)
 232             child_infos.append((bbdir, h, t, output))
 233
 234         while child_infos:
 235             handles = [self.hWaitStop] + [i[1] for i in child_infos]
 236
 237             rc = win32event.WaitForMultipleObjects(handles,
 238                                                    0, # bWaitAll
 239                                                    win32event.INFINITE)
 240             if rc == win32event.WAIT_OBJECT_0:
 241                 # user sent a stop service request
 242                 break
 243             else:
 244                 # A child process died.  For now, just log the output
 245                 # and forget the process.
 246                 index = rc - win32event.WAIT_OBJECT_0 - 1
 247                 bbdir, dead_handle, dead_thread, output_blocks = \
 248                                                         child_infos[index]
 249                 status = win32process.GetExitCodeProcess(dead_handle)
 250                 output = "".join(output_blocks)
 251                 if not output:
 252                     output = "The child process generated no output. " \
 253                              "Please check the twistd.log file in the " \
 254                              "indicated directory."
 255
 256                 self.warning("BuildBot for directory %r terminated with "
 257                              "exit code %d.\n%s" % (bbdir, status, output))
 258
 259                 del child_infos[index]
 260
 261                 if not child_infos:
 262                     self.warning("All BuildBot child processes have "
 263                                  "terminated.  Service stopping.")
 264
 265         # Either no child processes left, or stop event set.
 266         self.ReportServiceStatus(win32service.SERVICE_STOP_PENDING)
 267
 268         # The child processes should have also seen our stop signal
 269         # so wait for them to terminate.
 270         for bbdir, h, t, output in child_infos:
 271             for i in range(10): # 30 seconds to shutdown...
 272                 self.ReportServiceStatus(win32service.SERVICE_STOP_PENDING)
 273                 rc = win32event.WaitForSingleObject(h, 3000)
 274                 if rc == win32event.WAIT_OBJECT_0:
 275                     break
 276             # Process terminated - no need to try harder.
 277             if rc == win32event.WAIT_OBJECT_0:
 278                 break
 279
 280             self.ReportServiceStatus(win32service.SERVICE_STOP_PENDING)
 281             # If necessary, kill it
 282             if win32process.GetExitCodeProcess(h)==win32con.STILL_ACTIVE:
 283                 self.warning("BuildBot process at %r failed to terminate - "
 284                              "killing it" % (bbdir, ))
 285                 win32api.TerminateProcess(h, 3)
 286             self.ReportServiceStatus(win32service.SERVICE_STOP_PENDING)
 287
 288             # Wait for the redirect thread - it should have died as the remote
 289             # process terminated.
 290             # As we are shutting down, we do the join with a little more care,
 291             # reporting progress as we wait (even though we never will <wink>)
 292             for i in range(5):
 293                 t.join(1)
 294                 self.ReportServiceStatus(win32service.SERVICE_STOP_PENDING)
 295                 if not t.isAlive():
 296                     break
 297             else:
 298                 self.warning("Redirect thread did not stop!")
 299
 300         # All done.
 301         self.logmsg(servicemanager.PYS_SERVICE_STOPPED)
 302
 303     #
 304     # Error reporting/logging functions.
 305     #
 306
 307     def logmsg(self, event):
 308         # log a service event using servicemanager.LogMsg
 309         try:
 310             servicemanager.LogMsg(servicemanager.EVENTLOG_INFORMATION_TYPE,
 311                                   event,
 312                                   (self._svc_name_,
 313                                    " (%s)" % self._svc_display_name_))
 314         except win32api.error, details:
 315             # Failed to write a log entry - most likely problem is
 316             # that the event log is full.  We don't want this to kill us
 317             try:
 318                 print "FAILED to write INFO event", event, ":", details
 319             except IOError:
 320                 # No valid stdout!  Ignore it.
 321                 pass
 322
 323     def _dolog(self, func, msg):
 324         try:
 325             func(msg)
 326         except win32api.error, details:
 327             # Failed to write a log entry - most likely problem is
 328             # that the event log is full.  We don't want this to kill us
 329             try:
 330                 print "FAILED to write event log entry:", details
 331                 print msg
 332             except IOError:
 333                 pass
 334
 335     def info(self, s):
 336         self._dolog(servicemanager.LogInfoMsg, s)
 337
 338     def warning(self, s):
 339         self._dolog(servicemanager.LogWarningMsg, s)
 340
 341     def error(self, s):
 342         self._dolog(servicemanager.LogErrorMsg, s)
 343
 344     # Functions that spawn a child process, redirecting any output.
 345     # Although builtbot itself does this, it is very handy to debug issues
 346     # such as ImportErrors that happen before buildbot has redirected.
 347
 348     def createProcess(self, cmd):
 349         hInputRead, hInputWriteTemp = self.newPipe()
 350         hOutReadTemp, hOutWrite = self.newPipe()
 351         pid = win32api.GetCurrentProcess()
 352         # This one is duplicated as inheritable.
 353         hErrWrite = win32api.DuplicateHandle(pid, hOutWrite, pid, 0, 1,
 354                                        win32con.DUPLICATE_SAME_ACCESS)
 355
 356         # These are non-inheritable duplicates.
 357         hOutRead = self.dup(hOutReadTemp)
 358         hInputWrite = self.dup(hInputWriteTemp)
 359         # dup() closed hOutReadTemp, hInputWriteTemp
 360
 361         si = win32process.STARTUPINFO()
 362         si.hStdInput = hInputRead
 363         si.hStdOutput = hOutWrite
 364         si.hStdError = hErrWrite
 365         si.dwFlags = win32process.STARTF_USESTDHANDLES | \
 366                      win32process.STARTF_USESHOWWINDOW
 367         si.wShowWindow = win32con.SW_HIDE
 368
 369         # pass True to allow handles to be inherited.  Inheritance is
 370         # problematic in general, but should work in the controlled
 371         # circumstances of a service process.
 372         create_flags = win32process.CREATE_NEW_CONSOLE
 373         # info is (hProcess, hThread, pid, tid)
 374         info = win32process.CreateProcess(None, cmd, None, None, True,
 375                                           create_flags, None, None, si)
 376         # (NOTE: these really aren't necessary for Python - they are closed
 377         # as soon as they are collected)
 378         hOutWrite.Close()
 379         hErrWrite.Close()
 380         hInputRead.Close()
 381         # We don't use stdin
 382         hInputWrite.Close()
 383
 384         # start a thread collecting output
 385         blocks = []
 386         t = threading.Thread(target=self.redirectCaptureThread,
 387                              args = (hOutRead, blocks))
 388         t.start()
 389         return info[0], t, blocks
 390
 391     def redirectCaptureThread(self, handle, captured_blocks):
 392         # One of these running per child process we are watching.  It
 393         # handles both stdout and stderr on a single handle. The read data is
 394         # never referenced until the thread dies - so no need for locks
 395         # around self.captured_blocks.
 396         #self.info("Redirect thread starting")
 397         while 1:
 398             try:
 399                 ec, data = win32file.ReadFile(handle, CHILDCAPTURE_BLOCK_SIZE)
 400             except pywintypes.error, err:
 401                 # ERROR_BROKEN_PIPE means the child process closed the
 402                 # handle - ie, it terminated.
 403                 if err[0] != winerror.ERROR_BROKEN_PIPE:
 404                     self.warning("Error reading output from process: %s" % err)
 405                 break
 406             captured_blocks.append(data)
 407             del captured_blocks[CHILDCAPTURE_MAX_BLOCKS:]
 408         handle.Close()
 409         #self.info("Redirect capture thread terminating")
 410
 411     def newPipe(self):
 412         sa = win32security.SECURITY_ATTRIBUTES()
 413         sa.bInheritHandle = True
 414         return win32pipe.CreatePipe(sa, 0)
 415
 416     def dup(self, pipe):
 417         # create a duplicate handle that is not inherited, so that
 418         # it can be closed in the parent.  close the original pipe in
 419         # the process.
 420         pid = win32api.GetCurrentProcess()
 421         dup = win32api.DuplicateHandle(pid, pipe, pid, 0, 0,
 422                                        win32con.DUPLICATE_SAME_ACCESS)
 423         pipe.Close()
 424         return dup
 425
 426
 427 # Service registration and startup
 428
 429
 430 def RegisterWithFirewall(exe_name, description):
 431     # Register our executable as an exception with Windows Firewall.
 432     # taken from  http://msdn.microsoft.com/library/default.asp?url=\
 433     #/library/en-us/ics/ics/wf_adding_an_application.asp
 434     from win32com.client import Dispatch
 435     #  Set constants
 436     NET_FW_PROFILE_DOMAIN = 0
 437     NET_FW_PROFILE_STANDARD = 1
 438
 439     # Scope
 440     NET_FW_SCOPE_ALL = 0
 441
 442     # IP Version - ANY is the only allowable setting for now
 443     NET_FW_IP_VERSION_ANY = 2
 444
 445     fwMgr = Dispatch("HNetCfg.FwMgr")
 446
 447     # Get the current profile for the local firewall policy.
 448     profile = fwMgr.LocalPolicy.CurrentProfile
 449
 450     app = Dispatch("HNetCfg.FwAuthorizedApplication")
 451
 452     app.ProcessImageFileName = exe_name
 453     app.Name = description
 454     app.Scope = NET_FW_SCOPE_ALL
 455     # Use either Scope or RemoteAddresses, but not both
 456     #app.RemoteAddresses = "*"
 457     app.IpVersion = NET_FW_IP_VERSION_ANY
 458     app.Enabled = True
 459
 460     # Use this line if you want to add the app, but disabled.
 461     #app.Enabled = False
 462
 463     profile.AuthorizedApplications.Add(app)
 464
 465
 466 # A custom install function.
 467
 468
 469 def CustomInstall(opts):
 470     # Register this process with the Windows Firewaall
 471     import pythoncom
 472     try:
 473         RegisterWithFirewall(sys.executable, "BuildBot")
 474     except pythoncom.com_error, why:
 475         print "FAILED to register with the Windows firewall"
 476         print why
 477
 478
 479 # Magic code to allow shutdown.  Note that this code is executed in
 480 # the *child* process, by way of the service process executing us with
 481 # special cmdline args (which includes the service stop handle!)
 482
 483
 484 def _RunChild(runfn):
 485     del sys.argv[1] # The --spawn arg.
 486     # Create a new thread that just waits for the event to be signalled.
 487     t = threading.Thread(target=_WaitForShutdown,
 488                          args = (int(sys.argv[1]), )
 489                          )
 490     del sys.argv[1] # The stop handle
 491     # This child process will be sent a console handler notification as
 492     # users log off, or as the system shuts down.  We want to ignore these
 493     # signals as the service parent is responsible for our shutdown.
 494
 495     def ConsoleHandler(what):
 496         # We can ignore *everything* - ctrl+c will never be sent as this
 497         # process is never attached to a console the user can press the
 498         # key in!
 499         return True
 500     win32api.SetConsoleCtrlHandler(ConsoleHandler, True)
 501     t.setDaemon(True) # we don't want to wait for this to stop!
 502     t.start()
 503     if hasattr(sys, "frozen"):
 504         # py2exe sets this env vars that may screw our child process - reset
 505         del os.environ["PYTHONPATH"]
 506
 507     # Start the buildbot/buildslave app
 508     runfn()
 509     print "Service child process terminating normally."
 510
 511
 512 def _WaitForShutdown(h):
 513     win32event.WaitForSingleObject(h, win32event.INFINITE)
 514     print "Shutdown requested"
 515
 516     from twisted.internet import reactor
 517     reactor.callLater(0, reactor.stop)
 518
 519 def DetermineRunner(bbdir):
 520    '''Checks if the given directory is a buildslave or a master and returns the
 521    appropriate run function.'''
 522    try:
 523       import buildslave.scripts.runner
 524       tacfile = os.path.join(bbdir, 'buildbot.tac')
 525
 526       if os.path.exists(tacfile):
 527          with open(tacfile, 'r') as f:
 528             contents = f.read()
 529             if 'import BuildSlave' in contents:
 530                return buildslave.scripts.runner.run
 531
 532    except ImportError:
 533       # Use the default
 534       pass
 535
 536    import buildbot.scripts.runner
 537    return buildbot.scripts.runner.run
 538
 539 # This function is also called by the py2exe startup code.
 540
 541
 542 def HandleCommandLine():
 543     if len(sys.argv)>1 and sys.argv[1] == "--spawn":
 544         # Special command-line created by the service to execute the
 545         # child-process.
 546         # First arg is the handle to wait on
 547         # Fourth arg is the config directory to use for the buildbot/slave
 548         _RunChild(DetermineRunner(sys.argv[4]))
 549     else:
 550         win32serviceutil.HandleCommandLine(BBService,
 551                                            customOptionHandler=CustomInstall)
 552
 553
 554 if __name__ == '__main__':
 555     HandleCommandLine()