mysubprocess.py

   1 # See comment in the beginning of citool.py on why this file is here.
   2
   3
   4
   5 # subprocess - Subprocesses with accessible I/O streams
   6 #
   7 # For more information about this module, see PEP 324.
   8 #
   9 # Copyright (c) 2003-2004 by Peter Astrand <astrand@lysator.liu.se>
  10 #
  11 # By obtaining, using, and/or copying this software and/or its
  12 # associated documentation, you agree that you have read, understood,
  13 # and will comply with the following terms and conditions:
  14 #
  15 # Permission to use, copy, modify, and distribute this software and
  16 # its associated documentation for any purpose and without fee is
  17 # hereby granted, provided that the above copyright notice appears in
  18 # all copies, and that both that copyright notice and this permission
  19 # notice appear in supporting documentation, and that the name of the
  20 # author not be used in advertising or publicity pertaining to
  21 # distribution of the software without specific, written prior
  22 # permission.
  23 #
  24 # THE AUTHOR DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE,
  25 # INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS.
  26 # IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, INDIRECT OR
  27 # CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS
  28 # OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT,
  29 # NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION
  30 # WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
  31
  32 r"""subprocess - Subprocesses with accessible I/O streams
  33
  34 This module allows you to spawn processes, connect to their
  35 input/output/error pipes, and obtain their return codes.  This module
  36 intends to replace several other, older modules and functions, like:
  37
  38 os.system
  39 os.spawn*
  40 os.popen*
  41 popen2.*
  42 commands.*
  43
  44 Information about how the subprocess module can be used to replace these
  45 modules and functions can be found below.
  46
  47
  48
  49 Using the subprocess module
  50 ===========================
  51 This module defines one class called Popen:
  52
  53 class Popen(args, bufsize=0, executable=None,
  54             stdin=None, stdout=None, stderr=None,
  55             preexec_fn=None, close_fds=False, shell=False,
  56             cwd=None, env=None, universal_newlines=False,
  57             startupinfo=None, creationflags=0):
  58
  59
  60 Arguments are:
  61
  62 args should be a string, or a sequence of program arguments.  The
  63 program to execute is normally the first item in the args sequence or
  64 string, but can be explicitly set by using the executable argument.
  65
  66 On UNIX, with shell=False (default): In this case, the Popen class
  67 uses os.execvp() to execute the child program.  args should normally
  68 be a sequence.  A string will be treated as a sequence with the string
  69 as the only item (the program to execute).
  70
  71 On UNIX, with shell=True: If args is a string, it specifies the
  72 command string to execute through the shell.  If args is a sequence,
  73 the first item specifies the command string, and any additional items
  74 will be treated as additional shell arguments.
  75
  76 On Windows: the Popen class uses CreateProcess() to execute the child
  77 program, which operates on strings.  If args is a sequence, it will be
  78 converted to a string using the list2cmdline method.  Please note that
  79 not all MS Windows applications interpret the command line the same
  80 way: The list2cmdline is designed for applications using the same
  81 rules as the MS C runtime.
  82
  83 bufsize, if given, has the same meaning as the corresponding argument
  84 to the built-in open() function: 0 means unbuffered, 1 means line
  85 buffered, any other positive value means use a buffer of
  86 (approximately) that size.  A negative bufsize means to use the system
  87 default, which usually means fully buffered.  The default value for
  88 bufsize is 0 (unbuffered).
  89
  90 stdin, stdout and stderr specify the executed programs' standard
  91 input, standard output and standard error file handles, respectively.
  92 Valid values are PIPE, an existing file descriptor (a positive
  93 integer), an existing file object, and None.  PIPE indicates that a
  94 new pipe to the child should be created.  With None, no redirection
  95 will occur; the child's file handles will be inherited from the
  96 parent.  Additionally, stderr can be STDOUT, which indicates that the
  97 stderr data from the applications should be captured into the same
  98 file handle as for stdout.
  99
 100 If preexec_fn is set to a callable object, this object will be called
 101 in the child process just before the child is executed.
 102
 103 If close_fds is true, all file descriptors except 0, 1 and 2 will be
 104 closed before the child process is executed.
 105
 106 if shell is true, the specified command will be executed through the
 107 shell.
 108
 109 If cwd is not None, the current directory will be changed to cwd
 110 before the child is executed.
 111
 112 If env is not None, it defines the environment variables for the new
 113 process.
 114
 115 If universal_newlines is true, the file objects stdout and stderr are
 116 opened as a text files, but lines may be terminated by any of '\n',
 117 the Unix end-of-line convention, '\r', the Macintosh convention or
 118 '\r\n', the Windows convention.  All of these external representations
 119 are seen as '\n' by the Python program.  Note: This feature is only
 120 available if Python is built with universal newline support (the
 121 default).  Also, the newlines attribute of the file objects stdout,
 122 stdin and stderr are not updated by the communicate() method.
 123
 124 The startupinfo and creationflags, if given, will be passed to the
 125 underlying CreateProcess() function.  They can specify things such as
 126 appearance of the main window and priority for the new process.
 127 (Windows only)
 128
 129
 130 This module also defines two shortcut functions:
 131
 132 call(*args, **kwargs):
 133     Run command with arguments.  Wait for command to complete, then
 134     return the returncode attribute.
 135
 136     The arguments are the same as for the Popen constructor.  Example:
 137
 138     retcode = call(["ls", "-l"])
 139
 140
 141 Exceptions
 142 ----------
 143 Exceptions raised in the child process, before the new program has
 144 started to execute, will be re-raised in the parent.  Additionally,
 145 the exception object will have one extra attribute called
 146 'child_traceback', which is a string containing traceback information
 147 from the childs point of view.
 148
 149 The most common exception raised is OSError.  This occurs, for
 150 example, when trying to execute a non-existent file.  Applications
 151 should prepare for OSErrors.
 152
 153 A ValueError will be raised if Popen is called with invalid arguments.
 154
 155
 156 Security
 157 --------
 158 Unlike some other popen functions, this implementation will never call
 159 /bin/sh implicitly.  This means that all characters, including shell
 160 metacharacters, can safely be passed to child processes.
 161
 162
 163 Popen objects
 164 =============
 165 Instances of the Popen class have the following methods:
 166
 167 poll()
 168     Check if child process has terminated.  Returns returncode
 169     attribute.
 170
 171 wait()
 172     Wait for child process to terminate.  Returns returncode attribute.
 173
 174 communicate(input=None)
 175     Interact with process: Send data to stdin.  Read data from stdout
 176     and stderr, until end-of-file is reached.  Wait for process to
 177     terminate.  The optional stdin argument should be a string to be
 178     sent to the child process, or None, if no data should be sent to
 179     the child.
 180
 181     communicate() returns a tuple (stdout, stderr).
 182
 183     Note: The data read is buffered in memory, so do not use this
 184     method if the data size is large or unlimited.
 185
 186 The following attributes are also available:
 187
 188 stdin
 189     If the stdin argument is PIPE, this attribute is a file object
 190     that provides input to the child process.  Otherwise, it is None.
 191
 192 stdout
 193     If the stdout argument is PIPE, this attribute is a file object
 194     that provides output from the child process.  Otherwise, it is
 195     None.
 196
 197 stderr
 198     If the stderr argument is PIPE, this attribute is file object that
 199     provides error output from the child process.  Otherwise, it is
 200     None.
 201
 202 pid
 203     The process ID of the child process.
 204
 205 returncode
 206     The child return code.  A None value indicates that the process
 207     hasn't terminated yet.  A negative value -N indicates that the
 208     child was terminated by signal N (UNIX only).
 209
 210
 211 Replacing older functions with the subprocess module
 212 ====================================================
 213 In this section, "a ==> b" means that b can be used as a replacement
 214 for a.
 215
 216 Note: All functions in this section fail (more or less) silently if
 217 the executed program cannot be found; this module raises an OSError
 218 exception.
 219
 220 In the following examples, we assume that the subprocess module is
 221 imported with "from subprocess import *".
 222
 223
 224 Replacing /bin/sh shell backquote
 225 ---------------------------------
 226 output=`mycmd myarg`
 227 ==>
 228 output = Popen(["mycmd", "myarg"], stdout=PIPE).communicate()[0]
 229
 230
 231 Replacing shell pipe line
 232 -------------------------
 233 output=`dmesg | grep hda`
 234 ==>
 235 p1 = Popen(["dmesg"], stdout=PIPE)
 236 p2 = Popen(["grep", "hda"], stdin=p1.stdout, stdout=PIPE)
 237 output = p2.communicate()[0]
 238
 239
 240 Replacing os.system()
 241 ---------------------
 242 sts = os.system("mycmd" + " myarg")
 243 ==>
 244 p = Popen("mycmd" + " myarg", shell=True)
 245 sts = os.waitpid(p.pid, 0)
 246
 247 Note:
 248
 249 * Calling the program through the shell is usually not required.
 250
 251 * It's easier to look at the returncode attribute than the
 252   exitstatus.
 253
 254 A more real-world example would look like this:
 255
 256 try:
 257     retcode = call("mycmd" + " myarg", shell=True)
 258     if retcode < 0:
 259         print >>sys.stderr, "Child was terminated by signal", -retcode
 260     else:
 261         print >>sys.stderr, "Child returned", retcode
 262 except OSError, e:
 263     print >>sys.stderr, "Execution failed:", e
 264
 265
 266 Replacing os.spawn*
 267 -------------------
 268 P_NOWAIT example:
 269
 270 pid = os.spawnlp(os.P_NOWAIT, "/bin/mycmd", "mycmd", "myarg")
 271 ==>
 272 pid = Popen(["/bin/mycmd", "myarg"]).pid
 273
 274
 275 P_WAIT example:
 276
 277 retcode = os.spawnlp(os.P_WAIT, "/bin/mycmd", "mycmd", "myarg")
 278 ==>
 279 retcode = call(["/bin/mycmd", "myarg"])
 280
 281
 282 Vector example:
 283
 284 os.spawnvp(os.P_NOWAIT, path, args)
 285 ==>
 286 Popen([path] + args[1:])
 287
 288
 289 Environment example:
 290
 291 os.spawnlpe(os.P_NOWAIT, "/bin/mycmd", "mycmd", "myarg", env)
 292 ==>
 293 Popen(["/bin/mycmd", "myarg"], env={"PATH": "/usr/bin"})
 294
 295
 296 Replacing os.popen*
 297 -------------------
 298 pipe = os.popen(cmd, mode='r', bufsize)
 299 ==>
 300 pipe = Popen(cmd, shell=True, bufsize=bufsize, stdout=PIPE).stdout
 301
 302 pipe = os.popen(cmd, mode='w', bufsize)
 303 ==>
 304 pipe = Popen(cmd, shell=True, bufsize=bufsize, stdin=PIPE).stdin
 305
 306
 307 (child_stdin, child_stdout) = os.popen2(cmd, mode, bufsize)
 308 ==>
 309 p = Popen(cmd, shell=True, bufsize=bufsize,
 310           stdin=PIPE, stdout=PIPE, close_fds=True)
 311 (child_stdin, child_stdout) = (p.stdin, p.stdout)
 312
 313
 314 (child_stdin,
 315  child_stdout,
 316  child_stderr) = os.popen3(cmd, mode, bufsize)
 317 ==>
 318 p = Popen(cmd, shell=True, bufsize=bufsize,
 319           stdin=PIPE, stdout=PIPE, stderr=PIPE, close_fds=True)
 320 (child_stdin,
 321  child_stdout,
 322  child_stderr) = (p.stdin, p.stdout, p.stderr)
 323
 324
 325 (child_stdin, child_stdout_and_stderr) = os.popen4(cmd, mode, bufsize)
 326 ==>
 327 p = Popen(cmd, shell=True, bufsize=bufsize,
 328           stdin=PIPE, stdout=PIPE, stderr=STDOUT, close_fds=True)
 329 (child_stdin, child_stdout_and_stderr) = (p.stdin, p.stdout)
 330
 331
 332 Replacing popen2.*
 333 ------------------
 334 Note: If the cmd argument to popen2 functions is a string, the command
 335 is executed through /bin/sh.  If it is a list, the command is directly
 336 executed.
 337
 338 (child_stdout, child_stdin) = popen2.popen2("somestring", bufsize, mode)
 339 ==>
 340 p = Popen(["somestring"], shell=True, bufsize=bufsize
 341           stdin=PIPE, stdout=PIPE, close_fds=True)
 342 (child_stdout, child_stdin) = (p.stdout, p.stdin)
 343
 344
 345 (child_stdout, child_stdin) = popen2.popen2(["mycmd", "myarg"], bufsize, mode)
 346 ==>
 347 p = Popen(["mycmd", "myarg"], bufsize=bufsize,
 348           stdin=PIPE, stdout=PIPE, close_fds=True)
 349 (child_stdout, child_stdin) = (p.stdout, p.stdin)
 350
 351 The popen2.Popen3 and popen3.Popen4 basically works as subprocess.Popen,
 352 except that:
 353
 354 * subprocess.Popen raises an exception if the execution fails
 355 * the capturestderr argument is replaced with the stderr argument.
 356 * stdin=PIPE and stdout=PIPE must be specified.
 357 * popen2 closes all filedescriptors by default, but you have to specify
 358   close_fds=True with subprocess.Popen.
 359
 360
 361 """
 362
 363 import sys
 364 mswindows = (sys.platform == "win32")
 365
 366 import os
 367 import types
 368 import traceback
 369
 370 if mswindows:
 371     import threading
 372     import msvcrt
 373     if 0: # <-- change this to use pywin32 instead of the _subprocess driver
 374         import pywintypes
 375         from win32api import GetStdHandle, STD_INPUT_HANDLE, \
 376                              STD_OUTPUT_HANDLE, STD_ERROR_HANDLE
 377         from win32api import GetCurrentProcess, DuplicateHandle, \
 378                              GetModuleFileName, GetVersion
 379         from win32con import DUPLICATE_SAME_ACCESS, SW_HIDE
 380         from win32pipe import CreatePipe
 381         from win32process import CreateProcess, STARTUPINFO, \
 382                                  GetExitCodeProcess, STARTF_USESTDHANDLES, \
 383                                  STARTF_USESHOWWINDOW, CREATE_NEW_CONSOLE
 384         from win32event import WaitForSingleObject, INFINITE, WAIT_OBJECT_0
 385     else:
 386         from _subprocess import *
 387         class STARTUPINFO:
 388             dwFlags = 0
 389             hStdInput = None
 390             hStdOutput = None
 391             hStdError = None
 392         class pywintypes:
 393             error = IOError
 394 else:
 395     import select
 396     import errno
 397     import fcntl
 398     import pickle
 399
 400 __all__ = ["Popen", "PIPE", "STDOUT", "call"]
 401
 402 try:
 403     MAXFD = os.sysconf("SC_OPEN_MAX")
 404 except:
 405     MAXFD = 256
 406
 407 # True/False does not exist on 2.2.0
 408 try:
 409     False
 410 except NameError:
 411     False = 0
 412     True = 1
 413
 414 _active = []
 415
 416 def _cleanup():
 417     for inst in _active[:]:
 418         inst.poll()
 419
 420 PIPE = -1
 421 STDOUT = -2
 422
 423
 424 def call(*args, **kwargs):
 425     """Run command with arguments.  Wait for command to complete, then
 426     return the returncode attribute.
 427
 428     The arguments are the same as for the Popen constructor.  Example:
 429
 430     retcode = call(["ls", "-l"])
 431     """
 432     return Popen(*args, **kwargs).wait()
 433
 434
 435 def list2cmdline(seq):
 436     """
 437     Translate a sequence of arguments into a command line
 438     string, using the same rules as the MS C runtime:
 439
 440     1) Arguments are delimited by white space, which is either a
 441        space or a tab.
 442
 443     2) A string surrounded by double quotation marks is
 444        interpreted as a single argument, regardless of white space
 445        contained within.  A quoted string can be embedded in an
 446        argument.
 447
 448     3) A double quotation mark preceded by a backslash is
 449        interpreted as a literal double quotation mark.
 450
 451     4) Backslashes are interpreted literally, unless they
 452        immediately precede a double quotation mark.
 453
 454     5) If backslashes immediately precede a double quotation mark,
 455        every pair of backslashes is interpreted as a literal
 456        backslash.  If the number of backslashes is odd, the last
 457        backslash escapes the next double quotation mark as
 458        described in rule 3.
 459     """
 460
 461     # See
 462     # http://msdn.microsoft.com/library/en-us/vccelng/htm/progs_12.asp
 463     result = []
 464     needquote = False
 465     for arg in seq:
 466         bs_buf = []
 467
 468         # Add a space to separate this argument from the others
 469         if result:
 470             result.append(' ')
 471
 472         needquote = (" " in arg) or ("\t" in arg)
 473         if needquote:
 474             result.append('"')
 475
 476         for c in arg:
 477             if c == '\\':
 478                 # Don't know if we need to double yet.
 479                 bs_buf.append(c)
 480             elif c == '"':
 481                 # Double backspaces.
 482                 result.append('\\' * len(bs_buf)*2)
 483                 bs_buf = []
 484                 result.append('\\"')
 485             else:
 486                 # Normal char
 487                 if bs_buf:
 488                     result.extend(bs_buf)
 489                     bs_buf = []
 490                 result.append(c)
 491
 492         # Add remaining backspaces, if any.
 493         if bs_buf:
 494             result.extend(bs_buf)
 495
 496         if needquote:
 497             result.extend(bs_buf)
 498             result.append('"')
 499
 500     return ''.join(result)
 501
 502
 503 class Popen(object):
 504     def __init__(self, args, bufsize=0, executable=None,
 505                  stdin=None, stdout=None, stderr=None,
 506                  preexec_fn=None, close_fds=False, shell=False,
 507                  cwd=None, env=None, universal_newlines=False,
 508                  startupinfo=None, creationflags=0):
 509         """Create new Popen instance."""
 510         _cleanup()
 511
 512         if not isinstance(bufsize, (int, long)):
 513             raise TypeError("bufsize must be an integer")
 514
 515         if mswindows:
 516             if preexec_fn is not None:
 517                 raise ValueError("preexec_fn is not supported on Windows "
 518                                  "platforms")
 519             if close_fds:
 520                 raise ValueError("close_fds is not supported on Windows "
 521                                  "platforms")
 522         else:
 523             # POSIX
 524             if startupinfo is not None:
 525                 raise ValueError("startupinfo is only supported on Windows "
 526                                  "platforms")
 527             if creationflags != 0:
 528                 raise ValueError("creationflags is only supported on Windows "
 529                                  "platforms")
 530
 531         self.stdin = None
 532         self.stdout = None
 533         self.stderr = None
 534         self.pid = None
 535         self.returncode = None
 536         self.universal_newlines = universal_newlines
 537
 538         # Input and output objects. The general principle is like
 539         # this:
 540         #
 541         # Parent                   Child
 542         # ------                   -----
 543         # p2cwrite   ---stdin--->  p2cread
 544         # c2pread    <--stdout---  c2pwrite
 545         # errread    <--stderr---  errwrite
 546         #
 547         # On POSIX, the child objects are file descriptors.  On
 548         # Windows, these are Windows file handles.  The parent objects
 549         # are file descriptors on both platforms.  The parent objects
 550         # are None when not using PIPEs. The child objects are None
 551         # when not redirecting.
 552
 553         (p2cread, p2cwrite,
 554          c2pread, c2pwrite,
 555          errread, errwrite) = self._get_handles(stdin, stdout, stderr)
 556
 557         self._execute_child(args, executable, preexec_fn, close_fds,
 558                             cwd, env, universal_newlines,
 559                             startupinfo, creationflags, shell,
 560                             p2cread, p2cwrite,
 561                             c2pread, c2pwrite,
 562                             errread, errwrite)
 563
 564         if p2cwrite:
 565             self.stdin = os.fdopen(p2cwrite, 'wb', bufsize)
 566         if c2pread:
 567             if universal_newlines:
 568                 self.stdout = os.fdopen(c2pread, 'rU', bufsize)
 569             else:
 570                 self.stdout = os.fdopen(c2pread, 'rb', bufsize)
 571         if errread:
 572             if universal_newlines:
 573                 self.stderr = os.fdopen(errread, 'rU', bufsize)
 574             else:
 575                 self.stderr = os.fdopen(errread, 'rb', bufsize)
 576
 577         _active.append(self)
 578
 579
 580     def _translate_newlines(self, data):
 581         data = data.replace("\r\n", "\n")
 582         data = data.replace("\r", "\n")
 583         return data
 584
 585
 586     if mswindows:
 587         #
 588         # Windows methods
 589         #
 590         def _get_handles(self, stdin, stdout, stderr):
 591             """Construct and return tupel with IO objects:
 592             p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite
 593             """
 594             if stdin == None and stdout == None and stderr == None:
 595                 return (None, None, None, None, None, None)
 596
 597             p2cread, p2cwrite = None, None
 598             c2pread, c2pwrite = None, None
 599             errread, errwrite = None, None
 600
 601             if stdin == None:
 602                 p2cread = GetStdHandle(STD_INPUT_HANDLE)
 603             elif stdin == PIPE:
 604                 p2cread, p2cwrite = CreatePipe(None, 0)
 605                 # Detach and turn into fd
 606                 p2cwrite = p2cwrite.Detach()
 607                 p2cwrite = msvcrt.open_osfhandle(p2cwrite, 0)
 608             elif type(stdin) == types.IntType:
 609                 p2cread = msvcrt.get_osfhandle(stdin)
 610             else:
 611                 # Assuming file-like object
 612                 p2cread = msvcrt.get_osfhandle(stdin.fileno())
 613             p2cread = self._make_inheritable(p2cread)
 614
 615             if stdout == None:
 616                 c2pwrite = GetStdHandle(STD_OUTPUT_HANDLE)
 617             elif stdout == PIPE:
 618                 c2pread, c2pwrite = CreatePipe(None, 0)
 619                 # Detach and turn into fd
 620                 c2pread = c2pread.Detach()
 621                 c2pread = msvcrt.open_osfhandle(c2pread, 0)
 622             elif type(stdout) == types.IntType:
 623                 c2pwrite = msvcrt.get_osfhandle(stdout)
 624             else:
 625                 # Assuming file-like object
 626                 c2pwrite = msvcrt.get_osfhandle(stdout.fileno())
 627             c2pwrite = self._make_inheritable(c2pwrite)
 628
 629             if stderr == None:
 630                 errwrite = GetStdHandle(STD_ERROR_HANDLE)
 631             elif stderr == PIPE:
 632                 errread, errwrite = CreatePipe(None, 0)
 633                 # Detach and turn into fd
 634                 errread = errread.Detach()
 635                 errread = msvcrt.open_osfhandle(errread, 0)
 636             elif stderr == STDOUT:
 637                 errwrite = c2pwrite
 638             elif type(stderr) == types.IntType:
 639                 errwrite = msvcrt.get_osfhandle(stderr)
 640             else:
 641                 # Assuming file-like object
 642                 errwrite = msvcrt.get_osfhandle(stderr.fileno())
 643             errwrite = self._make_inheritable(errwrite)
 644
 645             return (p2cread, p2cwrite,
 646                     c2pread, c2pwrite,
 647                     errread, errwrite)
 648
 649
 650         def _make_inheritable(self, handle):
 651             """Return a duplicate of handle, which is inheritable"""
 652             return DuplicateHandle(GetCurrentProcess(), handle,
 653                                    GetCurrentProcess(), 0, 1,
 654                                    DUPLICATE_SAME_ACCESS)
 655
 656
 657         def _find_w9xpopen(self):
 658             """Find and return absolut path to w9xpopen.exe"""
 659             w9xpopen = os.path.join(os.path.dirname(GetModuleFileName(0)),
 660                                     "w9xpopen.exe")
 661             if not os.path.exists(w9xpopen):
 662                 # Eeek - file-not-found - possibly an embedding
 663                 # situation - see if we can locate it in sys.exec_prefix
 664                 w9xpopen = os.path.join(os.path.dirname(sys.exec_prefix),
 665                                         "w9xpopen.exe")
 666                 if not os.path.exists(w9xpopen):
 667                     raise RuntimeError("Cannot locate w9xpopen.exe, which is "
 668                                        "needed for Popen to work with your "
 669                                        "shell or platform.")
 670             return w9xpopen
 671
 672
 673         def _execute_child(self, args, executable, preexec_fn, close_fds,
 674                            cwd, env, universal_newlines,
 675                            startupinfo, creationflags, shell,
 676                            p2cread, p2cwrite,
 677                            c2pread, c2pwrite,
 678                            errread, errwrite):
 679             """Execute program (MS Windows version)"""
 680
 681             if not isinstance(args, types.StringTypes):
 682                 args = list2cmdline(args)
 683
 684             # Process startup details
 685             default_startupinfo = STARTUPINFO()
 686             if startupinfo == None:
 687                 startupinfo = default_startupinfo
 688             if not None in (p2cread, c2pwrite, errwrite):
 689                 startupinfo.dwFlags |= STARTF_USESTDHANDLES
 690                 startupinfo.hStdInput = p2cread
 691                 startupinfo.hStdOutput = c2pwrite
 692                 startupinfo.hStdError = errwrite
 693
 694             if shell:
 695                 default_startupinfo.dwFlags |= STARTF_USESHOWWINDOW
 696                 default_startupinfo.wShowWindow = SW_HIDE
 697                 comspec = os.environ.get("COMSPEC", "cmd.exe")
 698                 args = comspec + " /c " + args
 699                 if (GetVersion() >= 0x80000000L or
 700                         os.path.basename(comspec).lower() == "command.com"):
 701                     # Win9x, or using command.com on NT. We need to
 702                     # use the w9xpopen intermediate program. For more
 703                     # information, see KB Q150956
 704                     # (http://web.archive.org/web/20011105084002/http://support.microsoft.com/support/kb/articles/Q150/9/56.asp)
 705                     w9xpopen = self._find_w9xpopen()
 706                     args = '"%s" %s' % (w9xpopen, args)
 707                     # Not passing CREATE_NEW_CONSOLE has been known to
 708                     # cause random failures on win9x.  Specifically a
 709                     # dialog: "Your program accessed mem currently in
 710                     # use at xxx" and a hopeful warning about the
 711                     # stability of your system.  Cost is Ctrl+C wont
 712                     # kill children.
 713                     creationflags |= CREATE_NEW_CONSOLE
 714
 715             # Start the process
 716             try:
 717                 hp, ht, pid, tid = CreateProcess(executable, args,
 718                                          # no special security
 719                                          None, None,
 720                                          # must inherit handles to pass std
 721                                          # handles
 722                                          1,
 723                                          creationflags,
 724                                          env,
 725                                          cwd,
 726                                          startupinfo)
 727             except pywintypes.error, e:
 728                 # Translate pywintypes.error to WindowsError, which is
 729                 # a subclass of OSError.  FIXME: We should really
 730                 # translate errno using _sys_errlist (or simliar), but
 731                 # how can this be done from Python?
 732                 raise WindowsError(*e.args)
 733
 734             # Retain the process handle, but close the thread handle
 735             self._handle = hp
 736             self.pid = pid
 737             ht.Close()
 738
 739             # Child is launched. Close the parent's copy of those pipe
 740             # handles that only the child should have open.  You need
 741             # to make sure that no handles to the write end of the
 742             # output pipe are maintained in this process or else the
 743             # pipe will not close when the child process exits and the
 744             # ReadFile will hang.
 745             if p2cread != None:
 746                 p2cread.Close()
 747             if c2pwrite != None:
 748                 c2pwrite.Close()
 749             if errwrite != None:
 750                 errwrite.Close()
 751
 752
 753         def poll(self):
 754             """Check if child process has terminated.  Returns returncode
 755             attribute."""
 756             if self.returncode == None:
 757                 if WaitForSingleObject(self._handle, 0) == WAIT_OBJECT_0:
 758                     self.returncode = GetExitCodeProcess(self._handle)
 759                     _active.remove(self)
 760             return self.returncode
 761
 762
 763         def wait(self):
 764             """Wait for child process to terminate.  Returns returncode
 765             attribute."""
 766             if self.returncode == None:
 767                 obj = WaitForSingleObject(self._handle, INFINITE)
 768                 self.returncode = GetExitCodeProcess(self._handle)
 769                 _active.remove(self)
 770             return self.returncode
 771
 772
 773         def _readerthread(self, fh, buffer):
 774             buffer.append(fh.read())
 775
 776
 777         def communicate(self, input=None):
 778             """Interact with process: Send data to stdin.  Read data from
 779             stdout and stderr, until end-of-file is reached.  Wait for
 780             process to terminate.  The optional input argument should be a
 781             string to be sent to the child process, or None, if no data
 782             should be sent to the child.
 783
 784             communicate() returns a tuple (stdout, stderr)."""
 785             stdout = None # Return
 786             stderr = None # Return
 787
 788             if self.stdout:
 789                 stdout = []
 790                 stdout_thread = threading.Thread(target=self._readerthread,
 791                                                  args=(self.stdout, stdout))
 792                 stdout_thread.setDaemon(True)
 793                 stdout_thread.start()
 794             if self.stderr:
 795                 stderr = []
 796                 stderr_thread = threading.Thread(target=self._readerthread,
 797                                                  args=(self.stderr, stderr))
 798                 stderr_thread.setDaemon(True)
 799                 stderr_thread.start()
 800
 801             if self.stdin:
 802                 if input != None:
 803                     self.stdin.write(input)
 804                 self.stdin.close()
 805
 806             if self.stdout:
 807                 stdout_thread.join()
 808             if self.stderr:
 809                 stderr_thread.join()
 810
 811             # All data exchanged.  Translate lists into strings.
 812             if stdout != None:
 813                 stdout = stdout[0]
 814             if stderr != None:
 815                 stderr = stderr[0]
 816
 817             # Translate newlines, if requested.  We cannot let the file
 818             # object do the translation: It is based on stdio, which is
 819             # impossible to combine with select (unless forcing no
 820             # buffering).
 821             if self.universal_newlines and hasattr(open, 'newlines'):
 822                 if stdout:
 823                     stdout = self._translate_newlines(stdout)
 824                 if stderr:
 825                     stderr = self._translate_newlines(stderr)
 826
 827             self.wait()
 828             return (stdout, stderr)
 829
 830     else:
 831         #
 832         # POSIX methods
 833         #
 834         def _get_handles(self, stdin, stdout, stderr):
 835             """Construct and return tupel with IO objects:
 836             p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite
 837             """
 838             p2cread, p2cwrite = None, None
 839             c2pread, c2pwrite = None, None
 840             errread, errwrite = None, None
 841
 842             if stdin == None:
 843                 pass
 844             elif stdin == PIPE:
 845                 p2cread, p2cwrite = os.pipe()
 846             elif type(stdin) == types.IntType:
 847                 p2cread = stdin
 848             else:
 849                 # Assuming file-like object
 850                 p2cread = stdin.fileno()
 851
 852             if stdout == None:
 853                 pass
 854             elif stdout == PIPE:
 855                 c2pread, c2pwrite = os.pipe()
 856             elif type(stdout) == types.IntType:
 857                 c2pwrite = stdout
 858             else:
 859                 # Assuming file-like object
 860                 c2pwrite = stdout.fileno()
 861
 862             if stderr == None:
 863                 pass
 864             elif stderr == PIPE:
 865                 errread, errwrite = os.pipe()
 866             elif stderr == STDOUT:
 867                 errwrite = c2pwrite
 868             elif type(stderr) == types.IntType:
 869                 errwrite = stderr
 870             else:
 871                 # Assuming file-like object
 872                 errwrite = stderr.fileno()
 873
 874             return (p2cread, p2cwrite,
 875                     c2pread, c2pwrite,
 876                     errread, errwrite)
 877
 878
 879         def _set_cloexec_flag(self, fd):
 880             try:
 881                 cloexec_flag = fcntl.FD_CLOEXEC
 882             except AttributeError:
 883                 cloexec_flag = 1
 884
 885             old = fcntl.fcntl(fd, fcntl.F_GETFD)
 886             fcntl.fcntl(fd, fcntl.F_SETFD, old | cloexec_flag)
 887
 888
 889         def _close_fds(self, but):
 890             for i in range(3, MAXFD):
 891                 if i == but:
 892                     continue
 893                 try:
 894                     os.close(i)
 895                 except:
 896                     pass
 897
 898
 899         def _execute_child(self, args, executable, preexec_fn, close_fds,
 900                            cwd, env, universal_newlines,
 901                            startupinfo, creationflags, shell,
 902                            p2cread, p2cwrite,
 903                            c2pread, c2pwrite,
 904                            errread, errwrite):
 905             """Execute program (POSIX version)"""
 906
 907             if isinstance(args, types.StringTypes):
 908                 args = [args]
 909
 910             if shell:
 911                 args = ["/bin/sh", "-c"] + args
 912
 913             if executable == None:
 914                 executable = args[0]
 915
 916             # For transferring possible exec failure from child to parent
 917             # The first char specifies the exception type: 0 means
 918             # OSError, 1 means some other error.
 919             errpipe_read, errpipe_write = os.pipe()
 920             self._set_cloexec_flag(errpipe_write)
 921
 922             self.pid = os.fork()
 923             if self.pid == 0:
 924                 # Child
 925                 try:
 926                     # Close parent's pipe ends
 927                     if p2cwrite:
 928                         os.close(p2cwrite)
 929                     if c2pread:
 930                         os.close(c2pread)
 931                     if errread:
 932                         os.close(errread)
 933                     os.close(errpipe_read)
 934
 935                     # Dup fds for child
 936                     if p2cread:
 937                         os.dup2(p2cread, 0)
 938                     if c2pwrite:
 939                         os.dup2(c2pwrite, 1)
 940                     if errwrite:
 941                         os.dup2(errwrite, 2)
 942
 943                     # Close pipe fds.  Make sure we doesn't close the same
 944                     # fd more than once.
 945                     if p2cread:
 946                         os.close(p2cread)
 947                     if c2pwrite and c2pwrite not in (p2cread,):
 948                         os.close(c2pwrite)
 949                     if errwrite and errwrite not in (p2cread, c2pwrite):
 950                         os.close(errwrite)
 951
 952                     # Close all other fds, if asked for
 953                     if close_fds:
 954                         self._close_fds(but=errpipe_write)
 955
 956                     if cwd != None:
 957                         os.chdir(cwd)
 958
 959                     if preexec_fn:
 960                         apply(preexec_fn)
 961
 962                     if env == None:
 963                         os.execvp(executable, args)
 964                     else:
 965                         os.execvpe(executable, args, env)
 966
 967                 except:
 968                     exc_type, exc_value, tb = sys.exc_info()
 969                     # Save the traceback and attach it to the exception object
 970                     exc_lines = traceback.format_exception(exc_type,
 971                                                            exc_value,
 972                                                            tb)
 973                     exc_value.child_traceback = ''.join(exc_lines)
 974                     os.write(errpipe_write, pickle.dumps(exc_value))
 975
 976                 # This exitcode won't be reported to applications, so it
 977                 # really doesn't matter what we return.
 978                 os._exit(255)
 979
 980             # Parent
 981             os.close(errpipe_write)
 982             if p2cread and p2cwrite:
 983                 os.close(p2cread)
 984             if c2pwrite and c2pread:
 985                 os.close(c2pwrite)
 986             if errwrite and errread:
 987                 os.close(errwrite)
 988
 989             # Wait for exec to fail or succeed; possibly raising exception
 990             data = os.read(errpipe_read, 1048576) # Exceptions limited to 1 MB
 991             os.close(errpipe_read)
 992             if data != "":
 993                 os.waitpid(self.pid, 0)
 994                 child_exception = pickle.loads(data)
 995                 raise child_exception
 996
 997
 998         def _handle_exitstatus(self, sts):
 999             if os.WIFSIGNALED(sts):
1000                 self.returncode = -os.WTERMSIG(sts)
1001             elif os.WIFEXITED(sts):
1002                 self.returncode = os.WEXITSTATUS(sts)
1003             else:
1004                 # Should never happen
1005                 raise RuntimeError("Unknown child exit status!")
1006
1007             _active.remove(self)
1008
1009
1010         def poll(self):
1011             """Check if child process has terminated.  Returns returncode
1012             attribute."""
1013             if self.returncode == None:
1014                 try:
1015                     pid, sts = os.waitpid(self.pid, os.WNOHANG)
1016                     if pid == self.pid:
1017                         self._handle_exitstatus(sts)
1018                 except os.error:
1019                     pass
1020             return self.returncode
1021
1022
1023         def wait(self):
1024             """Wait for child process to terminate.  Returns returncode
1025             attribute."""
1026             if self.returncode == None:
1027                 pid, sts = os.waitpid(self.pid, 0)
1028                 self._handle_exitstatus(sts)
1029             return self.returncode
1030
1031
1032         def communicate(self, input=None):
1033             """Interact with process: Send data to stdin.  Read data from
1034             stdout and stderr, until end-of-file is reached.  Wait for
1035             process to terminate.  The optional input argument should be a
1036             string to be sent to the child process, or None, if no data
1037             should be sent to the child.
1038
1039             communicate() returns a tuple (stdout, stderr)."""
1040             read_set = []
1041             write_set = []
1042             stdout = None # Return
1043             stderr = None # Return
1044
1045             if self.stdin:
1046                 # Flush stdio buffer.  This might block, if the user has
1047                 # been writing to .stdin in an uncontrolled fashion.
1048                 self.stdin.flush()
1049                 if input:
1050                     write_set.append(self.stdin)
1051                 else:
1052                     self.stdin.close()
1053             if self.stdout:
1054                 read_set.append(self.stdout)
1055                 stdout = []
1056             if self.stderr:
1057                 read_set.append(self.stderr)
1058                 stderr = []
1059
1060             while read_set or write_set:
1061                 rlist, wlist, xlist = select.select(read_set, write_set, [])
1062
1063                 if self.stdin in wlist:
1064                     # When select has indicated that the file is writable,
1065                     # we can write up to PIPE_BUF bytes without risk
1066                     # blocking.  POSIX defines PIPE_BUF >= 512
1067                     bytes_written = os.write(self.stdin.fileno(), input[:512])
1068                     input = input[bytes_written:]
1069                     if not input:
1070                         self.stdin.close()
1071                         write_set.remove(self.stdin)
1072
1073                 if self.stdout in rlist:
1074                     data = os.read(self.stdout.fileno(), 1024)
1075                     if data == "":
1076                         self.stdout.close()
1077                         read_set.remove(self.stdout)
1078                     stdout.append(data)
1079
1080                 if self.stderr in rlist:
1081                     data = os.read(self.stderr.fileno(), 1024)
1082                     if data == "":
1083                         self.stderr.close()
1084                         read_set.remove(self.stderr)
1085                     stderr.append(data)
1086
1087             # All data exchanged.  Translate lists into strings.
1088             if stdout != None:
1089                 stdout = ''.join(stdout)
1090             if stderr != None:
1091                 stderr = ''.join(stderr)
1092
1093             # Translate newlines, if requested.  We cannot let the file
1094             # object do the translation: It is based on stdio, which is
1095             # impossible to combine with select (unless forcing no
1096             # buffering).
1097             if self.universal_newlines and hasattr(open, 'newlines'):
1098                 if stdout:
1099                     stdout = self._translate_newlines(stdout)
1100                 if stderr:
1101                     stderr = self._translate_newlines(stderr)
1102
1103             self.wait()
1104             return (stdout, stderr)
1105
1106
1107 def _demo_posix():
1108     #
1109     # Example 1: Simple redirection: Get process list
1110     #
1111     plist = Popen(["ps"], stdout=PIPE).communicate()[0]
1112     print "Process list:"
1113     print plist
1114
1115     #
1116     # Example 2: Change uid before executing child
1117     #
1118     if os.getuid() == 0:
1119         p = Popen(["id"], preexec_fn=lambda: os.setuid(100))
1120         p.wait()
1121
1122     #
1123     # Example 3: Connecting several subprocesses
1124     #
1125     print "Looking for 'hda'..."
1126     p1 = Popen(["dmesg"], stdout=PIPE)
1127     p2 = Popen(["grep", "hda"], stdin=p1.stdout, stdout=PIPE)
1128     print repr(p2.communicate()[0])
1129
1130     #
1131     # Example 4: Catch execution error
1132     #
1133     print
1134     print "Trying a weird file..."
1135     try:
1136         print Popen(["/this/path/does/not/exist"]).communicate()
1137     except OSError, e:
1138         if e.errno == errno.ENOENT:
1139             print "The file didn't exist.  I thought so..."
1140             print "Child traceback:"
1141             print e.child_traceback
1142         else:
1143             print "Error", e.errno
1144     else:
1145         print >>sys.stderr, "Gosh.  No error."
1146
1147
1148 def _demo_windows():
1149     #
1150     # Example 1: Connecting several subprocesses
1151     #
1152     print "Looking for 'PROMPT' in set output..."
1153     p1 = Popen("set", stdout=PIPE, shell=True)
1154     p2 = Popen('find "PROMPT"', stdin=p1.stdout, stdout=PIPE)
1155     print repr(p2.communicate()[0])
1156
1157     #
1158     # Example 2: Simple execution of program
1159     #
1160     print "Executing calc..."
1161     p = Popen("calc")
1162     p.wait()
1163
1164
1165 if __name__ == "__main__":
1166     if mswindows:
1167         _demo_windows()
1168     else:
1169         _demo_posix()