Lib/subprocess.py

   1 # subprocess - Subprocesses with accessible I/O streams
   2 #
   3 # For more information about this module, see PEP 324.
   4 #
   5 # This module should remain compatible with Python 2.2, see PEP 291.
   6 #
   7 # Copyright (c) 2003-2005 by Peter Astrand <astrand@lysator.liu.se>
   8 #
   9 # Licensed to PSF under a Contributor Agreement.
  10 # See http://www.python.org/2.4/license for licensing details.
  11
  12 r"""subprocess - Subprocesses with accessible I/O streams
  13
  14 This module allows you to spawn processes, connect to their
  15 input/output/error pipes, and obtain their return codes.  This module
  16 intends to replace several other, older modules and functions, like:
  17
  18 os.system
  19 os.spawn*
  20 os.popen*
  21 popen2.*
  22 commands.*
  23
  24 Information about how the subprocess module can be used to replace these
  25 modules and functions can be found below.
  26
  27
  28
  29 Using the subprocess module
  30 ===========================
  31 This module defines one class called Popen:
  32
  33 class Popen(args, bufsize=0, executable=None,
  34             stdin=None, stdout=None, stderr=None,
  35             preexec_fn=None, close_fds=False, shell=False,
  36             cwd=None, env=None, universal_newlines=False,
  37             startupinfo=None, creationflags=0):
  38
  39
  40 Arguments are:
  41
  42 args should be a string, or a sequence of program arguments.  The
  43 program to execute is normally the first item in the args sequence or
  44 string, but can be explicitly set by using the executable argument.
  45
  46 On UNIX, with shell=False (default): In this case, the Popen class
  47 uses os.execvp() to execute the child program.  args should normally
  48 be a sequence.  A string will be treated as a sequence with the string
  49 as the only item (the program to execute).
  50
  51 On UNIX, with shell=True: If args is a string, it specifies the
  52 command string to execute through the shell.  If args is a sequence,
  53 the first item specifies the command string, and any additional items
  54 will be treated as additional shell arguments.
  55
  56 On Windows: the Popen class uses CreateProcess() to execute the child
  57 program, which operates on strings.  If args is a sequence, it will be
  58 converted to a string using the list2cmdline method.  Please note that
  59 not all MS Windows applications interpret the command line the same
  60 way: The list2cmdline is designed for applications using the same
  61 rules as the MS C runtime.
  62
  63 bufsize, if given, has the same meaning as the corresponding argument
  64 to the built-in open() function: 0 means unbuffered, 1 means line
  65 buffered, any other positive value means use a buffer of
  66 (approximately) that size.  A negative bufsize means to use the system
  67 default, which usually means fully buffered.  The default value for
  68 bufsize is 0 (unbuffered).
  69
  70 stdin, stdout and stderr specify the executed programs' standard
  71 input, standard output and standard error file handles, respectively.
  72 Valid values are PIPE, an existing file descriptor (a positive
  73 integer), an existing file object, and None.  PIPE indicates that a
  74 new pipe to the child should be created.  With None, no redirection
  75 will occur; the child's file handles will be inherited from the
  76 parent.  Additionally, stderr can be STDOUT, which indicates that the
  77 stderr data from the applications should be captured into the same
  78 file handle as for stdout.
  79
  80 If preexec_fn is set to a callable object, this object will be called
  81 in the child process just before the child is executed.
  82
  83 If close_fds is true, all file descriptors except 0, 1 and 2 will be
  84 closed before the child process is executed.
  85
  86 if shell is true, the specified command will be executed through the
  87 shell.
  88
  89 If cwd is not None, the current directory will be changed to cwd
  90 before the child is executed.
  91
  92 If env is not None, it defines the environment variables for the new
  93 process.
  94
  95 If universal_newlines is true, the file objects stdout and stderr are
  96 opened as a text files, but lines may be terminated by any of '\n',
  97 the Unix end-of-line convention, '\r', the Macintosh convention or
  98 '\r\n', the Windows convention.  All of these external representations
  99 are seen as '\n' by the Python program.  Note: This feature is only
 100 available if Python is built with universal newline support (the
 101 default).  Also, the newlines attribute of the file objects stdout,
 102 stdin and stderr are not updated by the communicate() method.
 103
 104 The startupinfo and creationflags, if given, will be passed to the
 105 underlying CreateProcess() function.  They can specify things such as
 106 appearance of the main window and priority for the new process.
 107 (Windows only)
 108
 109
 110 This module also defines two shortcut functions:
 111
 112 call(*popenargs, **kwargs):
 113     Run command with arguments.  Wait for command to complete, then
 114     return the returncode attribute.
 115
 116     The arguments are the same as for the Popen constructor.  Example:
 117
 118     retcode = call(["ls", "-l"])
 119
 120 check_call(*popenargs, **kwargs):
 121     Run command with arguments.  Wait for command to complete.  If the
 122     exit code was zero then return, otherwise raise
 123     CalledProcessError.  The CalledProcessError object will have the
 124     return code in the errno attribute.
 125
 126     The arguments are the same as for the Popen constructor.  Example:
 127
 128     check_call(["ls", "-l"])
 129
 130 Exceptions
 131 ----------
 132 Exceptions raised in the child process, before the new program has
 133 started to execute, will be re-raised in the parent.  Additionally,
 134 the exception object will have one extra attribute called
 135 'child_traceback', which is a string containing traceback information
 136 from the childs point of view.
 137
 138 The most common exception raised is OSError.  This occurs, for
 139 example, when trying to execute a non-existent file.  Applications
 140 should prepare for OSErrors.
 141
 142 A ValueError will be raised if Popen is called with invalid arguments.
 143
 144 check_call() will raise CalledProcessError, which is a subclass of
 145 OSError, if the called process returns a non-zero return code.
 146
 147
 148 Security
 149 --------
 150 Unlike some other popen functions, this implementation will never call
 151 /bin/sh implicitly.  This means that all characters, including shell
 152 metacharacters, can safely be passed to child processes.
 153
 154
 155 Popen objects
 156 =============
 157 Instances of the Popen class have the following methods:
 158
 159 poll()
 160     Check if child process has terminated.  Returns returncode
 161     attribute.
 162
 163 wait()
 164     Wait for child process to terminate.  Returns returncode attribute.
 165
 166 communicate(input=None)
 167     Interact with process: Send data to stdin.  Read data from stdout
 168     and stderr, until end-of-file is reached.  Wait for process to
 169     terminate.  The optional stdin argument should be a string to be
 170     sent to the child process, or None, if no data should be sent to
 171     the child.
 172
 173     communicate() returns a tuple (stdout, stderr).
 174
 175     Note: The data read is buffered in memory, so do not use this
 176     method if the data size is large or unlimited.
 177
 178 The following attributes are also available:
 179
 180 stdin
 181     If the stdin argument is PIPE, this attribute is a file object
 182     that provides input to the child process.  Otherwise, it is None.
 183
 184 stdout
 185     If the stdout argument is PIPE, this attribute is a file object
 186     that provides output from the child process.  Otherwise, it is
 187     None.
 188
 189 stderr
 190     If the stderr argument is PIPE, this attribute is file object that
 191     provides error output from the child process.  Otherwise, it is
 192     None.
 193
 194 pid
 195     The process ID of the child process.
 196
 197 returncode
 198     The child return code.  A None value indicates that the process
 199     hasn't terminated yet.  A negative value -N indicates that the
 200     child was terminated by signal N (UNIX only).
 201
 202
 203 Replacing older functions with the subprocess module
 204 ====================================================
 205 In this section, "a ==> b" means that b can be used as a replacement
 206 for a.
 207
 208 Note: All functions in this section fail (more or less) silently if
 209 the executed program cannot be found; this module raises an OSError
 210 exception.
 211
 212 In the following examples, we assume that the subprocess module is
 213 imported with "from subprocess import *".
 214
 215
 216 Replacing /bin/sh shell backquote
 217 ---------------------------------
 218 output=`mycmd myarg`
 219 ==>
 220 output = Popen(["mycmd", "myarg"], stdout=PIPE).communicate()[0]
 221
 222
 223 Replacing shell pipe line
 224 -------------------------
 225 output=`dmesg | grep hda`
 226 ==>
 227 p1 = Popen(["dmesg"], stdout=PIPE)
 228 p2 = Popen(["grep", "hda"], stdin=p1.stdout, stdout=PIPE)
 229 output = p2.communicate()[0]
 230
 231
 232 Replacing os.system()
 233 ---------------------
 234 sts = os.system("mycmd" + " myarg")
 235 ==>
 236 p = Popen("mycmd" + " myarg", shell=True)
 237 sts = os.waitpid(p.pid, 0)
 238
 239 Note:
 240
 241 * Calling the program through the shell is usually not required.
 242
 243 * It's easier to look at the returncode attribute than the
 244   exitstatus.
 245
 246 A more real-world example would look like this:
 247
 248 try:
 249     retcode = call("mycmd" + " myarg", shell=True)
 250     if retcode < 0:
 251         print >>sys.stderr, "Child was terminated by signal", -retcode
 252     else:
 253         print >>sys.stderr, "Child returned", retcode
 254 except OSError, e:
 255     print >>sys.stderr, "Execution failed:", e
 256
 257
 258 Replacing os.spawn*
 259 -------------------
 260 P_NOWAIT example:
 261
 262 pid = os.spawnlp(os.P_NOWAIT, "/bin/mycmd", "mycmd", "myarg")
 263 ==>
 264 pid = Popen(["/bin/mycmd", "myarg"]).pid
 265
 266
 267 P_WAIT example:
 268
 269 retcode = os.spawnlp(os.P_WAIT, "/bin/mycmd", "mycmd", "myarg")
 270 ==>
 271 retcode = call(["/bin/mycmd", "myarg"])
 272
 273
 274 Vector example:
 275
 276 os.spawnvp(os.P_NOWAIT, path, args)
 277 ==>
 278 Popen([path] + args[1:])
 279
 280
 281 Environment example:
 282
 283 os.spawnlpe(os.P_NOWAIT, "/bin/mycmd", "mycmd", "myarg", env)
 284 ==>
 285 Popen(["/bin/mycmd", "myarg"], env={"PATH": "/usr/bin"})
 286
 287
 288 Replacing os.popen*
 289 -------------------
 290 pipe = os.popen(cmd, mode='r', bufsize)
 291 ==>
 292 pipe = Popen(cmd, shell=True, bufsize=bufsize, stdout=PIPE).stdout
 293
 294 pipe = os.popen(cmd, mode='w', bufsize)
 295 ==>
 296 pipe = Popen(cmd, shell=True, bufsize=bufsize, stdin=PIPE).stdin
 297
 298
 299 (child_stdin, child_stdout) = os.popen2(cmd, mode, bufsize)
 300 ==>
 301 p = Popen(cmd, shell=True, bufsize=bufsize,
 302           stdin=PIPE, stdout=PIPE, close_fds=True)
 303 (child_stdin, child_stdout) = (p.stdin, p.stdout)
 304
 305
 306 (child_stdin,
 307  child_stdout,
 308  child_stderr) = os.popen3(cmd, mode, bufsize)
 309 ==>
 310 p = Popen(cmd, shell=True, bufsize=bufsize,
 311           stdin=PIPE, stdout=PIPE, stderr=PIPE, close_fds=True)
 312 (child_stdin,
 313  child_stdout,
 314  child_stderr) = (p.stdin, p.stdout, p.stderr)
 315
 316
 317 (child_stdin, child_stdout_and_stderr) = os.popen4(cmd, mode, bufsize)
 318 ==>
 319 p = Popen(cmd, shell=True, bufsize=bufsize,
 320           stdin=PIPE, stdout=PIPE, stderr=STDOUT, close_fds=True)
 321 (child_stdin, child_stdout_and_stderr) = (p.stdin, p.stdout)
 322
 323
 324 Replacing popen2.*
 325 ------------------
 326 Note: If the cmd argument to popen2 functions is a string, the command
 327 is executed through /bin/sh.  If it is a list, the command is directly
 328 executed.
 329
 330 (child_stdout, child_stdin) = popen2.popen2("somestring", bufsize, mode)
 331 ==>
 332 p = Popen(["somestring"], shell=True, bufsize=bufsize
 333           stdin=PIPE, stdout=PIPE, close_fds=True)
 334 (child_stdout, child_stdin) = (p.stdout, p.stdin)
 335
 336
 337 (child_stdout, child_stdin) = popen2.popen2(["mycmd", "myarg"], bufsize, mode)
 338 ==>
 339 p = Popen(["mycmd", "myarg"], bufsize=bufsize,
 340           stdin=PIPE, stdout=PIPE, close_fds=True)
 341 (child_stdout, child_stdin) = (p.stdout, p.stdin)
 342
 343 The popen2.Popen3 and popen3.Popen4 basically works as subprocess.Popen,
 344 except that:
 345
 346 * subprocess.Popen raises an exception if the execution fails
 347 * the capturestderr argument is replaced with the stderr argument.
 348 * stdin=PIPE and stdout=PIPE must be specified.
 349 * popen2 closes all filedescriptors by default, but you have to specify
 350   close_fds=True with subprocess.Popen.
 351
 352
 353 """
 354
 355 import sys
 356 mswindows = (sys.platform == "win32")
 357
 358 import os
 359 import types
 360 import traceback
 361
 362 # Exception classes used by this module.
 363 class CalledProcessError(OSError):
 364     """This exception is raised when a process run by check_call() returns
 365     a non-zero exit status.  The exit status will be stored in the
 366     errno attribute.  This exception is a subclass of
 367     OSError."""
 368
 369 if mswindows:
 370     import threading
 371     import msvcrt
 372     if 0: # <-- change this to use pywin32 instead of the _subprocess driver
 373         import pywintypes
 374         from win32api import GetStdHandle, STD_INPUT_HANDLE, \
 375                              STD_OUTPUT_HANDLE, STD_ERROR_HANDLE
 376         from win32api import GetCurrentProcess, DuplicateHandle, \
 377                              GetModuleFileName, GetVersion
 378         from win32con import DUPLICATE_SAME_ACCESS, SW_HIDE
 379         from win32pipe import CreatePipe
 380         from win32process import CreateProcess, STARTUPINFO, \
 381                                  GetExitCodeProcess, STARTF_USESTDHANDLES, \
 382                                  STARTF_USESHOWWINDOW, CREATE_NEW_CONSOLE
 383         from win32event import WaitForSingleObject, INFINITE, WAIT_OBJECT_0
 384     else:
 385         from _subprocess import *
 386         class STARTUPINFO:
 387             dwFlags = 0
 388             hStdInput = None
 389             hStdOutput = None
 390             hStdError = None
 391             wShowWindow = 0
 392         class pywintypes:
 393             error = IOError
 394 else:
 395     import select
 396     import errno
 397     import fcntl
 398     import pickle
 399
 400 __all__ = ["Popen", "PIPE", "STDOUT", "call", "check_call", "CalledProcessError"]
 401
 402 try:
 403     MAXFD = os.sysconf("SC_OPEN_MAX")
 404 except:
 405     MAXFD = 256
 406
 407 # True/False does not exist on 2.2.0
 408 try:
 409     False
 410 except NameError:
 411     False = 0
 412     True = 1
 413
 414 _active = []
 415
 416 def _cleanup():
 417     for inst in _active[:]:
 418         if inst.poll(_deadstate=sys.maxint) >= 0:
 419             try:
 420                 _active.remove(inst)
 421             except ValueError:
 422                 # This can happen if two threads create a new Popen instance.
 423                 # It's harmless that it was already removed, so ignore.
 424                 pass
 425
 426 PIPE = -1
 427 STDOUT = -2
 428
 429
 430 def call(*popenargs, **kwargs):
 431     """Run command with arguments.  Wait for command to complete, then
 432     return the returncode attribute.
 433
 434     The arguments are the same as for the Popen constructor.  Example:
 435
 436     retcode = call(["ls", "-l"])
 437     """
 438     return Popen(*popenargs, **kwargs).wait()
 439
 440
 441 def check_call(*popenargs, **kwargs):
 442     """Run command with arguments.  Wait for command to complete.  If
 443     the exit code was zero then return, otherwise raise
 444     CalledProcessError.  The CalledProcessError object will have the
 445     return code in the errno attribute.
 446
 447     The arguments are the same as for the Popen constructor.  Example:
 448
 449     check_call(["ls", "-l"])
 450     """
 451     retcode = call(*popenargs, **kwargs)
 452     cmd = kwargs.get("args")
 453     if cmd is None:
 454         cmd = popenargs[0]
 455     if retcode:
 456         raise CalledProcessError(retcode, "Command %s returned non-zero exit status" % cmd)
 457     return retcode
 458
 459
 460 def list2cmdline(seq):
 461     """
 462     Translate a sequence of arguments into a command line
 463     string, using the same rules as the MS C runtime:
 464
 465     1) Arguments are delimited by white space, which is either a
 466        space or a tab.
 467
 468     2) A string surrounded by double quotation marks is
 469        interpreted as a single argument, regardless of white space
 470        contained within.  A quoted string can be embedded in an
 471        argument.
 472
 473     3) A double quotation mark preceded by a backslash is
 474        interpreted as a literal double quotation mark.
 475
 476     4) Backslashes are interpreted literally, unless they
 477        immediately precede a double quotation mark.
 478
 479     5) If backslashes immediately precede a double quotation mark,
 480        every pair of backslashes is interpreted as a literal
 481        backslash.  If the number of backslashes is odd, the last
 482        backslash escapes the next double quotation mark as
 483        described in rule 3.
 484     """
 485
 486     # See
 487     # http://msdn.microsoft.com/library/en-us/vccelng/htm/progs_12.asp
 488     result = []
 489     needquote = False
 490     for arg in seq:
 491         bs_buf = []
 492
 493         # Add a space to separate this argument from the others
 494         if result:
 495             result.append(' ')
 496
 497         needquote = (" " in arg) or ("\t" in arg)
 498         if needquote:
 499             result.append('"')
 500
 501         for c in arg:
 502             if c == '\\':
 503                 # Don't know if we need to double yet.
 504                 bs_buf.append(c)
 505             elif c == '"':
 506                 # Double backspaces.
 507                 result.append('\\' * len(bs_buf)*2)
 508                 bs_buf = []
 509                 result.append('\\"')
 510             else:
 511                 # Normal char
 512                 if bs_buf:
 513                     result.extend(bs_buf)
 514                     bs_buf = []
 515                 result.append(c)
 516
 517         # Add remaining backspaces, if any.
 518         if bs_buf:
 519             result.extend(bs_buf)
 520
 521         if needquote:
 522             result.extend(bs_buf)
 523             result.append('"')
 524
 525     return ''.join(result)
 526
 527
 528 class Popen(object):
 529     def __init__(self, args, bufsize=0, executable=None,
 530                  stdin=None, stdout=None, stderr=None,
 531                  preexec_fn=None, close_fds=False, shell=False,
 532                  cwd=None, env=None, universal_newlines=False,
 533                  startupinfo=None, creationflags=0):
 534         """Create new Popen instance."""
 535         _cleanup()
 536
 537         self._child_created = False
 538         if not isinstance(bufsize, (int, long)):
 539             raise TypeError("bufsize must be an integer")
 540
 541         if mswindows:
 542             if preexec_fn is not None:
 543                 raise ValueError("preexec_fn is not supported on Windows "
 544                                  "platforms")
 545             if close_fds:
 546                 raise ValueError("close_fds is not supported on Windows "
 547                                  "platforms")
 548         else:
 549             # POSIX
 550             if startupinfo is not None:
 551                 raise ValueError("startupinfo is only supported on Windows "
 552                                  "platforms")
 553             if creationflags != 0:
 554                 raise ValueError("creationflags is only supported on Windows "
 555                                  "platforms")
 556
 557         self.stdin = None
 558         self.stdout = None
 559         self.stderr = None
 560         self.pid = None
 561         self.returncode = None
 562         self.universal_newlines = universal_newlines
 563
 564         # Input and output objects. The general principle is like
 565         # this:
 566         #
 567         # Parent                   Child
 568         # ------                   -----
 569         # p2cwrite   ---stdin--->  p2cread
 570         # c2pread    <--stdout---  c2pwrite
 571         # errread    <--stderr---  errwrite
 572         #
 573         # On POSIX, the child objects are file descriptors.  On
 574         # Windows, these are Windows file handles.  The parent objects
 575         # are file descriptors on both platforms.  The parent objects
 576         # are None when not using PIPEs. The child objects are None
 577         # when not redirecting.
 578
 579         (p2cread, p2cwrite,
 580          c2pread, c2pwrite,
 581          errread, errwrite) = self._get_handles(stdin, stdout, stderr)
 582
 583         self._execute_child(args, executable, preexec_fn, close_fds,
 584                             cwd, env, universal_newlines,
 585                             startupinfo, creationflags, shell,
 586                             p2cread, p2cwrite,
 587                             c2pread, c2pwrite,
 588                             errread, errwrite)
 589
 590         if p2cwrite:
 591             self.stdin = os.fdopen(p2cwrite, 'wb', bufsize)
 592         if c2pread:
 593             if universal_newlines:
 594                 self.stdout = os.fdopen(c2pread, 'rU', bufsize)
 595             else:
 596                 self.stdout = os.fdopen(c2pread, 'rb', bufsize)
 597         if errread:
 598             if universal_newlines:
 599                 self.stderr = os.fdopen(errread, 'rU', bufsize)
 600             else:
 601                 self.stderr = os.fdopen(errread, 'rb', bufsize)
 602
 603
 604     def _translate_newlines(self, data):
 605         data = data.replace("\r\n", "\n")
 606         data = data.replace("\r", "\n")
 607         return data
 608
 609
 610     def __del__(self):
 611         if not self._child_created:
 612             # We didn't get to successfully create a child process.
 613             return
 614         # In case the child hasn't been waited on, check if it's done.
 615         self.poll(_deadstate=sys.maxint)
 616         if self.returncode is None:
 617             # Child is still running, keep us alive until we can wait on it.
 618             _active.append(self)
 619
 620
 621     def communicate(self, input=None):
 622         """Interact with process: Send data to stdin.  Read data from
 623         stdout and stderr, until end-of-file is reached.  Wait for
 624         process to terminate.  The optional input argument should be a
 625         string to be sent to the child process, or None, if no data
 626         should be sent to the child.
 627
 628         communicate() returns a tuple (stdout, stderr)."""
 629
 630         # Optimization: If we are only using one pipe, or no pipe at
 631         # all, using select() or threads is unnecessary.
 632         if [self.stdin, self.stdout, self.stderr].count(None) >= 2:
 633             stdout = None
 634             stderr = None
 635             if self.stdin:
 636                 if input:
 637                     self.stdin.write(input)
 638                 self.stdin.close()
 639             elif self.stdout:
 640                 stdout = self.stdout.read()
 641             elif self.stderr:
 642                 stderr = self.stderr.read()
 643             self.wait()
 644             return (stdout, stderr)
 645
 646         return self._communicate(input)
 647
 648
 649     if mswindows:
 650         #
 651         # Windows methods
 652         #
 653         def _get_handles(self, stdin, stdout, stderr):
 654             """Construct and return tupel with IO objects:
 655             p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite
 656             """
 657             if stdin is None and stdout is None and stderr is None:
 658                 return (None, None, None, None, None, None)
 659
 660             p2cread, p2cwrite = None, None
 661             c2pread, c2pwrite = None, None
 662             errread, errwrite = None, None
 663
 664             if stdin is None:
 665                 p2cread = GetStdHandle(STD_INPUT_HANDLE)
 666             elif stdin == PIPE:
 667                 p2cread, p2cwrite = CreatePipe(None, 0)
 668                 # Detach and turn into fd
 669                 p2cwrite = p2cwrite.Detach()
 670                 p2cwrite = msvcrt.open_osfhandle(p2cwrite, 0)
 671             elif isinstance(stdin, int):
 672                 p2cread = msvcrt.get_osfhandle(stdin)
 673             else:
 674                 # Assuming file-like object
 675                 p2cread = msvcrt.get_osfhandle(stdin.fileno())
 676             p2cread = self._make_inheritable(p2cread)
 677
 678             if stdout is None:
 679                 c2pwrite = GetStdHandle(STD_OUTPUT_HANDLE)
 680             elif stdout == PIPE:
 681                 c2pread, c2pwrite = CreatePipe(None, 0)
 682                 # Detach and turn into fd
 683                 c2pread = c2pread.Detach()
 684                 c2pread = msvcrt.open_osfhandle(c2pread, 0)
 685             elif isinstance(stdout, int):
 686                 c2pwrite = msvcrt.get_osfhandle(stdout)
 687             else:
 688                 # Assuming file-like object
 689                 c2pwrite = msvcrt.get_osfhandle(stdout.fileno())
 690             c2pwrite = self._make_inheritable(c2pwrite)
 691
 692             if stderr is None:
 693                 errwrite = GetStdHandle(STD_ERROR_HANDLE)
 694             elif stderr == PIPE:
 695                 errread, errwrite = CreatePipe(None, 0)
 696                 # Detach and turn into fd
 697                 errread = errread.Detach()
 698                 errread = msvcrt.open_osfhandle(errread, 0)
 699             elif stderr == STDOUT:
 700                 errwrite = c2pwrite
 701             elif isinstance(stderr, int):
 702                 errwrite = msvcrt.get_osfhandle(stderr)
 703             else:
 704                 # Assuming file-like object
 705                 errwrite = msvcrt.get_osfhandle(stderr.fileno())
 706             errwrite = self._make_inheritable(errwrite)
 707
 708             return (p2cread, p2cwrite,
 709                     c2pread, c2pwrite,
 710                     errread, errwrite)
 711
 712
 713         def _make_inheritable(self, handle):
 714             """Return a duplicate of handle, which is inheritable"""
 715             return DuplicateHandle(GetCurrentProcess(), handle,
 716                                    GetCurrentProcess(), 0, 1,
 717                                    DUPLICATE_SAME_ACCESS)
 718
 719
 720         def _find_w9xpopen(self):
 721             """Find and return absolut path to w9xpopen.exe"""
 722             w9xpopen = os.path.join(os.path.dirname(GetModuleFileName(0)),
 723                                     "w9xpopen.exe")
 724             if not os.path.exists(w9xpopen):
 725                 # Eeek - file-not-found - possibly an embedding
 726                 # situation - see if we can locate it in sys.exec_prefix
 727                 w9xpopen = os.path.join(os.path.dirname(sys.exec_prefix),
 728                                         "w9xpopen.exe")
 729                 if not os.path.exists(w9xpopen):
 730                     raise RuntimeError("Cannot locate w9xpopen.exe, which is "
 731                                        "needed for Popen to work with your "
 732                                        "shell or platform.")
 733             return w9xpopen
 734
 735
 736         def _execute_child(self, args, executable, preexec_fn, close_fds,
 737                            cwd, env, universal_newlines,
 738                            startupinfo, creationflags, shell,
 739                            p2cread, p2cwrite,
 740                            c2pread, c2pwrite,
 741                            errread, errwrite):
 742             """Execute program (MS Windows version)"""
 743
 744             if not isinstance(args, types.StringTypes):
 745                 args = list2cmdline(args)
 746
 747             # Process startup details
 748             if startupinfo is None:
 749                 startupinfo = STARTUPINFO()
 750             if None not in (p2cread, c2pwrite, errwrite):
 751                 startupinfo.dwFlags |= STARTF_USESTDHANDLES
 752                 startupinfo.hStdInput = p2cread
 753                 startupinfo.hStdOutput = c2pwrite
 754                 startupinfo.hStdError = errwrite
 755
 756             if shell:
 757                 startupinfo.dwFlags |= STARTF_USESHOWWINDOW
 758                 startupinfo.wShowWindow = SW_HIDE
 759                 comspec = os.environ.get("COMSPEC", "cmd.exe")
 760                 args = comspec + " /c " + args
 761                 if (GetVersion() >= 0x80000000L or
 762                         os.path.basename(comspec).lower() == "command.com"):
 763                     # Win9x, or using command.com on NT. We need to
 764                     # use the w9xpopen intermediate program. For more
 765                     # information, see KB Q150956
 766                     # (http://web.archive.org/web/20011105084002/http://support.microsoft.com/support/kb/articles/Q150/9/56.asp)
 767                     w9xpopen = self._find_w9xpopen()
 768                     args = '"%s" %s' % (w9xpopen, args)
 769                     # Not passing CREATE_NEW_CONSOLE has been known to
 770                     # cause random failures on win9x.  Specifically a
 771                     # dialog: "Your program accessed mem currently in
 772                     # use at xxx" and a hopeful warning about the
 773                     # stability of your system.  Cost is Ctrl+C wont
 774                     # kill children.
 775                     creationflags |= CREATE_NEW_CONSOLE
 776
 777             # Start the process
 778             try:
 779                 hp, ht, pid, tid = CreateProcess(executable, args,
 780                                          # no special security
 781                                          None, None,
 782                                          # must inherit handles to pass std
 783                                          # handles
 784                                          1,
 785                                          creationflags,
 786                                          env,
 787                                          cwd,
 788                                          startupinfo)
 789             except pywintypes.error, e:
 790                 # Translate pywintypes.error to WindowsError, which is
 791                 # a subclass of OSError.  FIXME: We should really
 792                 # translate errno using _sys_errlist (or simliar), but
 793                 # how can this be done from Python?
 794                 raise WindowsError(*e.args)
 795
 796             # Retain the process handle, but close the thread handle
 797             self._child_created = True
 798             self._handle = hp
 799             self.pid = pid
 800             ht.Close()
 801
 802             # Child is launched. Close the parent's copy of those pipe
 803             # handles that only the child should have open.  You need
 804             # to make sure that no handles to the write end of the
 805             # output pipe are maintained in this process or else the
 806             # pipe will not close when the child process exits and the
 807             # ReadFile will hang.
 808             if p2cread is not None:
 809                 p2cread.Close()
 810             if c2pwrite is not None:
 811                 c2pwrite.Close()
 812             if errwrite is not None:
 813                 errwrite.Close()
 814
 815
 816         def poll(self, _deadstate=None):
 817             """Check if child process has terminated.  Returns returncode
 818             attribute."""
 819             if self.returncode is None:
 820                 if WaitForSingleObject(self._handle, 0) == WAIT_OBJECT_0:
 821                     self.returncode = GetExitCodeProcess(self._handle)
 822             return self.returncode
 823
 824
 825         def wait(self):
 826             """Wait for child process to terminate.  Returns returncode
 827             attribute."""
 828             if self.returncode is None:
 829                 obj = WaitForSingleObject(self._handle, INFINITE)
 830                 self.returncode = GetExitCodeProcess(self._handle)
 831             return self.returncode
 832
 833
 834         def _readerthread(self, fh, buffer):
 835             buffer.append(fh.read())
 836
 837
 838         def _communicate(self, input):
 839             stdout = None # Return
 840             stderr = None # Return
 841
 842             if self.stdout:
 843                 stdout = []
 844                 stdout_thread = threading.Thread(target=self._readerthread,
 845                                                  args=(self.stdout, stdout))
 846                 stdout_thread.setDaemon(True)
 847                 stdout_thread.start()
 848             if self.stderr:
 849                 stderr = []
 850                 stderr_thread = threading.Thread(target=self._readerthread,
 851                                                  args=(self.stderr, stderr))
 852                 stderr_thread.setDaemon(True)
 853                 stderr_thread.start()
 854
 855             if self.stdin:
 856                 if input is not None:
 857                     self.stdin.write(input)
 858                 self.stdin.close()
 859
 860             if self.stdout:
 861                 stdout_thread.join()
 862             if self.stderr:
 863                 stderr_thread.join()
 864
 865             # All data exchanged.  Translate lists into strings.
 866             if stdout is not None:
 867                 stdout = stdout[0]
 868             if stderr is not None:
 869                 stderr = stderr[0]
 870
 871             # Translate newlines, if requested.  We cannot let the file
 872             # object do the translation: It is based on stdio, which is
 873             # impossible to combine with select (unless forcing no
 874             # buffering).
 875             if self.universal_newlines and hasattr(file, 'newlines'):
 876                 if stdout:
 877                     stdout = self._translate_newlines(stdout)
 878                 if stderr:
 879                     stderr = self._translate_newlines(stderr)
 880
 881             self.wait()
 882             return (stdout, stderr)
 883
 884     else:
 885         #
 886         # POSIX methods
 887         #
 888         def _get_handles(self, stdin, stdout, stderr):
 889             """Construct and return tupel with IO objects:
 890             p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite
 891             """
 892             p2cread, p2cwrite = None, None
 893             c2pread, c2pwrite = None, None
 894             errread, errwrite = None, None
 895
 896             if stdin is None:
 897                 pass
 898             elif stdin == PIPE:
 899                 p2cread, p2cwrite = os.pipe()
 900             elif isinstance(stdin, int):
 901                 p2cread = stdin
 902             else:
 903                 # Assuming file-like object
 904                 p2cread = stdin.fileno()
 905
 906             if stdout is None:
 907                 pass
 908             elif stdout == PIPE:
 909                 c2pread, c2pwrite = os.pipe()
 910             elif isinstance(stdout, int):
 911                 c2pwrite = stdout
 912             else:
 913                 # Assuming file-like object
 914                 c2pwrite = stdout.fileno()
 915
 916             if stderr is None:
 917                 pass
 918             elif stderr == PIPE:
 919                 errread, errwrite = os.pipe()
 920             elif stderr == STDOUT:
 921                 errwrite = c2pwrite
 922             elif isinstance(stderr, int):
 923                 errwrite = stderr
 924             else:
 925                 # Assuming file-like object
 926                 errwrite = stderr.fileno()
 927
 928             return (p2cread, p2cwrite,
 929                     c2pread, c2pwrite,
 930                     errread, errwrite)
 931
 932
 933         def _set_cloexec_flag(self, fd):
 934             try:
 935                 cloexec_flag = fcntl.FD_CLOEXEC
 936             except AttributeError:
 937                 cloexec_flag = 1
 938
 939             old = fcntl.fcntl(fd, fcntl.F_GETFD)
 940             fcntl.fcntl(fd, fcntl.F_SETFD, old | cloexec_flag)
 941
 942
 943         def _close_fds(self, but):
 944             for i in range(3, MAXFD):
 945                 if i == but:
 946                     continue
 947                 try:
 948                     os.close(i)
 949                 except:
 950                     pass
 951
 952
 953         def _execute_child(self, args, executable, preexec_fn, close_fds,
 954                            cwd, env, universal_newlines,
 955                            startupinfo, creationflags, shell,
 956                            p2cread, p2cwrite,
 957                            c2pread, c2pwrite,
 958                            errread, errwrite):
 959             """Execute program (POSIX version)"""
 960
 961             if isinstance(args, types.StringTypes):
 962                 args = [args]
 963
 964             if shell:
 965                 args = ["/bin/sh", "-c"] + args
 966
 967             if executable is None:
 968                 executable = args[0]
 969
 970             # For transferring possible exec failure from child to parent
 971             # The first char specifies the exception type: 0 means
 972             # OSError, 1 means some other error.
 973             errpipe_read, errpipe_write = os.pipe()
 974             self._set_cloexec_flag(errpipe_write)
 975
 976             self.pid = os.fork()
 977             self._child_created = True
 978             if self.pid == 0:
 979                 # Child
 980                 try:
 981                     # Close parent's pipe ends
 982                     if p2cwrite:
 983                         os.close(p2cwrite)
 984                     if c2pread:
 985                         os.close(c2pread)
 986                     if errread:
 987                         os.close(errread)
 988                     os.close(errpipe_read)
 989
 990                     # Dup fds for child
 991                     if p2cread:
 992                         os.dup2(p2cread, 0)
 993                     if c2pwrite:
 994                         os.dup2(c2pwrite, 1)
 995                     if errwrite:
 996                         os.dup2(errwrite, 2)
 997
 998                     # Close pipe fds.  Make sure we doesn't close the same
 999                     # fd more than once.
1000                     if p2cread:
1001                         os.close(p2cread)
1002                     if c2pwrite and c2pwrite not in (p2cread,):
1003                         os.close(c2pwrite)
1004                     if errwrite and errwrite not in (p2cread, c2pwrite):
1005                         os.close(errwrite)
1006
1007                     # Close all other fds, if asked for
1008                     if close_fds:
1009                         self._close_fds(but=errpipe_write)
1010
1011                     if cwd is not None:
1012                         os.chdir(cwd)
1013
1014                     if preexec_fn:
1015                         apply(preexec_fn)
1016
1017                     if env is None:
1018                         os.execvp(executable, args)
1019                     else:
1020                         os.execvpe(executable, args, env)
1021
1022                 except:
1023                     exc_type, exc_value, tb = sys.exc_info()
1024                     # Save the traceback and attach it to the exception object
1025                     exc_lines = traceback.format_exception(exc_type,
1026                                                            exc_value,
1027                                                            tb)
1028                     exc_value.child_traceback = ''.join(exc_lines)
1029                     os.write(errpipe_write, pickle.dumps(exc_value))
1030
1031                 # This exitcode won't be reported to applications, so it
1032                 # really doesn't matter what we return.
1033                 os._exit(255)
1034
1035             # Parent
1036             os.close(errpipe_write)
1037             if p2cread and p2cwrite:
1038                 os.close(p2cread)
1039             if c2pwrite and c2pread:
1040                 os.close(c2pwrite)
1041             if errwrite and errread:
1042                 os.close(errwrite)
1043
1044             # Wait for exec to fail or succeed; possibly raising exception
1045             data = os.read(errpipe_read, 1048576) # Exceptions limited to 1 MB
1046             os.close(errpipe_read)
1047             if data != "":
1048                 os.waitpid(self.pid, 0)
1049                 child_exception = pickle.loads(data)
1050                 raise child_exception
1051
1052
1053         def _handle_exitstatus(self, sts):
1054             if os.WIFSIGNALED(sts):
1055                 self.returncode = -os.WTERMSIG(sts)
1056             elif os.WIFEXITED(sts):
1057                 self.returncode = os.WEXITSTATUS(sts)
1058             else:
1059                 # Should never happen
1060                 raise RuntimeError("Unknown child exit status!")
1061
1062
1063         def poll(self, _deadstate=None):
1064             """Check if child process has terminated.  Returns returncode
1065             attribute."""
1066             if self.returncode is None:
1067                 try:
1068                     pid, sts = os.waitpid(self.pid, os.WNOHANG)
1069                     if pid == self.pid:
1070                         self._handle_exitstatus(sts)
1071                 except os.error:
1072                     if _deadstate is not None:
1073                         self.returncode = _deadstate
1074             return self.returncode
1075
1076
1077         def wait(self):
1078             """Wait for child process to terminate.  Returns returncode
1079             attribute."""
1080             if self.returncode is None:
1081                 pid, sts = os.waitpid(self.pid, 0)
1082                 self._handle_exitstatus(sts)
1083             return self.returncode
1084
1085
1086         def _communicate(self, input):
1087             read_set = []
1088             write_set = []
1089             stdout = None # Return
1090             stderr = None # Return
1091
1092             if self.stdin:
1093                 # Flush stdio buffer.  This might block, if the user has
1094                 # been writing to .stdin in an uncontrolled fashion.
1095                 self.stdin.flush()
1096                 if input:
1097                     write_set.append(self.stdin)
1098                 else:
1099                     self.stdin.close()
1100             if self.stdout:
1101                 read_set.append(self.stdout)
1102                 stdout = []
1103             if self.stderr:
1104                 read_set.append(self.stderr)
1105                 stderr = []
1106
1107             while read_set or write_set:
1108                 rlist, wlist, xlist = select.select(read_set, write_set, [])
1109
1110                 if self.stdin in wlist:
1111                     # When select has indicated that the file is writable,
1112                     # we can write up to PIPE_BUF bytes without risk
1113                     # blocking.  POSIX defines PIPE_BUF >= 512
1114                     bytes_written = os.write(self.stdin.fileno(), input[:512])
1115                     input = input[bytes_written:]
1116                     if not input:
1117                         self.stdin.close()
1118                         write_set.remove(self.stdin)
1119
1120                 if self.stdout in rlist:
1121                     data = os.read(self.stdout.fileno(), 1024)
1122                     if data == "":
1123                         self.stdout.close()
1124                         read_set.remove(self.stdout)
1125                     stdout.append(data)
1126
1127                 if self.stderr in rlist:
1128                     data = os.read(self.stderr.fileno(), 1024)
1129                     if data == "":
1130                         self.stderr.close()
1131                         read_set.remove(self.stderr)
1132                     stderr.append(data)
1133
1134             # All data exchanged.  Translate lists into strings.
1135             if stdout is not None:
1136                 stdout = ''.join(stdout)
1137             if stderr is not None:
1138                 stderr = ''.join(stderr)
1139
1140             # Translate newlines, if requested.  We cannot let the file
1141             # object do the translation: It is based on stdio, which is
1142             # impossible to combine with select (unless forcing no
1143             # buffering).
1144             if self.universal_newlines and hasattr(file, 'newlines'):
1145                 if stdout:
1146                     stdout = self._translate_newlines(stdout)
1147                 if stderr:
1148                     stderr = self._translate_newlines(stderr)
1149
1150             self.wait()
1151             return (stdout, stderr)
1152
1153
1154 def _demo_posix():
1155     #
1156     # Example 1: Simple redirection: Get process list
1157     #
1158     plist = Popen(["ps"], stdout=PIPE).communicate()[0]
1159     print "Process list:"
1160     print plist
1161
1162     #
1163     # Example 2: Change uid before executing child
1164     #
1165     if os.getuid() == 0:
1166         p = Popen(["id"], preexec_fn=lambda: os.setuid(100))
1167         p.wait()
1168
1169     #
1170     # Example 3: Connecting several subprocesses
1171     #
1172     print "Looking for 'hda'..."
1173     p1 = Popen(["dmesg"], stdout=PIPE)
1174     p2 = Popen(["grep", "hda"], stdin=p1.stdout, stdout=PIPE)
1175     print repr(p2.communicate()[0])
1176
1177     #
1178     # Example 4: Catch execution error
1179     #
1180     print
1181     print "Trying a weird file..."
1182     try:
1183         print Popen(["/this/path/does/not/exist"]).communicate()
1184     except OSError, e:
1185         if e.errno == errno.ENOENT:
1186             print "The file didn't exist.  I thought so..."
1187             print "Child traceback:"
1188             print e.child_traceback
1189         else:
1190             print "Error", e.errno
1191     else:
1192         print >>sys.stderr, "Gosh.  No error."
1193
1194
1195 def _demo_windows():
1196     #
1197     # Example 1: Connecting several subprocesses
1198     #
1199     print "Looking for 'PROMPT' in set output..."
1200     p1 = Popen("set", stdout=PIPE, shell=True)
1201     p2 = Popen('find "PROMPT"', stdin=p1.stdout, stdout=PIPE)
1202     print repr(p2.communicate()[0])
1203
1204     #
1205     # Example 2: Simple execution of program
1206     #
1207     print "Executing calc..."
1208     p = Popen("calc")
1209     p.wait()
1210
1211
1212 if __name__ == "__main__":
1213     if mswindows:
1214         _demo_windows()
1215     else:
1216         _demo_posix()