Lib/subprocess.py

   1 # subprocess - Subprocesses with accessible I/O streams
   2 #
   3 # For more information about this module, see PEP 324.
   4 #
   5 # This module should remain compatible with Python 2.2, see PEP 291.
   6 #
   7 # Copyright (c) 2003-2005 by Peter Astrand <astrand@lysator.liu.se>
   8 #
   9 # Licensed to PSF under a Contributor Agreement.
  10 # See http://www.python.org/2.4/license for licensing details.
  11
  12 r"""subprocess - Subprocesses with accessible I/O streams
  13
  14 This module allows you to spawn processes, connect to their
  15 input/output/error pipes, and obtain their return codes.  This module
  16 intends to replace several other, older modules and functions, like:
  17
  18 os.system
  19 os.spawn*
  20 os.popen*
  21 popen2.*
  22 commands.*
  23
  24 Information about how the subprocess module can be used to replace these
  25 modules and functions can be found below.
  26
  27
  28
  29 Using the subprocess module
  30 ===========================
  31 This module defines one class called Popen:
  32
  33 class Popen(args, bufsize=0, executable=None,
  34             stdin=None, stdout=None, stderr=None,
  35             preexec_fn=None, close_fds=False, shell=False,
  36             cwd=None, env=None, universal_newlines=False,
  37             startupinfo=None, creationflags=0):
  38
  39
  40 Arguments are:
  41
  42 args should be a string, or a sequence of program arguments.  The
  43 program to execute is normally the first item in the args sequence or
  44 string, but can be explicitly set by using the executable argument.
  45
  46 On UNIX, with shell=False (default): In this case, the Popen class
  47 uses os.execvp() to execute the child program.  args should normally
  48 be a sequence.  A string will be treated as a sequence with the string
  49 as the only item (the program to execute).
  50
  51 On UNIX, with shell=True: If args is a string, it specifies the
  52 command string to execute through the shell.  If args is a sequence,
  53 the first item specifies the command string, and any additional items
  54 will be treated as additional shell arguments.
  55
  56 On Windows: the Popen class uses CreateProcess() to execute the child
  57 program, which operates on strings.  If args is a sequence, it will be
  58 converted to a string using the list2cmdline method.  Please note that
  59 not all MS Windows applications interpret the command line the same
  60 way: The list2cmdline is designed for applications using the same
  61 rules as the MS C runtime.
  62
  63 bufsize, if given, has the same meaning as the corresponding argument
  64 to the built-in open() function: 0 means unbuffered, 1 means line
  65 buffered, any other positive value means use a buffer of
  66 (approximately) that size.  A negative bufsize means to use the system
  67 default, which usually means fully buffered.  The default value for
  68 bufsize is 0 (unbuffered).
  69
  70 stdin, stdout and stderr specify the executed programs' standard
  71 input, standard output and standard error file handles, respectively.
  72 Valid values are PIPE, an existing file descriptor (a positive
  73 integer), an existing file object, and None.  PIPE indicates that a
  74 new pipe to the child should be created.  With None, no redirection
  75 will occur; the child's file handles will be inherited from the
  76 parent.  Additionally, stderr can be STDOUT, which indicates that the
  77 stderr data from the applications should be captured into the same
  78 file handle as for stdout.
  79
  80 If preexec_fn is set to a callable object, this object will be called
  81 in the child process just before the child is executed.
  82
  83 If close_fds is true, all file descriptors except 0, 1 and 2 will be
  84 closed before the child process is executed.
  85
  86 if shell is true, the specified command will be executed through the
  87 shell.
  88
  89 If cwd is not None, the current directory will be changed to cwd
  90 before the child is executed.
  91
  92 If env is not None, it defines the environment variables for the new
  93 process.
  94
  95 If universal_newlines is true, the file objects stdout and stderr are
  96 opened as a text files, but lines may be terminated by any of '\n',
  97 the Unix end-of-line convention, '\r', the Macintosh convention or
  98 '\r\n', the Windows convention.  All of these external representations
  99 are seen as '\n' by the Python program.  Note: This feature is only
 100 available if Python is built with universal newline support (the
 101 default).  Also, the newlines attribute of the file objects stdout,
 102 stdin and stderr are not updated by the communicate() method.
 103
 104 The startupinfo and creationflags, if given, will be passed to the
 105 underlying CreateProcess() function.  They can specify things such as
 106 appearance of the main window and priority for the new process.
 107 (Windows only)
 108
 109
 110 This module also defines two shortcut functions:
 111
 112 call(*popenargs, **kwargs):
 113     Run command with arguments.  Wait for command to complete, then
 114     return the returncode attribute.
 115
 116     The arguments are the same as for the Popen constructor.  Example:
 117
 118     retcode = call(["ls", "-l"])
 119
 120 check_call(*popenargs, **kwargs):
 121     Run command with arguments.  Wait for command to complete.  If the
 122     exit code was zero then return, otherwise raise
 123     CalledProcessError.  The CalledProcessError object will have the
 124     return code in the returncode attribute.
 125
 126     The arguments are the same as for the Popen constructor.  Example:
 127
 128     check_call(["ls", "-l"])
 129
 130 Exceptions
 131 ----------
 132 Exceptions raised in the child process, before the new program has
 133 started to execute, will be re-raised in the parent.  Additionally,
 134 the exception object will have one extra attribute called
 135 'child_traceback', which is a string containing traceback information
 136 from the childs point of view.
 137
 138 The most common exception raised is OSError.  This occurs, for
 139 example, when trying to execute a non-existent file.  Applications
 140 should prepare for OSErrors.
 141
 142 A ValueError will be raised if Popen is called with invalid arguments.
 143
 144 check_call() will raise CalledProcessError, if the called process
 145 returns a non-zero return code.
 146
 147
 148 Security
 149 --------
 150 Unlike some other popen functions, this implementation will never call
 151 /bin/sh implicitly.  This means that all characters, including shell
 152 metacharacters, can safely be passed to child processes.
 153
 154
 155 Popen objects
 156 =============
 157 Instances of the Popen class have the following methods:
 158
 159 poll()
 160     Check if child process has terminated.  Returns returncode
 161     attribute.
 162
 163 wait()
 164     Wait for child process to terminate.  Returns returncode attribute.
 165
 166 communicate(input=None)
 167     Interact with process: Send data to stdin.  Read data from stdout
 168     and stderr, until end-of-file is reached.  Wait for process to
 169     terminate.  The optional input argument should be a string to be
 170     sent to the child process, or None, if no data should be sent to
 171     the child.
 172
 173     communicate() returns a tuple (stdout, stderr).
 174
 175     Note: The data read is buffered in memory, so do not use this
 176     method if the data size is large or unlimited.
 177
 178 The following attributes are also available:
 179
 180 stdin
 181     If the stdin argument is PIPE, this attribute is a file object
 182     that provides input to the child process.  Otherwise, it is None.
 183
 184 stdout
 185     If the stdout argument is PIPE, this attribute is a file object
 186     that provides output from the child process.  Otherwise, it is
 187     None.
 188
 189 stderr
 190     If the stderr argument is PIPE, this attribute is file object that
 191     provides error output from the child process.  Otherwise, it is
 192     None.
 193
 194 pid
 195     The process ID of the child process.
 196
 197 returncode
 198     The child return code.  A None value indicates that the process
 199     hasn't terminated yet.  A negative value -N indicates that the
 200     child was terminated by signal N (UNIX only).
 201
 202
 203 Replacing older functions with the subprocess module
 204 ====================================================
 205 In this section, "a ==> b" means that b can be used as a replacement
 206 for a.
 207
 208 Note: All functions in this section fail (more or less) silently if
 209 the executed program cannot be found; this module raises an OSError
 210 exception.
 211
 212 In the following examples, we assume that the subprocess module is
 213 imported with "from subprocess import *".
 214
 215
 216 Replacing /bin/sh shell backquote
 217 ---------------------------------
 218 output=`mycmd myarg`
 219 ==>
 220 output = Popen(["mycmd", "myarg"], stdout=PIPE).communicate()[0]
 221
 222
 223 Replacing shell pipe line
 224 -------------------------
 225 output=`dmesg | grep hda`
 226 ==>
 227 p1 = Popen(["dmesg"], stdout=PIPE)
 228 p2 = Popen(["grep", "hda"], stdin=p1.stdout, stdout=PIPE)
 229 output = p2.communicate()[0]
 230
 231
 232 Replacing os.system()
 233 ---------------------
 234 sts = os.system("mycmd" + " myarg")
 235 ==>
 236 p = Popen("mycmd" + " myarg", shell=True)
 237 pid, sts = os.waitpid(p.pid, 0)
 238
 239 Note:
 240
 241 * Calling the program through the shell is usually not required.
 242
 243 * It's easier to look at the returncode attribute than the
 244   exitstatus.
 245
 246 A more real-world example would look like this:
 247
 248 try:
 249     retcode = call("mycmd" + " myarg", shell=True)
 250     if retcode < 0:
 251         print >>sys.stderr, "Child was terminated by signal", -retcode
 252     else:
 253         print >>sys.stderr, "Child returned", retcode
 254 except OSError, e:
 255     print >>sys.stderr, "Execution failed:", e
 256
 257
 258 Replacing os.spawn*
 259 -------------------
 260 P_NOWAIT example:
 261
 262 pid = os.spawnlp(os.P_NOWAIT, "/bin/mycmd", "mycmd", "myarg")
 263 ==>
 264 pid = Popen(["/bin/mycmd", "myarg"]).pid
 265
 266
 267 P_WAIT example:
 268
 269 retcode = os.spawnlp(os.P_WAIT, "/bin/mycmd", "mycmd", "myarg")
 270 ==>
 271 retcode = call(["/bin/mycmd", "myarg"])
 272
 273
 274 Vector example:
 275
 276 os.spawnvp(os.P_NOWAIT, path, args)
 277 ==>
 278 Popen([path] + args[1:])
 279
 280
 281 Environment example:
 282
 283 os.spawnlpe(os.P_NOWAIT, "/bin/mycmd", "mycmd", "myarg", env)
 284 ==>
 285 Popen(["/bin/mycmd", "myarg"], env={"PATH": "/usr/bin"})
 286
 287
 288 Replacing os.popen*
 289 -------------------
 290 pipe = os.popen(cmd, mode='r', bufsize)
 291 ==>
 292 pipe = Popen(cmd, shell=True, bufsize=bufsize, stdout=PIPE).stdout
 293
 294 pipe = os.popen(cmd, mode='w', bufsize)
 295 ==>
 296 pipe = Popen(cmd, shell=True, bufsize=bufsize, stdin=PIPE).stdin
 297
 298
 299 (child_stdin, child_stdout) = os.popen2(cmd, mode, bufsize)
 300 ==>
 301 p = Popen(cmd, shell=True, bufsize=bufsize,
 302           stdin=PIPE, stdout=PIPE, close_fds=True)
 303 (child_stdin, child_stdout) = (p.stdin, p.stdout)
 304
 305
 306 (child_stdin,
 307  child_stdout,
 308  child_stderr) = os.popen3(cmd, mode, bufsize)
 309 ==>
 310 p = Popen(cmd, shell=True, bufsize=bufsize,
 311           stdin=PIPE, stdout=PIPE, stderr=PIPE, close_fds=True)
 312 (child_stdin,
 313  child_stdout,
 314  child_stderr) = (p.stdin, p.stdout, p.stderr)
 315
 316
 317 (child_stdin, child_stdout_and_stderr) = os.popen4(cmd, mode, bufsize)
 318 ==>
 319 p = Popen(cmd, shell=True, bufsize=bufsize,
 320           stdin=PIPE, stdout=PIPE, stderr=STDOUT, close_fds=True)
 321 (child_stdin, child_stdout_and_stderr) = (p.stdin, p.stdout)
 322
 323
 324 Replacing popen2.*
 325 ------------------
 326 Note: If the cmd argument to popen2 functions is a string, the command
 327 is executed through /bin/sh.  If it is a list, the command is directly
 328 executed.
 329
 330 (child_stdout, child_stdin) = popen2.popen2("somestring", bufsize, mode)
 331 ==>
 332 p = Popen(["somestring"], shell=True, bufsize=bufsize
 333           stdin=PIPE, stdout=PIPE, close_fds=True)
 334 (child_stdout, child_stdin) = (p.stdout, p.stdin)
 335
 336
 337 (child_stdout, child_stdin) = popen2.popen2(["mycmd", "myarg"], bufsize, mode)
 338 ==>
 339 p = Popen(["mycmd", "myarg"], bufsize=bufsize,
 340           stdin=PIPE, stdout=PIPE, close_fds=True)
 341 (child_stdout, child_stdin) = (p.stdout, p.stdin)
 342
 343 The popen2.Popen3 and popen3.Popen4 basically works as subprocess.Popen,
 344 except that:
 345
 346 * subprocess.Popen raises an exception if the execution fails
 347 * the capturestderr argument is replaced with the stderr argument.
 348 * stdin=PIPE and stdout=PIPE must be specified.
 349 * popen2 closes all filedescriptors by default, but you have to specify
 350   close_fds=True with subprocess.Popen.
 351
 352
 353 """
 354
 355 import sys
 356 mswindows = (sys.platform == "win32")
 357
 358 import os
 359 import types
 360 import traceback
 361
 362 # Exception classes used by this module.
 363 class CalledProcessError(Exception):
 364     """This exception is raised when a process run by check_call() returns
 365     a non-zero exit status.  The exit status will be stored in the
 366     returncode attribute."""
 367     def __init__(self, returncode, cmd):
 368         self.returncode = returncode
 369         self.cmd = cmd
 370     def __str__(self):
 371         return "Command '%s' returned non-zero exit status %d" % (self.cmd, self.returncode)
 372
 373
 374 if mswindows:
 375     import threading
 376     import msvcrt
 377     if 0: # <-- change this to use pywin32 instead of the _subprocess driver
 378         import pywintypes
 379         from win32api import GetStdHandle, STD_INPUT_HANDLE, \
 380                              STD_OUTPUT_HANDLE, STD_ERROR_HANDLE
 381         from win32api import GetCurrentProcess, DuplicateHandle, \
 382                              GetModuleFileName, GetVersion
 383         from win32con import DUPLICATE_SAME_ACCESS, SW_HIDE
 384         from win32pipe import CreatePipe
 385         from win32process import CreateProcess, STARTUPINFO, \
 386                                  GetExitCodeProcess, STARTF_USESTDHANDLES, \
 387                                  STARTF_USESHOWWINDOW, CREATE_NEW_CONSOLE
 388         from win32event import WaitForSingleObject, INFINITE, WAIT_OBJECT_0
 389     else:
 390         from _subprocess import *
 391         class STARTUPINFO:
 392             dwFlags = 0
 393             hStdInput = None
 394             hStdOutput = None
 395             hStdError = None
 396             wShowWindow = 0
 397         class pywintypes:
 398             error = IOError
 399 else:
 400     import select
 401     import errno
 402     import fcntl
 403     import pickle
 404
 405 __all__ = ["Popen", "PIPE", "STDOUT", "call", "check_call", "CalledProcessError"]
 406
 407 try:
 408     MAXFD = os.sysconf("SC_OPEN_MAX")
 409 except:
 410     MAXFD = 256
 411
 412 # True/False does not exist on 2.2.0
 413 try:
 414     False
 415 except NameError:
 416     False = 0
 417     True = 1
 418
 419 _active = []
 420
 421 def _cleanup():
 422     for inst in _active[:]:
 423         if inst.poll(_deadstate=sys.maxint) >= 0:
 424             try:
 425                 _active.remove(inst)
 426             except ValueError:
 427                 # This can happen if two threads create a new Popen instance.
 428                 # It's harmless that it was already removed, so ignore.
 429                 pass
 430
 431 PIPE = -1
 432 STDOUT = -2
 433
 434
 435 def call(*popenargs, **kwargs):
 436     """Run command with arguments.  Wait for command to complete, then
 437     return the returncode attribute.
 438
 439     The arguments are the same as for the Popen constructor.  Example:
 440
 441     retcode = call(["ls", "-l"])
 442     """
 443     return Popen(*popenargs, **kwargs).wait()
 444
 445
 446 def check_call(*popenargs, **kwargs):
 447     """Run command with arguments.  Wait for command to complete.  If
 448     the exit code was zero then return, otherwise raise
 449     CalledProcessError.  The CalledProcessError object will have the
 450     return code in the returncode attribute.
 451
 452     The arguments are the same as for the Popen constructor.  Example:
 453
 454     check_call(["ls", "-l"])
 455     """
 456     retcode = call(*popenargs, **kwargs)
 457     cmd = kwargs.get("args")
 458     if cmd is None:
 459         cmd = popenargs[0]
 460     if retcode:
 461         raise CalledProcessError(retcode, cmd)
 462     return retcode
 463
 464
 465 def list2cmdline(seq):
 466     """
 467     Translate a sequence of arguments into a command line
 468     string, using the same rules as the MS C runtime:
 469
 470     1) Arguments are delimited by white space, which is either a
 471        space or a tab.
 472
 473     2) A string surrounded by double quotation marks is
 474        interpreted as a single argument, regardless of white space
 475        contained within.  A quoted string can be embedded in an
 476        argument.
 477
 478     3) A double quotation mark preceded by a backslash is
 479        interpreted as a literal double quotation mark.
 480
 481     4) Backslashes are interpreted literally, unless they
 482        immediately precede a double quotation mark.
 483
 484     5) If backslashes immediately precede a double quotation mark,
 485        every pair of backslashes is interpreted as a literal
 486        backslash.  If the number of backslashes is odd, the last
 487        backslash escapes the next double quotation mark as
 488        described in rule 3.
 489     """
 490
 491     # See
 492     # http://msdn.microsoft.com/library/en-us/vccelng/htm/progs_12.asp
 493     result = []
 494     needquote = False
 495     for arg in seq:
 496         bs_buf = []
 497
 498         # Add a space to separate this argument from the others
 499         if result:
 500             result.append(' ')
 501
 502         needquote = (" " in arg) or ("\t" in arg) or arg == ""
 503         if needquote:
 504             result.append('"')
 505
 506         for c in arg:
 507             if c == '\\':
 508                 # Don't know if we need to double yet.
 509                 bs_buf.append(c)
 510             elif c == '"':
 511                 # Double backspaces.
 512                 result.append('\\' * len(bs_buf)*2)
 513                 bs_buf = []
 514                 result.append('\\"')
 515             else:
 516                 # Normal char
 517                 if bs_buf:
 518                     result.extend(bs_buf)
 519                     bs_buf = []
 520                 result.append(c)
 521
 522         # Add remaining backspaces, if any.
 523         if bs_buf:
 524             result.extend(bs_buf)
 525
 526         if needquote:
 527             result.extend(bs_buf)
 528             result.append('"')
 529
 530     return ''.join(result)
 531
 532
 533 class Popen(object):
 534     def __init__(self, args, bufsize=0, executable=None,
 535                  stdin=None, stdout=None, stderr=None,
 536                  preexec_fn=None, close_fds=False, shell=False,
 537                  cwd=None, env=None, universal_newlines=False,
 538                  startupinfo=None, creationflags=0):
 539         """Create new Popen instance."""
 540         _cleanup()
 541
 542         self._child_created = False
 543         if not isinstance(bufsize, (int, long)):
 544             raise TypeError("bufsize must be an integer")
 545
 546         if mswindows:
 547             if preexec_fn is not None:
 548                 raise ValueError("preexec_fn is not supported on Windows "
 549                                  "platforms")
 550             if close_fds:
 551                 raise ValueError("close_fds is not supported on Windows "
 552                                  "platforms")
 553         else:
 554             # POSIX
 555             if startupinfo is not None:
 556                 raise ValueError("startupinfo is only supported on Windows "
 557                                  "platforms")
 558             if creationflags != 0:
 559                 raise ValueError("creationflags is only supported on Windows "
 560                                  "platforms")
 561
 562         self.stdin = None
 563         self.stdout = None
 564         self.stderr = None
 565         self.pid = None
 566         self.returncode = None
 567         self.universal_newlines = universal_newlines
 568
 569         # Input and output objects. The general principle is like
 570         # this:
 571         #
 572         # Parent                   Child
 573         # ------                   -----
 574         # p2cwrite   ---stdin--->  p2cread
 575         # c2pread    <--stdout---  c2pwrite
 576         # errread    <--stderr---  errwrite
 577         #
 578         # On POSIX, the child objects are file descriptors.  On
 579         # Windows, these are Windows file handles.  The parent objects
 580         # are file descriptors on both platforms.  The parent objects
 581         # are None when not using PIPEs. The child objects are None
 582         # when not redirecting.
 583
 584         (p2cread, p2cwrite,
 585          c2pread, c2pwrite,
 586          errread, errwrite) = self._get_handles(stdin, stdout, stderr)
 587
 588         self._execute_child(args, executable, preexec_fn, close_fds,
 589                             cwd, env, universal_newlines,
 590                             startupinfo, creationflags, shell,
 591                             p2cread, p2cwrite,
 592                             c2pread, c2pwrite,
 593                             errread, errwrite)
 594
 595         # On Windows, you cannot just redirect one or two handles: You
 596         # either have to redirect all three or none. If the subprocess
 597         # user has only redirected one or two handles, we are
 598         # automatically creating PIPEs for the rest. We should close
 599         # these after the process is started. See bug #1124861.
 600         if mswindows:
 601             if stdin is None and p2cwrite is not None:
 602                 os.close(p2cwrite)
 603                 p2cwrite = None
 604             if stdout is None and c2pread is not None:
 605                 os.close(c2pread)
 606                 c2pread = None
 607             if stderr is None and errread is not None:
 608                 os.close(errread)
 609                 errread = None
 610
 611         if p2cwrite is not None:
 612             self.stdin = os.fdopen(p2cwrite, 'wb', bufsize)
 613         if c2pread is not None:
 614             if universal_newlines:
 615                 self.stdout = os.fdopen(c2pread, 'rU', bufsize)
 616             else:
 617                 self.stdout = os.fdopen(c2pread, 'rb', bufsize)
 618         if errread is not None:
 619             if universal_newlines:
 620                 self.stderr = os.fdopen(errread, 'rU', bufsize)
 621             else:
 622                 self.stderr = os.fdopen(errread, 'rb', bufsize)
 623
 624
 625     def _translate_newlines(self, data):
 626         data = data.replace("\r\n", "\n")
 627         data = data.replace("\r", "\n")
 628         return data
 629
 630
 631     def __del__(self, sys=sys):
 632         if not self._child_created:
 633             # We didn't get to successfully create a child process.
 634             return
 635         # In case the child hasn't been waited on, check if it's done.
 636         self.poll(_deadstate=sys.maxint)
 637         if self.returncode is None and _active is not None:
 638             # Child is still running, keep us alive until we can wait on it.
 639             _active.append(self)
 640
 641
 642     def communicate(self, input=None):
 643         """Interact with process: Send data to stdin.  Read data from
 644         stdout and stderr, until end-of-file is reached.  Wait for
 645         process to terminate.  The optional input argument should be a
 646         string to be sent to the child process, or None, if no data
 647         should be sent to the child.
 648
 649         communicate() returns a tuple (stdout, stderr)."""
 650
 651         # Optimization: If we are only using one pipe, or no pipe at
 652         # all, using select() or threads is unnecessary.
 653         if [self.stdin, self.stdout, self.stderr].count(None) >= 2:
 654             stdout = None
 655             stderr = None
 656             if self.stdin:
 657                 if input:
 658                     self.stdin.write(input)
 659                 self.stdin.close()
 660             elif self.stdout:
 661                 stdout = self.stdout.read()
 662             elif self.stderr:
 663                 stderr = self.stderr.read()
 664             self.wait()
 665             return (stdout, stderr)
 666
 667         return self._communicate(input)
 668
 669
 670     if mswindows:
 671         #
 672         # Windows methods
 673         #
 674         def _get_handles(self, stdin, stdout, stderr):
 675             """Construct and return tupel with IO objects:
 676             p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite
 677             """
 678             if stdin is None and stdout is None and stderr is None:
 679                 return (None, None, None, None, None, None)
 680
 681             p2cread, p2cwrite = None, None
 682             c2pread, c2pwrite = None, None
 683             errread, errwrite = None, None
 684
 685             if stdin is None:
 686                 p2cread = GetStdHandle(STD_INPUT_HANDLE)
 687             if p2cread is not None:
 688                 pass
 689             elif stdin is None or stdin == PIPE:
 690                 p2cread, p2cwrite = CreatePipe(None, 0)
 691                 # Detach and turn into fd
 692                 p2cwrite = p2cwrite.Detach()
 693                 p2cwrite = msvcrt.open_osfhandle(p2cwrite, 0)
 694             elif isinstance(stdin, int):
 695                 p2cread = msvcrt.get_osfhandle(stdin)
 696             else:
 697                 # Assuming file-like object
 698                 p2cread = msvcrt.get_osfhandle(stdin.fileno())
 699             p2cread = self._make_inheritable(p2cread)
 700
 701             if stdout is None:
 702                 c2pwrite = GetStdHandle(STD_OUTPUT_HANDLE)
 703             if c2pwrite is not None:
 704                 pass
 705             elif stdout is None or stdout == PIPE:
 706                 c2pread, c2pwrite = CreatePipe(None, 0)
 707                 # Detach and turn into fd
 708                 c2pread = c2pread.Detach()
 709                 c2pread = msvcrt.open_osfhandle(c2pread, 0)
 710             elif isinstance(stdout, int):
 711                 c2pwrite = msvcrt.get_osfhandle(stdout)
 712             else:
 713                 # Assuming file-like object
 714                 c2pwrite = msvcrt.get_osfhandle(stdout.fileno())
 715             c2pwrite = self._make_inheritable(c2pwrite)
 716
 717             if stderr is None:
 718                 errwrite = GetStdHandle(STD_ERROR_HANDLE)
 719             if errwrite is not None:
 720                 pass
 721             elif stderr is None or stderr == PIPE:
 722                 errread, errwrite = CreatePipe(None, 0)
 723                 # Detach and turn into fd
 724                 errread = errread.Detach()
 725                 errread = msvcrt.open_osfhandle(errread, 0)
 726             elif stderr == STDOUT:
 727                 errwrite = c2pwrite
 728             elif isinstance(stderr, int):
 729                 errwrite = msvcrt.get_osfhandle(stderr)
 730             else:
 731                 # Assuming file-like object
 732                 errwrite = msvcrt.get_osfhandle(stderr.fileno())
 733             errwrite = self._make_inheritable(errwrite)
 734
 735             return (p2cread, p2cwrite,
 736                     c2pread, c2pwrite,
 737                     errread, errwrite)
 738
 739
 740         def _make_inheritable(self, handle):
 741             """Return a duplicate of handle, which is inheritable"""
 742             return DuplicateHandle(GetCurrentProcess(), handle,
 743                                    GetCurrentProcess(), 0, 1,
 744                                    DUPLICATE_SAME_ACCESS)
 745
 746
 747         def _find_w9xpopen(self):
 748             """Find and return absolut path to w9xpopen.exe"""
 749             w9xpopen = os.path.join(os.path.dirname(GetModuleFileName(0)),
 750                                     "w9xpopen.exe")
 751             if not os.path.exists(w9xpopen):
 752                 # Eeek - file-not-found - possibly an embedding
 753                 # situation - see if we can locate it in sys.exec_prefix
 754                 w9xpopen = os.path.join(os.path.dirname(sys.exec_prefix),
 755                                         "w9xpopen.exe")
 756                 if not os.path.exists(w9xpopen):
 757                     raise RuntimeError("Cannot locate w9xpopen.exe, which is "
 758                                        "needed for Popen to work with your "
 759                                        "shell or platform.")
 760             return w9xpopen
 761
 762
 763         def _execute_child(self, args, executable, preexec_fn, close_fds,
 764                            cwd, env, universal_newlines,
 765                            startupinfo, creationflags, shell,
 766                            p2cread, p2cwrite,
 767                            c2pread, c2pwrite,
 768                            errread, errwrite):
 769             """Execute program (MS Windows version)"""
 770
 771             if not isinstance(args, types.StringTypes):
 772                 args = list2cmdline(args)
 773
 774             # Process startup details
 775             if startupinfo is None:
 776                 startupinfo = STARTUPINFO()
 777             if None not in (p2cread, c2pwrite, errwrite):
 778                 startupinfo.dwFlags |= STARTF_USESTDHANDLES
 779                 startupinfo.hStdInput = p2cread
 780                 startupinfo.hStdOutput = c2pwrite
 781                 startupinfo.hStdError = errwrite
 782
 783             if shell:
 784                 startupinfo.dwFlags |= STARTF_USESHOWWINDOW
 785                 startupinfo.wShowWindow = SW_HIDE
 786                 comspec = os.environ.get("COMSPEC", "cmd.exe")
 787                 args = comspec + " /c " + args
 788                 if (GetVersion() >= 0x80000000L or
 789                         os.path.basename(comspec).lower() == "command.com"):
 790                     # Win9x, or using command.com on NT. We need to
 791                     # use the w9xpopen intermediate program. For more
 792                     # information, see KB Q150956
 793                     # (http://web.archive.org/web/20011105084002/http://support.microsoft.com/support/kb/articles/Q150/9/56.asp)
 794                     w9xpopen = self._find_w9xpopen()
 795                     args = '"%s" %s' % (w9xpopen, args)
 796                     # Not passing CREATE_NEW_CONSOLE has been known to
 797                     # cause random failures on win9x.  Specifically a
 798                     # dialog: "Your program accessed mem currently in
 799                     # use at xxx" and a hopeful warning about the
 800                     # stability of your system.  Cost is Ctrl+C wont
 801                     # kill children.
 802                     creationflags |= CREATE_NEW_CONSOLE
 803
 804             # Start the process
 805             try:
 806                 hp, ht, pid, tid = CreateProcess(executable, args,
 807                                          # no special security
 808                                          None, None,
 809                                          # must inherit handles to pass std
 810                                          # handles
 811                                          1,
 812                                          creationflags,
 813                                          env,
 814                                          cwd,
 815                                          startupinfo)
 816             except pywintypes.error, e:
 817                 # Translate pywintypes.error to WindowsError, which is
 818                 # a subclass of OSError.  FIXME: We should really
 819                 # translate errno using _sys_errlist (or simliar), but
 820                 # how can this be done from Python?
 821                 raise WindowsError(*e.args)
 822
 823             # Retain the process handle, but close the thread handle
 824             self._child_created = True
 825             self._handle = hp
 826             self.pid = pid
 827             ht.Close()
 828
 829             # Child is launched. Close the parent's copy of those pipe
 830             # handles that only the child should have open.  You need
 831             # to make sure that no handles to the write end of the
 832             # output pipe are maintained in this process or else the
 833             # pipe will not close when the child process exits and the
 834             # ReadFile will hang.
 835             if p2cread is not None:
 836                 p2cread.Close()
 837             if c2pwrite is not None:
 838                 c2pwrite.Close()
 839             if errwrite is not None:
 840                 errwrite.Close()
 841
 842
 843         def poll(self, _deadstate=None):
 844             """Check if child process has terminated.  Returns returncode
 845             attribute."""
 846             if self.returncode is None:
 847                 if WaitForSingleObject(self._handle, 0) == WAIT_OBJECT_0:
 848                     self.returncode = GetExitCodeProcess(self._handle)
 849             return self.returncode
 850
 851
 852         def wait(self):
 853             """Wait for child process to terminate.  Returns returncode
 854             attribute."""
 855             if self.returncode is None:
 856                 obj = WaitForSingleObject(self._handle, INFINITE)
 857                 self.returncode = GetExitCodeProcess(self._handle)
 858             return self.returncode
 859
 860
 861         def _readerthread(self, fh, buffer):
 862             buffer.append(fh.read())
 863
 864
 865         def _communicate(self, input):
 866             stdout = None # Return
 867             stderr = None # Return
 868
 869             if self.stdout:
 870                 stdout = []
 871                 stdout_thread = threading.Thread(target=self._readerthread,
 872                                                  args=(self.stdout, stdout))
 873                 stdout_thread.setDaemon(True)
 874                 stdout_thread.start()
 875             if self.stderr:
 876                 stderr = []
 877                 stderr_thread = threading.Thread(target=self._readerthread,
 878                                                  args=(self.stderr, stderr))
 879                 stderr_thread.setDaemon(True)
 880                 stderr_thread.start()
 881
 882             if self.stdin:
 883                 if input is not None:
 884                     self.stdin.write(input)
 885                 self.stdin.close()
 886
 887             if self.stdout:
 888                 stdout_thread.join()
 889             if self.stderr:
 890                 stderr_thread.join()
 891
 892             # All data exchanged.  Translate lists into strings.
 893             if stdout is not None:
 894                 stdout = stdout[0]
 895             if stderr is not None:
 896                 stderr = stderr[0]
 897
 898             # Translate newlines, if requested.  We cannot let the file
 899             # object do the translation: It is based on stdio, which is
 900             # impossible to combine with select (unless forcing no
 901             # buffering).
 902             if self.universal_newlines and hasattr(file, 'newlines'):
 903                 if stdout:
 904                     stdout = self._translate_newlines(stdout)
 905                 if stderr:
 906                     stderr = self._translate_newlines(stderr)
 907
 908             self.wait()
 909             return (stdout, stderr)
 910
 911     else:
 912         #
 913         # POSIX methods
 914         #
 915         def _get_handles(self, stdin, stdout, stderr):
 916             """Construct and return tupel with IO objects:
 917             p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite
 918             """
 919             p2cread, p2cwrite = None, None
 920             c2pread, c2pwrite = None, None
 921             errread, errwrite = None, None
 922
 923             if stdin is None:
 924                 pass
 925             elif stdin == PIPE:
 926                 p2cread, p2cwrite = os.pipe()
 927             elif isinstance(stdin, int):
 928                 p2cread = stdin
 929             else:
 930                 # Assuming file-like object
 931                 p2cread = stdin.fileno()
 932
 933             if stdout is None:
 934                 pass
 935             elif stdout == PIPE:
 936                 c2pread, c2pwrite = os.pipe()
 937             elif isinstance(stdout, int):
 938                 c2pwrite = stdout
 939             else:
 940                 # Assuming file-like object
 941                 c2pwrite = stdout.fileno()
 942
 943             if stderr is None:
 944                 pass
 945             elif stderr == PIPE:
 946                 errread, errwrite = os.pipe()
 947             elif stderr == STDOUT:
 948                 errwrite = c2pwrite
 949             elif isinstance(stderr, int):
 950                 errwrite = stderr
 951             else:
 952                 # Assuming file-like object
 953                 errwrite = stderr.fileno()
 954
 955             return (p2cread, p2cwrite,
 956                     c2pread, c2pwrite,
 957                     errread, errwrite)
 958
 959
 960         def _set_cloexec_flag(self, fd):
 961             try:
 962                 cloexec_flag = fcntl.FD_CLOEXEC
 963             except AttributeError:
 964                 cloexec_flag = 1
 965
 966             old = fcntl.fcntl(fd, fcntl.F_GETFD)
 967             fcntl.fcntl(fd, fcntl.F_SETFD, old | cloexec_flag)
 968
 969
 970         def _close_fds(self, but):
 971             for i in xrange(3, MAXFD):
 972                 if i == but:
 973                     continue
 974                 try:
 975                     os.close(i)
 976                 except:
 977                     pass
 978
 979
 980         def _execute_child(self, args, executable, preexec_fn, close_fds,
 981                            cwd, env, universal_newlines,
 982                            startupinfo, creationflags, shell,
 983                            p2cread, p2cwrite,
 984                            c2pread, c2pwrite,
 985                            errread, errwrite):
 986             """Execute program (POSIX version)"""
 987
 988             if isinstance(args, types.StringTypes):
 989                 args = [args]
 990             else:
 991                 args = list(args)
 992
 993             if shell:
 994                 args = ["/bin/sh", "-c"] + args
 995
 996             if executable is None:
 997                 executable = args[0]
 998
 999             # For transferring possible exec failure from child to parent
1000             # The first char specifies the exception type: 0 means
1001             # OSError, 1 means some other error.
1002             errpipe_read, errpipe_write = os.pipe()
1003             self._set_cloexec_flag(errpipe_write)
1004
1005             self.pid = os.fork()
1006             self._child_created = True
1007             if self.pid == 0:
1008                 # Child
1009                 try:
1010                     # Close parent's pipe ends
1011                     if p2cwrite is not None:
1012                         os.close(p2cwrite)
1013                     if c2pread is not None:
1014                         os.close(c2pread)
1015                     if errread is not None:
1016                         os.close(errread)
1017                     os.close(errpipe_read)
1018
1019                     # Dup fds for child
1020                     if p2cread is not None:
1021                         os.dup2(p2cread, 0)
1022                     if c2pwrite is not None:
1023                         os.dup2(c2pwrite, 1)
1024                     if errwrite is not None:
1025                         os.dup2(errwrite, 2)
1026
1027                     # Close pipe fds.  Make sure we don't close the same
1028                     # fd more than once, or standard fds.
1029                     if p2cread is not None and p2cread not in (0,):
1030                         os.close(p2cread)
1031                     if c2pwrite is not None and c2pwrite not in (p2cread, 1):
1032                         os.close(c2pwrite)
1033                     if errwrite is not None and errwrite not in (p2cread, c2pwrite, 2):
1034                         os.close(errwrite)
1035
1036                     # Close all other fds, if asked for
1037                     if close_fds:
1038                         self._close_fds(but=errpipe_write)
1039
1040                     if cwd is not None:
1041                         os.chdir(cwd)
1042
1043                     if preexec_fn:
1044                         apply(preexec_fn)
1045
1046                     if env is None:
1047                         os.execvp(executable, args)
1048                     else:
1049                         os.execvpe(executable, args, env)
1050
1051                 except:
1052                     exc_type, exc_value, tb = sys.exc_info()
1053                     # Save the traceback and attach it to the exception object
1054                     exc_lines = traceback.format_exception(exc_type,
1055                                                            exc_value,
1056                                                            tb)
1057                     exc_value.child_traceback = ''.join(exc_lines)
1058                     os.write(errpipe_write, pickle.dumps(exc_value))
1059
1060                 # This exitcode won't be reported to applications, so it
1061                 # really doesn't matter what we return.
1062                 os._exit(255)
1063
1064             # Parent
1065             os.close(errpipe_write)
1066             if p2cread is not None and p2cwrite is not None:
1067                 os.close(p2cread)
1068             if c2pwrite is not None and c2pread is not None:
1069                 os.close(c2pwrite)
1070             if errwrite is not None and errread is not None:
1071                 os.close(errwrite)
1072
1073             # Wait for exec to fail or succeed; possibly raising exception
1074             data = os.read(errpipe_read, 1048576) # Exceptions limited to 1 MB
1075             os.close(errpipe_read)
1076             if data != "":
1077                 os.waitpid(self.pid, 0)
1078                 child_exception = pickle.loads(data)
1079                 raise child_exception
1080
1081
1082         def _handle_exitstatus(self, sts):
1083             if os.WIFSIGNALED(sts):
1084                 self.returncode = -os.WTERMSIG(sts)
1085             elif os.WIFEXITED(sts):
1086                 self.returncode = os.WEXITSTATUS(sts)
1087             else:
1088                 # Should never happen
1089                 raise RuntimeError("Unknown child exit status!")
1090
1091
1092         def poll(self, _deadstate=None):
1093             """Check if child process has terminated.  Returns returncode
1094             attribute."""
1095             if self.returncode is None:
1096                 try:
1097                     pid, sts = os.waitpid(self.pid, os.WNOHANG)
1098                     if pid == self.pid:
1099                         self._handle_exitstatus(sts)
1100                 except os.error:
1101                     if _deadstate is not None:
1102                         self.returncode = _deadstate
1103             return self.returncode
1104
1105
1106         def wait(self):
1107             """Wait for child process to terminate.  Returns returncode
1108             attribute."""
1109             if self.returncode is None:
1110                 pid, sts = os.waitpid(self.pid, 0)
1111                 self._handle_exitstatus(sts)
1112             return self.returncode
1113
1114
1115         def _communicate(self, input):
1116             read_set = []
1117             write_set = []
1118             stdout = None # Return
1119             stderr = None # Return
1120
1121             if self.stdin:
1122                 # Flush stdio buffer.  This might block, if the user has
1123                 # been writing to .stdin in an uncontrolled fashion.
1124                 self.stdin.flush()
1125                 if input:
1126                     write_set.append(self.stdin)
1127                 else:
1128                     self.stdin.close()
1129             if self.stdout:
1130                 read_set.append(self.stdout)
1131                 stdout = []
1132             if self.stderr:
1133                 read_set.append(self.stderr)
1134                 stderr = []
1135
1136             input_offset = 0
1137             while read_set or write_set:
1138                 rlist, wlist, xlist = select.select(read_set, write_set, [])
1139
1140                 if self.stdin in wlist:
1141                     # When select has indicated that the file is writable,
1142                     # we can write up to PIPE_BUF bytes without risk
1143                     # blocking.  POSIX defines PIPE_BUF >= 512
1144                     bytes_written = os.write(self.stdin.fileno(), buffer(input, input_offset, 512))
1145                     input_offset += bytes_written
1146                     if input_offset >= len(input):
1147                         self.stdin.close()
1148                         write_set.remove(self.stdin)
1149
1150                 if self.stdout in rlist:
1151                     data = os.read(self.stdout.fileno(), 1024)
1152                     if data == "":
1153                         self.stdout.close()
1154                         read_set.remove(self.stdout)
1155                     stdout.append(data)
1156
1157                 if self.stderr in rlist:
1158                     data = os.read(self.stderr.fileno(), 1024)
1159                     if data == "":
1160                         self.stderr.close()
1161                         read_set.remove(self.stderr)
1162                     stderr.append(data)
1163
1164             # All data exchanged.  Translate lists into strings.
1165             if stdout is not None:
1166                 stdout = ''.join(stdout)
1167             if stderr is not None:
1168                 stderr = ''.join(stderr)
1169
1170             # Translate newlines, if requested.  We cannot let the file
1171             # object do the translation: It is based on stdio, which is
1172             # impossible to combine with select (unless forcing no
1173             # buffering).
1174             if self.universal_newlines and hasattr(file, 'newlines'):
1175                 if stdout:
1176                     stdout = self._translate_newlines(stdout)
1177                 if stderr:
1178                     stderr = self._translate_newlines(stderr)
1179
1180             self.wait()
1181             return (stdout, stderr)
1182
1183
1184 def _demo_posix():
1185     #
1186     # Example 1: Simple redirection: Get process list
1187     #
1188     plist = Popen(["ps"], stdout=PIPE).communicate()[0]
1189     print "Process list:"
1190     print plist
1191
1192     #
1193     # Example 2: Change uid before executing child
1194     #
1195     if os.getuid() == 0:
1196         p = Popen(["id"], preexec_fn=lambda: os.setuid(100))
1197         p.wait()
1198
1199     #
1200     # Example 3: Connecting several subprocesses
1201     #
1202     print "Looking for 'hda'..."
1203     p1 = Popen(["dmesg"], stdout=PIPE)
1204     p2 = Popen(["grep", "hda"], stdin=p1.stdout, stdout=PIPE)
1205     print repr(p2.communicate()[0])
1206
1207     #
1208     # Example 4: Catch execution error
1209     #
1210     print
1211     print "Trying a weird file..."
1212     try:
1213         print Popen(["/this/path/does/not/exist"]).communicate()
1214     except OSError, e:
1215         if e.errno == errno.ENOENT:
1216             print "The file didn't exist.  I thought so..."
1217             print "Child traceback:"
1218             print e.child_traceback
1219         else:
1220             print "Error", e.errno
1221     else:
1222         print >>sys.stderr, "Gosh.  No error."
1223
1224
1225 def _demo_windows():
1226     #
1227     # Example 1: Connecting several subprocesses
1228     #
1229     print "Looking for 'PROMPT' in set output..."
1230     p1 = Popen("set", stdout=PIPE, shell=True)
1231     p2 = Popen('find "PROMPT"', stdin=p1.stdout, stdout=PIPE)
1232     print repr(p2.communicate()[0])
1233
1234     #
1235     # Example 2: Simple execution of program
1236     #
1237     print "Executing calc..."
1238     p = Popen("calc")
1239     p.wait()
1240
1241
1242 if __name__ == "__main__":
1243     if mswindows:
1244         _demo_windows()
1245     else:
1246         _demo_posix()