Lib/subprocess.py

   1 # subprocess - Subprocesses with accessible I/O streams
   2 #
   3 # For more information about this module, see PEP 324.
   4 #
   5 # This module should remain compatible with Python 2.2, see PEP 291.
   6 #
   7 # Copyright (c) 2003-2005 by Peter Astrand <astrand@lysator.liu.se>
   8 #
   9 # Licensed to PSF under a Contributor Agreement.
  10 # See http://www.python.org/2.4/license for licensing details.
  11
  12 r"""subprocess - Subprocesses with accessible I/O streams
  13
  14 This module allows you to spawn processes, connect to their
  15 input/output/error pipes, and obtain their return codes.  This module
  16 intends to replace several other, older modules and functions, like:
  17
  18 os.system
  19 os.spawn*
  20 os.popen*
  21 popen2.*
  22 commands.*
  23
  24 Information about how the subprocess module can be used to replace these
  25 modules and functions can be found below.
  26
  27
  28
  29 Using the subprocess module
  30 ===========================
  31 This module defines one class called Popen:
  32
  33 class Popen(args, bufsize=0, executable=None,
  34             stdin=None, stdout=None, stderr=None,
  35             preexec_fn=None, close_fds=False, shell=False,
  36             cwd=None, env=None, universal_newlines=False,
  37             startupinfo=None, creationflags=0):
  38
  39
  40 Arguments are:
  41
  42 args should be a string, or a sequence of program arguments.  The
  43 program to execute is normally the first item in the args sequence or
  44 string, but can be explicitly set by using the executable argument.
  45
  46 On UNIX, with shell=False (default): In this case, the Popen class
  47 uses os.execvp() to execute the child program.  args should normally
  48 be a sequence.  A string will be treated as a sequence with the string
  49 as the only item (the program to execute).
  50
  51 On UNIX, with shell=True: If args is a string, it specifies the
  52 command string to execute through the shell.  If args is a sequence,
  53 the first item specifies the command string, and any additional items
  54 will be treated as additional shell arguments.
  55
  56 On Windows: the Popen class uses CreateProcess() to execute the child
  57 program, which operates on strings.  If args is a sequence, it will be
  58 converted to a string using the list2cmdline method.  Please note that
  59 not all MS Windows applications interpret the command line the same
  60 way: The list2cmdline is designed for applications using the same
  61 rules as the MS C runtime.
  62
  63 bufsize, if given, has the same meaning as the corresponding argument
  64 to the built-in open() function: 0 means unbuffered, 1 means line
  65 buffered, any other positive value means use a buffer of
  66 (approximately) that size.  A negative bufsize means to use the system
  67 default, which usually means fully buffered.  The default value for
  68 bufsize is 0 (unbuffered).
  69
  70 stdin, stdout and stderr specify the executed programs' standard
  71 input, standard output and standard error file handles, respectively.
  72 Valid values are PIPE, an existing file descriptor (a positive
  73 integer), an existing file object, and None.  PIPE indicates that a
  74 new pipe to the child should be created.  With None, no redirection
  75 will occur; the child's file handles will be inherited from the
  76 parent.  Additionally, stderr can be STDOUT, which indicates that the
  77 stderr data from the applications should be captured into the same
  78 file handle as for stdout.
  79
  80 If preexec_fn is set to a callable object, this object will be called
  81 in the child process just before the child is executed.
  82
  83 If close_fds is true, all file descriptors except 0, 1 and 2 will be
  84 closed before the child process is executed.
  85
  86 if shell is true, the specified command will be executed through the
  87 shell.
  88
  89 If cwd is not None, the current directory will be changed to cwd
  90 before the child is executed.
  91
  92 If env is not None, it defines the environment variables for the new
  93 process.
  94
  95 If universal_newlines is true, the file objects stdout and stderr are
  96 opened as a text files, but lines may be terminated by any of '\n',
  97 the Unix end-of-line convention, '\r', the Macintosh convention or
  98 '\r\n', the Windows convention.  All of these external representations
  99 are seen as '\n' by the Python program.  Note: This feature is only
 100 available if Python is built with universal newline support (the
 101 default).  Also, the newlines attribute of the file objects stdout,
 102 stdin and stderr are not updated by the communicate() method.
 103
 104 The startupinfo and creationflags, if given, will be passed to the
 105 underlying CreateProcess() function.  They can specify things such as
 106 appearance of the main window and priority for the new process.
 107 (Windows only)
 108
 109
 110 This module also defines two shortcut functions:
 111
 112 call(*popenargs, **kwargs):
 113     Run command with arguments.  Wait for command to complete, then
 114     return the returncode attribute.
 115
 116     The arguments are the same as for the Popen constructor.  Example:
 117
 118     retcode = call(["ls", "-l"])
 119
 120 check_call(*popenargs, **kwargs):
 121     Run command with arguments.  Wait for command to complete.  If the
 122     exit code was zero then return, otherwise raise
 123     CalledProcessError.  The CalledProcessError object will have the
 124     return code in the returncode attribute.
 125
 126     The arguments are the same as for the Popen constructor.  Example:
 127
 128     check_call(["ls", "-l"])
 129
 130 Exceptions
 131 ----------
 132 Exceptions raised in the child process, before the new program has
 133 started to execute, will be re-raised in the parent.  Additionally,
 134 the exception object will have one extra attribute called
 135 'child_traceback', which is a string containing traceback information
 136 from the childs point of view.
 137
 138 The most common exception raised is OSError.  This occurs, for
 139 example, when trying to execute a non-existent file.  Applications
 140 should prepare for OSErrors.
 141
 142 A ValueError will be raised if Popen is called with invalid arguments.
 143
 144 check_call() will raise CalledProcessError, if the called process
 145 returns a non-zero return code.
 146
 147
 148 Security
 149 --------
 150 Unlike some other popen functions, this implementation will never call
 151 /bin/sh implicitly.  This means that all characters, including shell
 152 metacharacters, can safely be passed to child processes.
 153
 154
 155 Popen objects
 156 =============
 157 Instances of the Popen class have the following methods:
 158
 159 poll()
 160     Check if child process has terminated.  Returns returncode
 161     attribute.
 162
 163 wait()
 164     Wait for child process to terminate.  Returns returncode attribute.
 165
 166 communicate(input=None)
 167     Interact with process: Send data to stdin.  Read data from stdout
 168     and stderr, until end-of-file is reached.  Wait for process to
 169     terminate.  The optional input argument should be a string to be
 170     sent to the child process, or None, if no data should be sent to
 171     the child.
 172
 173     communicate() returns a tuple (stdout, stderr).
 174
 175     Note: The data read is buffered in memory, so do not use this
 176     method if the data size is large or unlimited.
 177
 178 The following attributes are also available:
 179
 180 stdin
 181     If the stdin argument is PIPE, this attribute is a file object
 182     that provides input to the child process.  Otherwise, it is None.
 183
 184 stdout
 185     If the stdout argument is PIPE, this attribute is a file object
 186     that provides output from the child process.  Otherwise, it is
 187     None.
 188
 189 stderr
 190     If the stderr argument is PIPE, this attribute is file object that
 191     provides error output from the child process.  Otherwise, it is
 192     None.
 193
 194 pid
 195     The process ID of the child process.
 196
 197 returncode
 198     The child return code.  A None value indicates that the process
 199     hasn't terminated yet.  A negative value -N indicates that the
 200     child was terminated by signal N (UNIX only).
 201
 202
 203 Replacing older functions with the subprocess module
 204 ====================================================
 205 In this section, "a ==> b" means that b can be used as a replacement
 206 for a.
 207
 208 Note: All functions in this section fail (more or less) silently if
 209 the executed program cannot be found; this module raises an OSError
 210 exception.
 211
 212 In the following examples, we assume that the subprocess module is
 213 imported with "from subprocess import *".
 214
 215
 216 Replacing /bin/sh shell backquote
 217 ---------------------------------
 218 output=`mycmd myarg`
 219 ==>
 220 output = Popen(["mycmd", "myarg"], stdout=PIPE).communicate()[0]
 221
 222
 223 Replacing shell pipe line
 224 -------------------------
 225 output=`dmesg | grep hda`
 226 ==>
 227 p1 = Popen(["dmesg"], stdout=PIPE)
 228 p2 = Popen(["grep", "hda"], stdin=p1.stdout, stdout=PIPE)
 229 output = p2.communicate()[0]
 230
 231
 232 Replacing os.system()
 233 ---------------------
 234 sts = os.system("mycmd" + " myarg")
 235 ==>
 236 p = Popen("mycmd" + " myarg", shell=True)
 237 pid, sts = os.waitpid(p.pid, 0)
 238
 239 Note:
 240
 241 * Calling the program through the shell is usually not required.
 242
 243 * It's easier to look at the returncode attribute than the
 244   exitstatus.
 245
 246 A more real-world example would look like this:
 247
 248 try:
 249     retcode = call("mycmd" + " myarg", shell=True)
 250     if retcode < 0:
 251         print >>sys.stderr, "Child was terminated by signal", -retcode
 252     else:
 253         print >>sys.stderr, "Child returned", retcode
 254 except OSError, e:
 255     print >>sys.stderr, "Execution failed:", e
 256
 257
 258 Replacing os.spawn*
 259 -------------------
 260 P_NOWAIT example:
 261
 262 pid = os.spawnlp(os.P_NOWAIT, "/bin/mycmd", "mycmd", "myarg")
 263 ==>
 264 pid = Popen(["/bin/mycmd", "myarg"]).pid
 265
 266
 267 P_WAIT example:
 268
 269 retcode = os.spawnlp(os.P_WAIT, "/bin/mycmd", "mycmd", "myarg")
 270 ==>
 271 retcode = call(["/bin/mycmd", "myarg"])
 272
 273
 274 Vector example:
 275
 276 os.spawnvp(os.P_NOWAIT, path, args)
 277 ==>
 278 Popen([path] + args[1:])
 279
 280
 281 Environment example:
 282
 283 os.spawnlpe(os.P_NOWAIT, "/bin/mycmd", "mycmd", "myarg", env)
 284 ==>
 285 Popen(["/bin/mycmd", "myarg"], env={"PATH": "/usr/bin"})
 286
 287
 288 Replacing os.popen*
 289 -------------------
 290 pipe = os.popen(cmd, mode='r', bufsize)
 291 ==>
 292 pipe = Popen(cmd, shell=True, bufsize=bufsize, stdout=PIPE).stdout
 293
 294 pipe = os.popen(cmd, mode='w', bufsize)
 295 ==>
 296 pipe = Popen(cmd, shell=True, bufsize=bufsize, stdin=PIPE).stdin
 297
 298
 299 (child_stdin, child_stdout) = os.popen2(cmd, mode, bufsize)
 300 ==>
 301 p = Popen(cmd, shell=True, bufsize=bufsize,
 302           stdin=PIPE, stdout=PIPE, close_fds=True)
 303 (child_stdin, child_stdout) = (p.stdin, p.stdout)
 304
 305
 306 (child_stdin,
 307  child_stdout,
 308  child_stderr) = os.popen3(cmd, mode, bufsize)
 309 ==>
 310 p = Popen(cmd, shell=True, bufsize=bufsize,
 311           stdin=PIPE, stdout=PIPE, stderr=PIPE, close_fds=True)
 312 (child_stdin,
 313  child_stdout,
 314  child_stderr) = (p.stdin, p.stdout, p.stderr)
 315
 316
 317 (child_stdin, child_stdout_and_stderr) = os.popen4(cmd, mode, bufsize)
 318 ==>
 319 p = Popen(cmd, shell=True, bufsize=bufsize,
 320           stdin=PIPE, stdout=PIPE, stderr=STDOUT, close_fds=True)
 321 (child_stdin, child_stdout_and_stderr) = (p.stdin, p.stdout)
 322
 323
 324 Replacing popen2.*
 325 ------------------
 326 Note: If the cmd argument to popen2 functions is a string, the command
 327 is executed through /bin/sh.  If it is a list, the command is directly
 328 executed.
 329
 330 (child_stdout, child_stdin) = popen2.popen2("somestring", bufsize, mode)
 331 ==>
 332 p = Popen(["somestring"], shell=True, bufsize=bufsize
 333           stdin=PIPE, stdout=PIPE, close_fds=True)
 334 (child_stdout, child_stdin) = (p.stdout, p.stdin)
 335
 336
 337 (child_stdout, child_stdin) = popen2.popen2(["mycmd", "myarg"], bufsize, mode)
 338 ==>
 339 p = Popen(["mycmd", "myarg"], bufsize=bufsize,
 340           stdin=PIPE, stdout=PIPE, close_fds=True)
 341 (child_stdout, child_stdin) = (p.stdout, p.stdin)
 342
 343 The popen2.Popen3 and popen2.Popen4 basically works as subprocess.Popen,
 344 except that:
 345
 346 * subprocess.Popen raises an exception if the execution fails
 347 * the capturestderr argument is replaced with the stderr argument.
 348 * stdin=PIPE and stdout=PIPE must be specified.
 349 * popen2 closes all filedescriptors by default, but you have to specify
 350   close_fds=True with subprocess.Popen.
 351 """
 352
 353 import sys
 354 mswindows = (sys.platform == "win32")
 355
 356 import os
 357 import types
 358 import traceback
 359 import gc
 360 import signal
 361
 362 # Exception classes used by this module.
 363 class CalledProcessError(Exception):
 364     """This exception is raised when a process run by check_call() returns
 365     a non-zero exit status.  The exit status will be stored in the
 366     returncode attribute."""
 367     def __init__(self, returncode, cmd):
 368         self.returncode = returncode
 369         self.cmd = cmd
 370     def __str__(self):
 371         return "Command '%s' returned non-zero exit status %d" % (self.cmd, self.returncode)
 372
 373
 374 if mswindows:
 375     import threading
 376     import msvcrt
 377     if 0: # <-- change this to use pywin32 instead of the _subprocess driver
 378         import pywintypes
 379         from win32api import GetStdHandle, STD_INPUT_HANDLE, \
 380                              STD_OUTPUT_HANDLE, STD_ERROR_HANDLE
 381         from win32api import GetCurrentProcess, DuplicateHandle, \
 382                              GetModuleFileName, GetVersion
 383         from win32con import DUPLICATE_SAME_ACCESS, SW_HIDE
 384         from win32pipe import CreatePipe
 385         from win32process import CreateProcess, STARTUPINFO, \
 386                                  GetExitCodeProcess, STARTF_USESTDHANDLES, \
 387                                  STARTF_USESHOWWINDOW, CREATE_NEW_CONSOLE
 388         from win32process import TerminateProcess
 389         from win32event import WaitForSingleObject, INFINITE, WAIT_OBJECT_0
 390     else:
 391         from _subprocess import *
 392         class STARTUPINFO:
 393             dwFlags = 0
 394             hStdInput = None
 395             hStdOutput = None
 396             hStdError = None
 397             wShowWindow = 0
 398         class pywintypes:
 399             error = IOError
 400 else:
 401     import select
 402     import errno
 403     import fcntl
 404     import pickle
 405
 406 __all__ = ["Popen", "PIPE", "STDOUT", "call", "check_call", "CalledProcessError"]
 407
 408 try:
 409     MAXFD = os.sysconf("SC_OPEN_MAX")
 410 except:
 411     MAXFD = 256
 412
 413 # True/False does not exist on 2.2.0
 414 try:
 415     False
 416 except NameError:
 417     False = 0
 418     True = 1
 419
 420 _active = []
 421
 422 def _cleanup():
 423     for inst in _active[:]:
 424         if inst.poll(_deadstate=sys.maxint) >= 0:
 425             try:
 426                 _active.remove(inst)
 427             except ValueError:
 428                 # This can happen if two threads create a new Popen instance.
 429                 # It's harmless that it was already removed, so ignore.
 430                 pass
 431
 432 PIPE = -1
 433 STDOUT = -2
 434
 435
 436 def call(*popenargs, **kwargs):
 437     """Run command with arguments.  Wait for command to complete, then
 438     return the returncode attribute.
 439
 440     The arguments are the same as for the Popen constructor.  Example:
 441
 442     retcode = call(["ls", "-l"])
 443     """
 444     return Popen(*popenargs, **kwargs).wait()
 445
 446
 447 def check_call(*popenargs, **kwargs):
 448     """Run command with arguments.  Wait for command to complete.  If
 449     the exit code was zero then return, otherwise raise
 450     CalledProcessError.  The CalledProcessError object will have the
 451     return code in the returncode attribute.
 452
 453     The arguments are the same as for the Popen constructor.  Example:
 454
 455     check_call(["ls", "-l"])
 456     """
 457     retcode = call(*popenargs, **kwargs)
 458     cmd = kwargs.get("args")
 459     if cmd is None:
 460         cmd = popenargs[0]
 461     if retcode:
 462         raise CalledProcessError(retcode, cmd)
 463     return retcode
 464
 465
 466 def list2cmdline(seq):
 467     """
 468     Translate a sequence of arguments into a command line
 469     string, using the same rules as the MS C runtime:
 470
 471     1) Arguments are delimited by white space, which is either a
 472        space or a tab.
 473
 474     2) A string surrounded by double quotation marks is
 475        interpreted as a single argument, regardless of white space
 476        or pipe characters contained within.  A quoted string can be
 477        embedded in an argument.
 478
 479     3) A double quotation mark preceded by a backslash is
 480        interpreted as a literal double quotation mark.
 481
 482     4) Backslashes are interpreted literally, unless they
 483        immediately precede a double quotation mark.
 484
 485     5) If backslashes immediately precede a double quotation mark,
 486        every pair of backslashes is interpreted as a literal
 487        backslash.  If the number of backslashes is odd, the last
 488        backslash escapes the next double quotation mark as
 489        described in rule 3.
 490     """
 491
 492     # See
 493     # http://msdn.microsoft.com/library/en-us/vccelng/htm/progs_12.asp
 494     result = []
 495     needquote = False
 496     for arg in seq:
 497         bs_buf = []
 498
 499         # Add a space to separate this argument from the others
 500         if result:
 501             result.append(' ')
 502
 503         needquote = (" " in arg) or ("\t" in arg) or ("|" in arg) or not arg
 504         if needquote:
 505             result.append('"')
 506
 507         for c in arg:
 508             if c == '\\':
 509                 # Don't know if we need to double yet.
 510                 bs_buf.append(c)
 511             elif c == '"':
 512                 # Double backslashes.
 513                 result.append('\\' * len(bs_buf)*2)
 514                 bs_buf = []
 515                 result.append('\\"')
 516             else:
 517                 # Normal char
 518                 if bs_buf:
 519                     result.extend(bs_buf)
 520                     bs_buf = []
 521                 result.append(c)
 522
 523         # Add remaining backslashes, if any.
 524         if bs_buf:
 525             result.extend(bs_buf)
 526
 527         if needquote:
 528             result.extend(bs_buf)
 529             result.append('"')
 530
 531     return ''.join(result)
 532
 533
 534 class Popen(object):
 535     def __init__(self, args, bufsize=0, executable=None,
 536                  stdin=None, stdout=None, stderr=None,
 537                  preexec_fn=None, close_fds=False, shell=False,
 538                  cwd=None, env=None, universal_newlines=False,
 539                  startupinfo=None, creationflags=0):
 540         """Create new Popen instance."""
 541         _cleanup()
 542
 543         self._child_created = False
 544         if not isinstance(bufsize, (int, long)):
 545             raise TypeError("bufsize must be an integer")
 546
 547         if mswindows:
 548             if preexec_fn is not None:
 549                 raise ValueError("preexec_fn is not supported on Windows "
 550                                  "platforms")
 551             if close_fds and (stdin is not None or stdout is not None or
 552                               stderr is not None):
 553                 raise ValueError("close_fds is not supported on Windows "
 554                                  "platforms if you redirect stdin/stdout/stderr")
 555         else:
 556             # POSIX
 557             if startupinfo is not None:
 558                 raise ValueError("startupinfo is only supported on Windows "
 559                                  "platforms")
 560             if creationflags != 0:
 561                 raise ValueError("creationflags is only supported on Windows "
 562                                  "platforms")
 563
 564         self.stdin = None
 565         self.stdout = None
 566         self.stderr = None
 567         self.pid = None
 568         self.returncode = None
 569         self.universal_newlines = universal_newlines
 570
 571         # Input and output objects. The general principle is like
 572         # this:
 573         #
 574         # Parent                   Child
 575         # ------                   -----
 576         # p2cwrite   ---stdin--->  p2cread
 577         # c2pread    <--stdout---  c2pwrite
 578         # errread    <--stderr---  errwrite
 579         #
 580         # On POSIX, the child objects are file descriptors.  On
 581         # Windows, these are Windows file handles.  The parent objects
 582         # are file descriptors on both platforms.  The parent objects
 583         # are None when not using PIPEs. The child objects are None
 584         # when not redirecting.
 585
 586         (p2cread, p2cwrite,
 587          c2pread, c2pwrite,
 588          errread, errwrite) = self._get_handles(stdin, stdout, stderr)
 589
 590         self._execute_child(args, executable, preexec_fn, close_fds,
 591                             cwd, env, universal_newlines,
 592                             startupinfo, creationflags, shell,
 593                             p2cread, p2cwrite,
 594                             c2pread, c2pwrite,
 595                             errread, errwrite)
 596
 597         # On Windows, you cannot just redirect one or two handles: You
 598         # either have to redirect all three or none. If the subprocess
 599         # user has only redirected one or two handles, we are
 600         # automatically creating PIPEs for the rest. We should close
 601         # these after the process is started. See bug #1124861.
 602         if mswindows:
 603             if stdin is None and p2cwrite is not None:
 604                 os.close(p2cwrite)
 605                 p2cwrite = None
 606             if stdout is None and c2pread is not None:
 607                 os.close(c2pread)
 608                 c2pread = None
 609             if stderr is None and errread is not None:
 610                 os.close(errread)
 611                 errread = None
 612
 613         if p2cwrite is not None:
 614             self.stdin = os.fdopen(p2cwrite, 'wb', bufsize)
 615         if c2pread is not None:
 616             if universal_newlines:
 617                 self.stdout = os.fdopen(c2pread, 'rU', bufsize)
 618             else:
 619                 self.stdout = os.fdopen(c2pread, 'rb', bufsize)
 620         if errread is not None:
 621             if universal_newlines:
 622                 self.stderr = os.fdopen(errread, 'rU', bufsize)
 623             else:
 624                 self.stderr = os.fdopen(errread, 'rb', bufsize)
 625
 626
 627     def _translate_newlines(self, data):
 628         data = data.replace("\r\n", "\n")
 629         data = data.replace("\r", "\n")
 630         return data
 631
 632
 633     def __del__(self, sys=sys):
 634         if not self._child_created:
 635             # We didn't get to successfully create a child process.
 636             return
 637         # In case the child hasn't been waited on, check if it's done.
 638         self.poll(_deadstate=sys.maxint)
 639         if self.returncode is None and _active is not None:
 640             # Child is still running, keep us alive until we can wait on it.
 641             _active.append(self)
 642
 643
 644     def communicate(self, input=None):
 645         """Interact with process: Send data to stdin.  Read data from
 646         stdout and stderr, until end-of-file is reached.  Wait for
 647         process to terminate.  The optional input argument should be a
 648         string to be sent to the child process, or None, if no data
 649         should be sent to the child.
 650
 651         communicate() returns a tuple (stdout, stderr)."""
 652
 653         # Optimization: If we are only using one pipe, or no pipe at
 654         # all, using select() or threads is unnecessary.
 655         if [self.stdin, self.stdout, self.stderr].count(None) >= 2:
 656             stdout = None
 657             stderr = None
 658             if self.stdin:
 659                 if input:
 660                     self.stdin.write(input)
 661                 self.stdin.close()
 662             elif self.stdout:
 663                 stdout = self.stdout.read()
 664             elif self.stderr:
 665                 stderr = self.stderr.read()
 666             self.wait()
 667             return (stdout, stderr)
 668
 669         return self._communicate(input)
 670
 671
 672     if mswindows:
 673         #
 674         # Windows methods
 675         #
 676         def _get_handles(self, stdin, stdout, stderr):
 677             """Construct and return tupel with IO objects:
 678             p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite
 679             """
 680             if stdin is None and stdout is None and stderr is None:
 681                 return (None, None, None, None, None, None)
 682
 683             p2cread, p2cwrite = None, None
 684             c2pread, c2pwrite = None, None
 685             errread, errwrite = None, None
 686
 687             if stdin is None:
 688                 p2cread = GetStdHandle(STD_INPUT_HANDLE)
 689             if p2cread is not None:
 690                 pass
 691             elif stdin is None or stdin == PIPE:
 692                 p2cread, p2cwrite = CreatePipe(None, 0)
 693                 # Detach and turn into fd
 694                 p2cwrite = p2cwrite.Detach()
 695                 p2cwrite = msvcrt.open_osfhandle(p2cwrite, 0)
 696             elif isinstance(stdin, int):
 697                 p2cread = msvcrt.get_osfhandle(stdin)
 698             else:
 699                 # Assuming file-like object
 700                 p2cread = msvcrt.get_osfhandle(stdin.fileno())
 701             p2cread = self._make_inheritable(p2cread)
 702
 703             if stdout is None:
 704                 c2pwrite = GetStdHandle(STD_OUTPUT_HANDLE)
 705             if c2pwrite is not None:
 706                 pass
 707             elif stdout is None or stdout == PIPE:
 708                 c2pread, c2pwrite = CreatePipe(None, 0)
 709                 # Detach and turn into fd
 710                 c2pread = c2pread.Detach()
 711                 c2pread = msvcrt.open_osfhandle(c2pread, 0)
 712             elif isinstance(stdout, int):
 713                 c2pwrite = msvcrt.get_osfhandle(stdout)
 714             else:
 715                 # Assuming file-like object
 716                 c2pwrite = msvcrt.get_osfhandle(stdout.fileno())
 717             c2pwrite = self._make_inheritable(c2pwrite)
 718
 719             if stderr is None:
 720                 errwrite = GetStdHandle(STD_ERROR_HANDLE)
 721             if errwrite is not None:
 722                 pass
 723             elif stderr is None or stderr == PIPE:
 724                 errread, errwrite = CreatePipe(None, 0)
 725                 # Detach and turn into fd
 726                 errread = errread.Detach()
 727                 errread = msvcrt.open_osfhandle(errread, 0)
 728             elif stderr == STDOUT:
 729                 errwrite = c2pwrite
 730             elif isinstance(stderr, int):
 731                 errwrite = msvcrt.get_osfhandle(stderr)
 732             else:
 733                 # Assuming file-like object
 734                 errwrite = msvcrt.get_osfhandle(stderr.fileno())
 735             errwrite = self._make_inheritable(errwrite)
 736
 737             return (p2cread, p2cwrite,
 738                     c2pread, c2pwrite,
 739                     errread, errwrite)
 740
 741
 742         def _make_inheritable(self, handle):
 743             """Return a duplicate of handle, which is inheritable"""
 744             return DuplicateHandle(GetCurrentProcess(), handle,
 745                                    GetCurrentProcess(), 0, 1,
 746                                    DUPLICATE_SAME_ACCESS)
 747
 748
 749         def _find_w9xpopen(self):
 750             """Find and return absolut path to w9xpopen.exe"""
 751             w9xpopen = os.path.join(os.path.dirname(GetModuleFileName(0)),
 752                                     "w9xpopen.exe")
 753             if not os.path.exists(w9xpopen):
 754                 # Eeek - file-not-found - possibly an embedding
 755                 # situation - see if we can locate it in sys.exec_prefix
 756                 w9xpopen = os.path.join(os.path.dirname(sys.exec_prefix),
 757                                         "w9xpopen.exe")
 758                 if not os.path.exists(w9xpopen):
 759                     raise RuntimeError("Cannot locate w9xpopen.exe, which is "
 760                                        "needed for Popen to work with your "
 761                                        "shell or platform.")
 762             return w9xpopen
 763
 764
 765         def _execute_child(self, args, executable, preexec_fn, close_fds,
 766                            cwd, env, universal_newlines,
 767                            startupinfo, creationflags, shell,
 768                            p2cread, p2cwrite,
 769                            c2pread, c2pwrite,
 770                            errread, errwrite):
 771             """Execute program (MS Windows version)"""
 772
 773             if not isinstance(args, types.StringTypes):
 774                 args = list2cmdline(args)
 775
 776             # Process startup details
 777             if startupinfo is None:
 778                 startupinfo = STARTUPINFO()
 779             if None not in (p2cread, c2pwrite, errwrite):
 780                 startupinfo.dwFlags |= STARTF_USESTDHANDLES
 781                 startupinfo.hStdInput = p2cread
 782                 startupinfo.hStdOutput = c2pwrite
 783                 startupinfo.hStdError = errwrite
 784
 785             if shell:
 786                 startupinfo.dwFlags |= STARTF_USESHOWWINDOW
 787                 startupinfo.wShowWindow = SW_HIDE
 788                 comspec = os.environ.get("COMSPEC", "cmd.exe")
 789                 args = comspec + " /c " + args
 790                 if (GetVersion() >= 0x80000000L or
 791                         os.path.basename(comspec).lower() == "command.com"):
 792                     # Win9x, or using command.com on NT. We need to
 793                     # use the w9xpopen intermediate program. For more
 794                     # information, see KB Q150956
 795                     # (http://web.archive.org/web/20011105084002/http://support.microsoft.com/support/kb/articles/Q150/9/56.asp)
 796                     w9xpopen = self._find_w9xpopen()
 797                     args = '"%s" %s' % (w9xpopen, args)
 798                     # Not passing CREATE_NEW_CONSOLE has been known to
 799                     # cause random failures on win9x.  Specifically a
 800                     # dialog: "Your program accessed mem currently in
 801                     # use at xxx" and a hopeful warning about the
 802                     # stability of your system.  Cost is Ctrl+C wont
 803                     # kill children.
 804                     creationflags |= CREATE_NEW_CONSOLE
 805
 806             # Start the process
 807             try:
 808                 hp, ht, pid, tid = CreateProcess(executable, args,
 809                                          # no special security
 810                                          None, None,
 811                                          int(not close_fds),
 812                                          creationflags,
 813                                          env,
 814                                          cwd,
 815                                          startupinfo)
 816             except pywintypes.error, e:
 817                 # Translate pywintypes.error to WindowsError, which is
 818                 # a subclass of OSError.  FIXME: We should really
 819                 # translate errno using _sys_errlist (or simliar), but
 820                 # how can this be done from Python?
 821                 raise WindowsError(*e.args)
 822
 823             # Retain the process handle, but close the thread handle
 824             self._child_created = True
 825             self._handle = hp
 826             self.pid = pid
 827             ht.Close()
 828
 829             # Child is launched. Close the parent's copy of those pipe
 830             # handles that only the child should have open.  You need
 831             # to make sure that no handles to the write end of the
 832             # output pipe are maintained in this process or else the
 833             # pipe will not close when the child process exits and the
 834             # ReadFile will hang.
 835             if p2cread is not None:
 836                 p2cread.Close()
 837             if c2pwrite is not None:
 838                 c2pwrite.Close()
 839             if errwrite is not None:
 840                 errwrite.Close()
 841
 842
 843         def poll(self, _deadstate=None):
 844             """Check if child process has terminated.  Returns returncode
 845             attribute."""
 846             if self.returncode is None:
 847                 if WaitForSingleObject(self._handle, 0) == WAIT_OBJECT_0:
 848                     self.returncode = GetExitCodeProcess(self._handle)
 849             return self.returncode
 850
 851
 852         def wait(self):
 853             """Wait for child process to terminate.  Returns returncode
 854             attribute."""
 855             if self.returncode is None:
 856                 obj = WaitForSingleObject(self._handle, INFINITE)
 857                 self.returncode = GetExitCodeProcess(self._handle)
 858             return self.returncode
 859
 860
 861         def _readerthread(self, fh, buffer):
 862             buffer.append(fh.read())
 863
 864
 865         def _communicate(self, input):
 866             stdout = None # Return
 867             stderr = None # Return
 868
 869             if self.stdout:
 870                 stdout = []
 871                 stdout_thread = threading.Thread(target=self._readerthread,
 872                                                  args=(self.stdout, stdout))
 873                 stdout_thread.setDaemon(True)
 874                 stdout_thread.start()
 875             if self.stderr:
 876                 stderr = []
 877                 stderr_thread = threading.Thread(target=self._readerthread,
 878                                                  args=(self.stderr, stderr))
 879                 stderr_thread.setDaemon(True)
 880                 stderr_thread.start()
 881
 882             if self.stdin:
 883                 if input is not None:
 884                     self.stdin.write(input)
 885                 self.stdin.close()
 886
 887             if self.stdout:
 888                 stdout_thread.join()
 889             if self.stderr:
 890                 stderr_thread.join()
 891
 892             # All data exchanged.  Translate lists into strings.
 893             if stdout is not None:
 894                 stdout = stdout[0]
 895             if stderr is not None:
 896                 stderr = stderr[0]
 897
 898             # Translate newlines, if requested.  We cannot let the file
 899             # object do the translation: It is based on stdio, which is
 900             # impossible to combine with select (unless forcing no
 901             # buffering).
 902             if self.universal_newlines and hasattr(file, 'newlines'):
 903                 if stdout:
 904                     stdout = self._translate_newlines(stdout)
 905                 if stderr:
 906                     stderr = self._translate_newlines(stderr)
 907
 908             self.wait()
 909             return (stdout, stderr)
 910
 911         def send_signal(self, sig):
 912             """Send a signal to the process
 913             """
 914             if sig == signal.SIGTERM:
 915                 self.terminate()
 916             else:
 917                 raise ValueError("Only SIGTERM is supported on Windows")
 918
 919         def terminate(self):
 920             """Terminates the process
 921             """
 922             TerminateProcess(self._handle, 1)
 923
 924         kill = terminate
 925
 926     else:
 927         #
 928         # POSIX methods
 929         #
 930         def _get_handles(self, stdin, stdout, stderr):
 931             """Construct and return tupel with IO objects:
 932             p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite
 933             """
 934             p2cread, p2cwrite = None, None
 935             c2pread, c2pwrite = None, None
 936             errread, errwrite = None, None
 937
 938             if stdin is None:
 939                 pass
 940             elif stdin == PIPE:
 941                 p2cread, p2cwrite = os.pipe()
 942             elif isinstance(stdin, int):
 943                 p2cread = stdin
 944             else:
 945                 # Assuming file-like object
 946                 p2cread = stdin.fileno()
 947
 948             if stdout is None:
 949                 pass
 950             elif stdout == PIPE:
 951                 c2pread, c2pwrite = os.pipe()
 952             elif isinstance(stdout, int):
 953                 c2pwrite = stdout
 954             else:
 955                 # Assuming file-like object
 956                 c2pwrite = stdout.fileno()
 957
 958             if stderr is None:
 959                 pass
 960             elif stderr == PIPE:
 961                 errread, errwrite = os.pipe()
 962             elif stderr == STDOUT:
 963                 errwrite = c2pwrite
 964             elif isinstance(stderr, int):
 965                 errwrite = stderr
 966             else:
 967                 # Assuming file-like object
 968                 errwrite = stderr.fileno()
 969
 970             return (p2cread, p2cwrite,
 971                     c2pread, c2pwrite,
 972                     errread, errwrite)
 973
 974
 975         def _set_cloexec_flag(self, fd):
 976             try:
 977                 cloexec_flag = fcntl.FD_CLOEXEC
 978             except AttributeError:
 979                 cloexec_flag = 1
 980
 981             old = fcntl.fcntl(fd, fcntl.F_GETFD)
 982             fcntl.fcntl(fd, fcntl.F_SETFD, old | cloexec_flag)
 983
 984
 985         def _close_fds(self, but):
 986             os.closerange(3, but)
 987             os.closerange(but + 1, MAXFD)
 988
 989
 990         def _execute_child(self, args, executable, preexec_fn, close_fds,
 991                            cwd, env, universal_newlines,
 992                            startupinfo, creationflags, shell,
 993                            p2cread, p2cwrite,
 994                            c2pread, c2pwrite,
 995                            errread, errwrite):
 996             """Execute program (POSIX version)"""
 997
 998             if isinstance(args, types.StringTypes):
 999                 args = [args]
1000             else:
1001                 args = list(args)
1002
1003             if shell:
1004                 args = ["/bin/sh", "-c"] + args
1005
1006             if executable is None:
1007                 executable = args[0]
1008
1009             # For transferring possible exec failure from child to parent
1010             # The first char specifies the exception type: 0 means
1011             # OSError, 1 means some other error.
1012             errpipe_read, errpipe_write = os.pipe()
1013             self._set_cloexec_flag(errpipe_write)
1014
1015             gc_was_enabled = gc.isenabled()
1016             # Disable gc to avoid bug where gc -> file_dealloc ->
1017             # write to stderr -> hang.  http://bugs.python.org/issue1336
1018             gc.disable()
1019             try:
1020                 self.pid = os.fork()
1021             except:
1022                 if gc_was_enabled:
1023                     gc.enable()
1024                 raise
1025             self._child_created = True
1026             if self.pid == 0:
1027                 # Child
1028                 try:
1029                     # Close parent's pipe ends
1030                     if p2cwrite is not None:
1031                         os.close(p2cwrite)
1032                     if c2pread is not None:
1033                         os.close(c2pread)
1034                     if errread is not None:
1035                         os.close(errread)
1036                     os.close(errpipe_read)
1037
1038                     # Dup fds for child
1039                     if p2cread is not None:
1040                         os.dup2(p2cread, 0)
1041                     if c2pwrite is not None:
1042                         os.dup2(c2pwrite, 1)
1043                     if errwrite is not None:
1044                         os.dup2(errwrite, 2)
1045
1046                     # Close pipe fds.  Make sure we don't close the same
1047                     # fd more than once, or standard fds.
1048                     if p2cread is not None and p2cread not in (0,):
1049                         os.close(p2cread)
1050                     if c2pwrite is not None and c2pwrite not in (p2cread, 1):
1051                         os.close(c2pwrite)
1052                     if errwrite is not None and errwrite not in (p2cread, c2pwrite, 2):
1053                         os.close(errwrite)
1054
1055                     # Close all other fds, if asked for
1056                     if close_fds:
1057                         self._close_fds(but=errpipe_write)
1058
1059                     if cwd is not None:
1060                         os.chdir(cwd)
1061
1062                     if preexec_fn:
1063                         apply(preexec_fn)
1064
1065                     if env is None:
1066                         os.execvp(executable, args)
1067                     else:
1068                         os.execvpe(executable, args, env)
1069
1070                 except:
1071                     exc_type, exc_value, tb = sys.exc_info()
1072                     # Save the traceback and attach it to the exception object
1073                     exc_lines = traceback.format_exception(exc_type,
1074                                                            exc_value,
1075                                                            tb)
1076                     exc_value.child_traceback = ''.join(exc_lines)
1077                     os.write(errpipe_write, pickle.dumps(exc_value))
1078
1079                 # This exitcode won't be reported to applications, so it
1080                 # really doesn't matter what we return.
1081                 os._exit(255)
1082
1083             # Parent
1084             if gc_was_enabled:
1085                 gc.enable()
1086             os.close(errpipe_write)
1087             if p2cread is not None and p2cwrite is not None:
1088                 os.close(p2cread)
1089             if c2pwrite is not None and c2pread is not None:
1090                 os.close(c2pwrite)
1091             if errwrite is not None and errread is not None:
1092                 os.close(errwrite)
1093
1094             # Wait for exec to fail or succeed; possibly raising exception
1095             data = os.read(errpipe_read, 1048576) # Exceptions limited to 1 MB
1096             os.close(errpipe_read)
1097             if data != "":
1098                 os.waitpid(self.pid, 0)
1099                 child_exception = pickle.loads(data)
1100                 raise child_exception
1101
1102
1103         def _handle_exitstatus(self, sts):
1104             if os.WIFSIGNALED(sts):
1105                 self.returncode = -os.WTERMSIG(sts)
1106             elif os.WIFEXITED(sts):
1107                 self.returncode = os.WEXITSTATUS(sts)
1108             else:
1109                 # Should never happen
1110                 raise RuntimeError("Unknown child exit status!")
1111
1112
1113         def poll(self, _deadstate=None):
1114             """Check if child process has terminated.  Returns returncode
1115             attribute."""
1116             if self.returncode is None:
1117                 try:
1118                     pid, sts = os.waitpid(self.pid, os.WNOHANG)
1119                     if pid == self.pid:
1120                         self._handle_exitstatus(sts)
1121                 except os.error:
1122                     if _deadstate is not None:
1123                         self.returncode = _deadstate
1124             return self.returncode
1125
1126
1127         def wait(self):
1128             """Wait for child process to terminate.  Returns returncode
1129             attribute."""
1130             if self.returncode is None:
1131                 pid, sts = os.waitpid(self.pid, 0)
1132                 self._handle_exitstatus(sts)
1133             return self.returncode
1134
1135
1136         def _communicate(self, input):
1137             read_set = []
1138             write_set = []
1139             stdout = None # Return
1140             stderr = None # Return
1141
1142             if self.stdin:
1143                 # Flush stdio buffer.  This might block, if the user has
1144                 # been writing to .stdin in an uncontrolled fashion.
1145                 self.stdin.flush()
1146                 if input:
1147                     write_set.append(self.stdin)
1148                 else:
1149                     self.stdin.close()
1150             if self.stdout:
1151                 read_set.append(self.stdout)
1152                 stdout = []
1153             if self.stderr:
1154                 read_set.append(self.stderr)
1155                 stderr = []
1156
1157             input_offset = 0
1158             while read_set or write_set:
1159                 rlist, wlist, xlist = select.select(read_set, write_set, [])
1160
1161                 if self.stdin in wlist:
1162                     # When select has indicated that the file is writable,
1163                     # we can write up to PIPE_BUF bytes without risk
1164                     # blocking.  POSIX defines PIPE_BUF >= 512
1165                     bytes_written = os.write(self.stdin.fileno(), buffer(input, input_offset, 512))
1166                     input_offset += bytes_written
1167                     if input_offset >= len(input):
1168                         self.stdin.close()
1169                         write_set.remove(self.stdin)
1170
1171                 if self.stdout in rlist:
1172                     data = os.read(self.stdout.fileno(), 1024)
1173                     if data == "":
1174                         self.stdout.close()
1175                         read_set.remove(self.stdout)
1176                     stdout.append(data)
1177
1178                 if self.stderr in rlist:
1179                     data = os.read(self.stderr.fileno(), 1024)
1180                     if data == "":
1181                         self.stderr.close()
1182                         read_set.remove(self.stderr)
1183                     stderr.append(data)
1184
1185             # All data exchanged.  Translate lists into strings.
1186             if stdout is not None:
1187                 stdout = ''.join(stdout)
1188             if stderr is not None:
1189                 stderr = ''.join(stderr)
1190
1191             # Translate newlines, if requested.  We cannot let the file
1192             # object do the translation: It is based on stdio, which is
1193             # impossible to combine with select (unless forcing no
1194             # buffering).
1195             if self.universal_newlines and hasattr(file, 'newlines'):
1196                 if stdout:
1197                     stdout = self._translate_newlines(stdout)
1198                 if stderr:
1199                     stderr = self._translate_newlines(stderr)
1200
1201             self.wait()
1202             return (stdout, stderr)
1203
1204         def send_signal(self, sig):
1205             """Send a signal to the process
1206             """
1207             os.kill(self.pid, sig)
1208
1209         def terminate(self):
1210             """Terminate the process with SIGTERM
1211             """
1212             self.send_signal(signal.SIGTERM)
1213
1214         def kill(self):
1215             """Kill the process with SIGKILL
1216             """
1217             self.send_signal(signal.SIGKILL)
1218
1219
1220 def _demo_posix():
1221     #
1222     # Example 1: Simple redirection: Get process list
1223     #
1224     plist = Popen(["ps"], stdout=PIPE).communicate()[0]
1225     print "Process list:"
1226     print plist
1227
1228     #
1229     # Example 2: Change uid before executing child
1230     #
1231     if os.getuid() == 0:
1232         p = Popen(["id"], preexec_fn=lambda: os.setuid(100))
1233         p.wait()
1234
1235     #
1236     # Example 3: Connecting several subprocesses
1237     #
1238     print "Looking for 'hda'..."
1239     p1 = Popen(["dmesg"], stdout=PIPE)
1240     p2 = Popen(["grep", "hda"], stdin=p1.stdout, stdout=PIPE)
1241     print repr(p2.communicate()[0])
1242
1243     #
1244     # Example 4: Catch execution error
1245     #
1246     print
1247     print "Trying a weird file..."
1248     try:
1249         print Popen(["/this/path/does/not/exist"]).communicate()
1250     except OSError, e:
1251         if e.errno == errno.ENOENT:
1252             print "The file didn't exist.  I thought so..."
1253             print "Child traceback:"
1254             print e.child_traceback
1255         else:
1256             print "Error", e.errno
1257     else:
1258         print >>sys.stderr, "Gosh.  No error."
1259
1260
1261 def _demo_windows():
1262     #
1263     # Example 1: Connecting several subprocesses
1264     #
1265     print "Looking for 'PROMPT' in set output..."
1266     p1 = Popen("set", stdout=PIPE, shell=True)
1267     p2 = Popen('find "PROMPT"', stdin=p1.stdout, stdout=PIPE)
1268     print repr(p2.communicate()[0])
1269
1270     #
1271     # Example 2: Simple execution of program
1272     #
1273     print "Executing calc..."
1274     p = Popen("calc")
1275     p.wait()
1276
1277
1278 if __name__ == "__main__":
1279     if mswindows:
1280         _demo_windows()
1281     else:
1282         _demo_posix()