Lib/subprocess.py

   1 # subprocess - Subprocesses with accessible I/O streams
   2 #
   3 # For more information about this module, see PEP 324.
   4 #
   5 # This module should remain compatible with Python 2.2, see PEP 291.
   6 #
   7 # Copyright (c) 2003-2005 by Peter Astrand <astrand@lysator.liu.se>
   8 #
   9 # Licensed to PSF under a Contributor Agreement.
  10 # See http://www.python.org/2.4/license for licensing details.
  11
  12 r"""subprocess - Subprocesses with accessible I/O streams
  13
  14 This module allows you to spawn processes, connect to their
  15 input/output/error pipes, and obtain their return codes.  This module
  16 intends to replace several other, older modules and functions, like:
  17
  18 os.system
  19 os.spawn*
  20 os.popen*
  21 popen2.*
  22 commands.*
  23
  24 Information about how the subprocess module can be used to replace these
  25 modules and functions can be found below.
  26
  27
  28
  29 Using the subprocess module
  30 ===========================
  31 This module defines one class called Popen:
  32
  33 class Popen(args, bufsize=0, executable=None,
  34             stdin=None, stdout=None, stderr=None,
  35             preexec_fn=None, close_fds=False, shell=False,
  36             cwd=None, env=None, universal_newlines=False,
  37             startupinfo=None, creationflags=0):
  38
  39
  40 Arguments are:
  41
  42 args should be a string, or a sequence of program arguments.  The
  43 program to execute is normally the first item in the args sequence or
  44 string, but can be explicitly set by using the executable argument.
  45
  46 On UNIX, with shell=False (default): In this case, the Popen class
  47 uses os.execvp() to execute the child program.  args should normally
  48 be a sequence.  A string will be treated as a sequence with the string
  49 as the only item (the program to execute).
  50
  51 On UNIX, with shell=True: If args is a string, it specifies the
  52 command string to execute through the shell.  If args is a sequence,
  53 the first item specifies the command string, and any additional items
  54 will be treated as additional shell arguments.
  55
  56 On Windows: the Popen class uses CreateProcess() to execute the child
  57 program, which operates on strings.  If args is a sequence, it will be
  58 converted to a string using the list2cmdline method.  Please note that
  59 not all MS Windows applications interpret the command line the same
  60 way: The list2cmdline is designed for applications using the same
  61 rules as the MS C runtime.
  62
  63 bufsize, if given, has the same meaning as the corresponding argument
  64 to the built-in open() function: 0 means unbuffered, 1 means line
  65 buffered, any other positive value means use a buffer of
  66 (approximately) that size.  A negative bufsize means to use the system
  67 default, which usually means fully buffered.  The default value for
  68 bufsize is 0 (unbuffered).
  69
  70 stdin, stdout and stderr specify the executed programs' standard
  71 input, standard output and standard error file handles, respectively.
  72 Valid values are PIPE, an existing file descriptor (a positive
  73 integer), an existing file object, and None.  PIPE indicates that a
  74 new pipe to the child should be created.  With None, no redirection
  75 will occur; the child's file handles will be inherited from the
  76 parent.  Additionally, stderr can be STDOUT, which indicates that the
  77 stderr data from the applications should be captured into the same
  78 file handle as for stdout.
  79
  80 If preexec_fn is set to a callable object, this object will be called
  81 in the child process just before the child is executed.
  82
  83 If close_fds is true, all file descriptors except 0, 1 and 2 will be
  84 closed before the child process is executed.
  85
  86 if shell is true, the specified command will be executed through the
  87 shell.
  88
  89 If cwd is not None, the current directory will be changed to cwd
  90 before the child is executed.
  91
  92 If env is not None, it defines the environment variables for the new
  93 process.
  94
  95 If universal_newlines is true, the file objects stdout and stderr are
  96 opened as a text files, but lines may be terminated by any of '\n',
  97 the Unix end-of-line convention, '\r', the Macintosh convention or
  98 '\r\n', the Windows convention.  All of these external representations
  99 are seen as '\n' by the Python program.  Note: This feature is only
 100 available if Python is built with universal newline support (the
 101 default).  Also, the newlines attribute of the file objects stdout,
 102 stdin and stderr are not updated by the communicate() method.
 103
 104 The startupinfo and creationflags, if given, will be passed to the
 105 underlying CreateProcess() function.  They can specify things such as
 106 appearance of the main window and priority for the new process.
 107 (Windows only)
 108
 109
 110 This module also defines some shortcut functions:
 111
 112 call(*popenargs, **kwargs):
 113     Run command with arguments.  Wait for command to complete, then
 114     return the returncode attribute.
 115
 116     The arguments are the same as for the Popen constructor.  Example:
 117
 118     retcode = call(["ls", "-l"])
 119
 120 check_call(*popenargs, **kwargs):
 121     Run command with arguments.  Wait for command to complete.  If the
 122     exit code was zero then return, otherwise raise
 123     CalledProcessError.  The CalledProcessError object will have the
 124     return code in the returncode attribute.
 125
 126     The arguments are the same as for the Popen constructor.  Example:
 127
 128     check_call(["ls", "-l"])
 129
 130 check_output(*popenargs, **kwargs):
 131    Run command with arguments and return its output as a byte string.
 132
 133    If the exit code was non-zero it raises a CalledProcessError.  The
 134    CalledProcessError object will have the return code in the returncode
 135    attribute and output in the output attribute.
 136
 137    The arguments are the same as for the Popen constructor.  Example:
 138
 139       output = subprocess.check_output(["ls", "-l", "/dev/null"])
 140
 141 Exceptions
 142 ----------
 143 Exceptions raised in the child process, before the new program has
 144 started to execute, will be re-raised in the parent.  Additionally,
 145 the exception object will have one extra attribute called
 146 'child_traceback', which is a string containing traceback information
 147 from the childs point of view.
 148
 149 The most common exception raised is OSError.  This occurs, for
 150 example, when trying to execute a non-existent file.  Applications
 151 should prepare for OSErrors.
 152
 153 A ValueError will be raised if Popen is called with invalid arguments.
 154
 155 check_call() and check_output() will raise CalledProcessError, if the
 156 called process returns a non-zero return code.
 157
 158
 159 Security
 160 --------
 161 Unlike some other popen functions, this implementation will never call
 162 /bin/sh implicitly.  This means that all characters, including shell
 163 metacharacters, can safely be passed to child processes.
 164
 165
 166 Popen objects
 167 =============
 168 Instances of the Popen class have the following methods:
 169
 170 poll()
 171     Check if child process has terminated.  Returns returncode
 172     attribute.
 173
 174 wait()
 175     Wait for child process to terminate.  Returns returncode attribute.
 176
 177 communicate(input=None)
 178     Interact with process: Send data to stdin.  Read data from stdout
 179     and stderr, until end-of-file is reached.  Wait for process to
 180     terminate.  The optional input argument should be a string to be
 181     sent to the child process, or None, if no data should be sent to
 182     the child.
 183
 184     communicate() returns a tuple (stdout, stderr).
 185
 186     Note: The data read is buffered in memory, so do not use this
 187     method if the data size is large or unlimited.
 188
 189 The following attributes are also available:
 190
 191 stdin
 192     If the stdin argument is PIPE, this attribute is a file object
 193     that provides input to the child process.  Otherwise, it is None.
 194
 195 stdout
 196     If the stdout argument is PIPE, this attribute is a file object
 197     that provides output from the child process.  Otherwise, it is
 198     None.
 199
 200 stderr
 201     If the stderr argument is PIPE, this attribute is file object that
 202     provides error output from the child process.  Otherwise, it is
 203     None.
 204
 205 pid
 206     The process ID of the child process.
 207
 208 returncode
 209     The child return code.  A None value indicates that the process
 210     hasn't terminated yet.  A negative value -N indicates that the
 211     child was terminated by signal N (UNIX only).
 212
 213
 214 Replacing older functions with the subprocess module
 215 ====================================================
 216 In this section, "a ==> b" means that b can be used as a replacement
 217 for a.
 218
 219 Note: All functions in this section fail (more or less) silently if
 220 the executed program cannot be found; this module raises an OSError
 221 exception.
 222
 223 In the following examples, we assume that the subprocess module is
 224 imported with "from subprocess import *".
 225
 226
 227 Replacing /bin/sh shell backquote
 228 ---------------------------------
 229 output=`mycmd myarg`
 230 ==>
 231 output = Popen(["mycmd", "myarg"], stdout=PIPE).communicate()[0]
 232
 233
 234 Replacing shell pipe line
 235 -------------------------
 236 output=`dmesg | grep hda`
 237 ==>
 238 p1 = Popen(["dmesg"], stdout=PIPE)
 239 p2 = Popen(["grep", "hda"], stdin=p1.stdout, stdout=PIPE)
 240 output = p2.communicate()[0]
 241
 242
 243 Replacing os.system()
 244 ---------------------
 245 sts = os.system("mycmd" + " myarg")
 246 ==>
 247 p = Popen("mycmd" + " myarg", shell=True)
 248 pid, sts = os.waitpid(p.pid, 0)
 249
 250 Note:
 251
 252 * Calling the program through the shell is usually not required.
 253
 254 * It's easier to look at the returncode attribute than the
 255   exitstatus.
 256
 257 A more real-world example would look like this:
 258
 259 try:
 260     retcode = call("mycmd" + " myarg", shell=True)
 261     if retcode < 0:
 262         print >>sys.stderr, "Child was terminated by signal", -retcode
 263     else:
 264         print >>sys.stderr, "Child returned", retcode
 265 except OSError, e:
 266     print >>sys.stderr, "Execution failed:", e
 267
 268
 269 Replacing os.spawn*
 270 -------------------
 271 P_NOWAIT example:
 272
 273 pid = os.spawnlp(os.P_NOWAIT, "/bin/mycmd", "mycmd", "myarg")
 274 ==>
 275 pid = Popen(["/bin/mycmd", "myarg"]).pid
 276
 277
 278 P_WAIT example:
 279
 280 retcode = os.spawnlp(os.P_WAIT, "/bin/mycmd", "mycmd", "myarg")
 281 ==>
 282 retcode = call(["/bin/mycmd", "myarg"])
 283
 284
 285 Vector example:
 286
 287 os.spawnvp(os.P_NOWAIT, path, args)
 288 ==>
 289 Popen([path] + args[1:])
 290
 291
 292 Environment example:
 293
 294 os.spawnlpe(os.P_NOWAIT, "/bin/mycmd", "mycmd", "myarg", env)
 295 ==>
 296 Popen(["/bin/mycmd", "myarg"], env={"PATH": "/usr/bin"})
 297
 298
 299 Replacing os.popen*
 300 -------------------
 301 pipe = os.popen(cmd, mode='r', bufsize)
 302 ==>
 303 pipe = Popen(cmd, shell=True, bufsize=bufsize, stdout=PIPE).stdout
 304
 305 pipe = os.popen(cmd, mode='w', bufsize)
 306 ==>
 307 pipe = Popen(cmd, shell=True, bufsize=bufsize, stdin=PIPE).stdin
 308
 309
 310 (child_stdin, child_stdout) = os.popen2(cmd, mode, bufsize)
 311 ==>
 312 p = Popen(cmd, shell=True, bufsize=bufsize,
 313           stdin=PIPE, stdout=PIPE, close_fds=True)
 314 (child_stdin, child_stdout) = (p.stdin, p.stdout)
 315
 316
 317 (child_stdin,
 318  child_stdout,
 319  child_stderr) = os.popen3(cmd, mode, bufsize)
 320 ==>
 321 p = Popen(cmd, shell=True, bufsize=bufsize,
 322           stdin=PIPE, stdout=PIPE, stderr=PIPE, close_fds=True)
 323 (child_stdin,
 324  child_stdout,
 325  child_stderr) = (p.stdin, p.stdout, p.stderr)
 326
 327
 328 (child_stdin, child_stdout_and_stderr) = os.popen4(cmd, mode, bufsize)
 329 ==>
 330 p = Popen(cmd, shell=True, bufsize=bufsize,
 331           stdin=PIPE, stdout=PIPE, stderr=STDOUT, close_fds=True)
 332 (child_stdin, child_stdout_and_stderr) = (p.stdin, p.stdout)
 333
 334
 335 Replacing popen2.*
 336 ------------------
 337 Note: If the cmd argument to popen2 functions is a string, the command
 338 is executed through /bin/sh.  If it is a list, the command is directly
 339 executed.
 340
 341 (child_stdout, child_stdin) = popen2.popen2("somestring", bufsize, mode)
 342 ==>
 343 p = Popen(["somestring"], shell=True, bufsize=bufsize
 344           stdin=PIPE, stdout=PIPE, close_fds=True)
 345 (child_stdout, child_stdin) = (p.stdout, p.stdin)
 346
 347
 348 (child_stdout, child_stdin) = popen2.popen2(["mycmd", "myarg"], bufsize, mode)
 349 ==>
 350 p = Popen(["mycmd", "myarg"], bufsize=bufsize,
 351           stdin=PIPE, stdout=PIPE, close_fds=True)
 352 (child_stdout, child_stdin) = (p.stdout, p.stdin)
 353
 354 The popen2.Popen3 and popen2.Popen4 basically works as subprocess.Popen,
 355 except that:
 356
 357 * subprocess.Popen raises an exception if the execution fails
 358 * the capturestderr argument is replaced with the stderr argument.
 359 * stdin=PIPE and stdout=PIPE must be specified.
 360 * popen2 closes all filedescriptors by default, but you have to specify
 361   close_fds=True with subprocess.Popen.
 362 """
 363
 364 import sys
 365 mswindows = (sys.platform == "win32")
 366
 367 import os
 368 import types
 369 import traceback
 370 import gc
 371 import signal
 372
 373 # Exception classes used by this module.
 374 class CalledProcessError(Exception):
 375     """This exception is raised when a process run by check_call() or
 376     check_output() returns a non-zero exit status.
 377     The exit status will be stored in the returncode attribute;
 378     check_output() will also store the output in the output attribute.
 379     """
 380     def __init__(self, returncode, cmd, output=None):
 381         self.returncode = returncode
 382         self.cmd = cmd
 383         self.output = output
 384     def __str__(self):
 385         return "Command '%s' returned non-zero exit status %d" % (self.cmd, self.returncode)
 386
 387
 388 if mswindows:
 389     import threading
 390     import msvcrt
 391     if 0: # <-- change this to use pywin32 instead of the _subprocess driver
 392         import pywintypes
 393         from win32api import GetStdHandle, STD_INPUT_HANDLE, \
 394                              STD_OUTPUT_HANDLE, STD_ERROR_HANDLE
 395         from win32api import GetCurrentProcess, DuplicateHandle, \
 396                              GetModuleFileName, GetVersion
 397         from win32con import DUPLICATE_SAME_ACCESS, SW_HIDE
 398         from win32pipe import CreatePipe
 399         from win32process import CreateProcess, STARTUPINFO, \
 400                                  GetExitCodeProcess, STARTF_USESTDHANDLES, \
 401                                  STARTF_USESHOWWINDOW, CREATE_NEW_CONSOLE
 402         from win32process import TerminateProcess
 403         from win32event import WaitForSingleObject, INFINITE, WAIT_OBJECT_0
 404     else:
 405         from _subprocess import *
 406         class STARTUPINFO:
 407             dwFlags = 0
 408             hStdInput = None
 409             hStdOutput = None
 410             hStdError = None
 411             wShowWindow = 0
 412         class pywintypes:
 413             error = IOError
 414 else:
 415     import select
 416     import errno
 417     import fcntl
 418     import pickle
 419
 420 __all__ = ["Popen", "PIPE", "STDOUT", "call", "check_call",
 421            "check_output", "CalledProcessError"]
 422
 423 try:
 424     MAXFD = os.sysconf("SC_OPEN_MAX")
 425 except:
 426     MAXFD = 256
 427
 428 # True/False does not exist on 2.2.0
 429 #try:
 430 #    False
 431 #except NameError:
 432 #    False = 0
 433 #    True = 1
 434
 435 _active = []
 436
 437 def _cleanup():
 438     for inst in _active[:]:
 439         if inst._internal_poll(_deadstate=sys.maxint) >= 0:
 440             try:
 441                 _active.remove(inst)
 442             except ValueError:
 443                 # This can happen if two threads create a new Popen instance.
 444                 # It's harmless that it was already removed, so ignore.
 445                 pass
 446
 447 PIPE = -1
 448 STDOUT = -2
 449
 450
 451 def call(*popenargs, **kwargs):
 452     """Run command with arguments.  Wait for command to complete, then
 453     return the returncode attribute.
 454
 455     The arguments are the same as for the Popen constructor.  Example:
 456
 457     retcode = call(["ls", "-l"])
 458     """
 459     return Popen(*popenargs, **kwargs).wait()
 460
 461
 462 def check_call(*popenargs, **kwargs):
 463     """Run command with arguments.  Wait for command to complete.  If
 464     the exit code was zero then return, otherwise raise
 465     CalledProcessError.  The CalledProcessError object will have the
 466     return code in the returncode attribute.
 467
 468     The arguments are the same as for the Popen constructor.  Example:
 469
 470     check_call(["ls", "-l"])
 471     """
 472     retcode = call(*popenargs, **kwargs)
 473     if retcode:
 474         cmd = kwargs.get("args")
 475         if cmd is None:
 476             cmd = popenargs[0]
 477         raise CalledProcessError(retcode, cmd)
 478     return 0
 479
 480
 481 def check_output(*popenargs, **kwargs):
 482     """Run command with arguments and return its output as a byte string.
 483
 484     If the exit code was non-zero it raises a CalledProcessError.  The
 485     CalledProcessError object will have the return code in the returncode
 486     attribute and output in the output attribute.
 487
 488     The arguments are the same as for the Popen constructor.  Example:
 489
 490     >>> check_output(["ls", "-l", "/dev/null"])
 491     'crw-rw-rw- 1 root root 1, 3 Oct 18  2007 /dev/null\n'
 492
 493     The stdout argument is not allowed as it is used internally.
 494     To capture standard error in the result, use stderr=subprocess.STDOUT.
 495
 496     >>> check_output(["/bin/sh", "-c",
 497                       "ls -l non_existant_file ; exit 0"],
 498                      stderr=subprocess.STDOUT)
 499     'ls: non_existant_file: No such file or directory\n'
 500     """
 501     if 'stdout' in kwargs:
 502         raise ValueError('stdout argument not allowed, it will be overridden.')
 503     process = Popen(*popenargs, stdout=PIPE, **kwargs)
 504     output, unused_err = process.communicate()
 505     retcode = process.poll()
 506     if retcode:
 507         cmd = kwargs.get("args")
 508         if cmd is None:
 509             cmd = popenargs[0]
 510         raise CalledProcessError(retcode, cmd, output=output)
 511     return output
 512
 513
 514 def list2cmdline(seq):
 515     """
 516     Translate a sequence of arguments into a command line
 517     string, using the same rules as the MS C runtime:
 518
 519     1) Arguments are delimited by white space, which is either a
 520        space or a tab.
 521
 522     2) A string surrounded by double quotation marks is
 523        interpreted as a single argument, regardless of white space
 524        or pipe characters contained within.  A quoted string can be
 525        embedded in an argument.
 526
 527     3) A double quotation mark preceded by a backslash is
 528        interpreted as a literal double quotation mark.
 529
 530     4) Backslashes are interpreted literally, unless they
 531        immediately precede a double quotation mark.
 532
 533     5) If backslashes immediately precede a double quotation mark,
 534        every pair of backslashes is interpreted as a literal
 535        backslash.  If the number of backslashes is odd, the last
 536        backslash escapes the next double quotation mark as
 537        described in rule 3.
 538     """
 539
 540     # See
 541     # http://msdn.microsoft.com/library/en-us/vccelng/htm/progs_12.asp
 542     result = []
 543     needquote = False
 544     for arg in seq:
 545         bs_buf = []
 546
 547         # Add a space to separate this argument from the others
 548         if result:
 549             result.append(' ')
 550
 551         needquote = (" " in arg) or ("\t" in arg) or ("|" in arg) or not arg
 552         if needquote:
 553             result.append('"')
 554
 555         for c in arg:
 556             if c == '\\':
 557                 # Don't know if we need to double yet.
 558                 bs_buf.append(c)
 559             elif c == '"':
 560                 # Double backslashes.
 561                 result.append('\\' * len(bs_buf)*2)
 562                 bs_buf = []
 563                 result.append('\\"')
 564             else:
 565                 # Normal char
 566                 if bs_buf:
 567                     result.extend(bs_buf)
 568                     bs_buf = []
 569                 result.append(c)
 570
 571         # Add remaining backslashes, if any.
 572         if bs_buf:
 573             result.extend(bs_buf)
 574
 575         if needquote:
 576             result.extend(bs_buf)
 577             result.append('"')
 578
 579     return ''.join(result)
 580
 581
 582 class Popen(object):
 583     def __init__(self, args, bufsize=0, executable=None,
 584                  stdin=None, stdout=None, stderr=None,
 585                  preexec_fn=None, close_fds=False, shell=False,
 586                  cwd=None, env=None, universal_newlines=False,
 587                  startupinfo=None, creationflags=0):
 588         """Create new Popen instance."""
 589         _cleanup()
 590
 591         self._child_created = False
 592         if not isinstance(bufsize, (int, long)):
 593             raise TypeError("bufsize must be an integer")
 594
 595         if mswindows:
 596             if preexec_fn is not None:
 597                 raise ValueError("preexec_fn is not supported on Windows "
 598                                  "platforms")
 599             if close_fds and (stdin is not None or stdout is not None or
 600                               stderr is not None):
 601                 raise ValueError("close_fds is not supported on Windows "
 602                                  "platforms if you redirect stdin/stdout/stderr")
 603         else:
 604             # POSIX
 605             if startupinfo is not None:
 606                 raise ValueError("startupinfo is only supported on Windows "
 607                                  "platforms")
 608             if creationflags != 0:
 609                 raise ValueError("creationflags is only supported on Windows "
 610                                  "platforms")
 611
 612         self.stdin = None
 613         self.stdout = None
 614         self.stderr = None
 615         self.pid = None
 616         self.returncode = None
 617         self.universal_newlines = universal_newlines
 618
 619         # Input and output objects. The general principle is like
 620         # this:
 621         #
 622         # Parent                   Child
 623         # ------                   -----
 624         # p2cwrite   ---stdin--->  p2cread
 625         # c2pread    <--stdout---  c2pwrite
 626         # errread    <--stderr---  errwrite
 627         #
 628         # On POSIX, the child objects are file descriptors.  On
 629         # Windows, these are Windows file handles.  The parent objects
 630         # are file descriptors on both platforms.  The parent objects
 631         # are None when not using PIPEs. The child objects are None
 632         # when not redirecting.
 633
 634         (p2cread, p2cwrite,
 635          c2pread, c2pwrite,
 636          errread, errwrite) = self._get_handles(stdin, stdout, stderr)
 637
 638         self._execute_child(args, executable, preexec_fn, close_fds,
 639                             cwd, env, universal_newlines,
 640                             startupinfo, creationflags, shell,
 641                             p2cread, p2cwrite,
 642                             c2pread, c2pwrite,
 643                             errread, errwrite)
 644
 645         # On Windows, you cannot just redirect one or two handles: You
 646         # either have to redirect all three or none. If the subprocess
 647         # user has only redirected one or two handles, we are
 648         # automatically creating PIPEs for the rest. We should close
 649         # these after the process is started. See bug #1124861.
 650         if mswindows:
 651             if stdin is None and p2cwrite is not None:
 652                 os.close(p2cwrite)
 653                 p2cwrite = None
 654             if stdout is None and c2pread is not None:
 655                 os.close(c2pread)
 656                 c2pread = None
 657             if stderr is None and errread is not None:
 658                 os.close(errread)
 659                 errread = None
 660
 661         if p2cwrite is not None:
 662             self.stdin = os.fdopen(p2cwrite, 'wb', bufsize)
 663         if c2pread is not None:
 664             if universal_newlines:
 665                 self.stdout = os.fdopen(c2pread, 'rU', bufsize)
 666             else:
 667                 self.stdout = os.fdopen(c2pread, 'rb', bufsize)
 668         if errread is not None:
 669             if universal_newlines:
 670                 self.stderr = os.fdopen(errread, 'rU', bufsize)
 671             else:
 672                 self.stderr = os.fdopen(errread, 'rb', bufsize)
 673
 674
 675     def _translate_newlines(self, data):
 676         data = data.replace("\r\n", "\n")
 677         data = data.replace("\r", "\n")
 678         return data
 679
 680
 681     def __del__(self, sys=sys):
 682         if not self._child_created:
 683             # We didn't get to successfully create a child process.
 684             return
 685         # In case the child hasn't been waited on, check if it's done.
 686         self._internal_poll(_deadstate=sys.maxint)
 687         if self.returncode is None and _active is not None:
 688             # Child is still running, keep us alive until we can wait on it.
 689             _active.append(self)
 690
 691
 692     def communicate(self, input=None):
 693         """Interact with process: Send data to stdin.  Read data from
 694         stdout and stderr, until end-of-file is reached.  Wait for
 695         process to terminate.  The optional input argument should be a
 696         string to be sent to the child process, or None, if no data
 697         should be sent to the child.
 698
 699         communicate() returns a tuple (stdout, stderr)."""
 700
 701         # Optimization: If we are only using one pipe, or no pipe at
 702         # all, using select() or threads is unnecessary.
 703         if [self.stdin, self.stdout, self.stderr].count(None) >= 2:
 704             stdout = None
 705             stderr = None
 706             if self.stdin:
 707                 if input:
 708                     self.stdin.write(input)
 709                 self.stdin.close()
 710             elif self.stdout:
 711                 stdout = self.stdout.read()
 712                 self.stdout.close()
 713             elif self.stderr:
 714                 stderr = self.stderr.read()
 715                 self.stderr.close()
 716             self.wait()
 717             return (stdout, stderr)
 718
 719         return self._communicate(input)
 720
 721
 722     def poll(self):
 723         return self._internal_poll()
 724
 725
 726     if mswindows:
 727         #
 728         # Windows methods
 729         #
 730         def _get_handles(self, stdin, stdout, stderr):
 731             """Construct and return tupel with IO objects:
 732             p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite
 733             """
 734             if stdin is None and stdout is None and stderr is None:
 735                 return (None, None, None, None, None, None)
 736
 737             p2cread, p2cwrite = None, None
 738             c2pread, c2pwrite = None, None
 739             errread, errwrite = None, None
 740
 741             if stdin is None:
 742                 p2cread = GetStdHandle(STD_INPUT_HANDLE)
 743             if p2cread is not None:
 744                 pass
 745             elif stdin is None or stdin == PIPE:
 746                 p2cread, p2cwrite = CreatePipe(None, 0)
 747                 # Detach and turn into fd
 748                 p2cwrite = p2cwrite.Detach()
 749                 p2cwrite = msvcrt.open_osfhandle(p2cwrite, 0)
 750             elif isinstance(stdin, int):
 751                 p2cread = msvcrt.get_osfhandle(stdin)
 752             else:
 753                 # Assuming file-like object
 754                 p2cread = msvcrt.get_osfhandle(stdin.fileno())
 755             p2cread = self._make_inheritable(p2cread)
 756
 757             if stdout is None:
 758                 c2pwrite = GetStdHandle(STD_OUTPUT_HANDLE)
 759             if c2pwrite is not None:
 760                 pass
 761             elif stdout is None or stdout == PIPE:
 762                 c2pread, c2pwrite = CreatePipe(None, 0)
 763                 # Detach and turn into fd
 764                 c2pread = c2pread.Detach()
 765                 c2pread = msvcrt.open_osfhandle(c2pread, 0)
 766             elif isinstance(stdout, int):
 767                 c2pwrite = msvcrt.get_osfhandle(stdout)
 768             else:
 769                 # Assuming file-like object
 770                 c2pwrite = msvcrt.get_osfhandle(stdout.fileno())
 771             c2pwrite = self._make_inheritable(c2pwrite)
 772
 773             if stderr is None:
 774                 errwrite = GetStdHandle(STD_ERROR_HANDLE)
 775             if errwrite is not None:
 776                 pass
 777             elif stderr is None or stderr == PIPE:
 778                 errread, errwrite = CreatePipe(None, 0)
 779                 # Detach and turn into fd
 780                 errread = errread.Detach()
 781                 errread = msvcrt.open_osfhandle(errread, 0)
 782             elif stderr == STDOUT:
 783                 errwrite = c2pwrite
 784             elif isinstance(stderr, int):
 785                 errwrite = msvcrt.get_osfhandle(stderr)
 786             else:
 787                 # Assuming file-like object
 788                 errwrite = msvcrt.get_osfhandle(stderr.fileno())
 789             errwrite = self._make_inheritable(errwrite)
 790
 791             return (p2cread, p2cwrite,
 792                     c2pread, c2pwrite,
 793                     errread, errwrite)
 794
 795
 796         def _make_inheritable(self, handle):
 797             """Return a duplicate of handle, which is inheritable"""
 798             return DuplicateHandle(GetCurrentProcess(), handle,
 799                                    GetCurrentProcess(), 0, 1,
 800                                    DUPLICATE_SAME_ACCESS)
 801
 802
 803         def _find_w9xpopen(self):
 804             """Find and return absolut path to w9xpopen.exe"""
 805             w9xpopen = os.path.join(os.path.dirname(GetModuleFileName(0)),
 806                                     "w9xpopen.exe")
 807             if not os.path.exists(w9xpopen):
 808                 # Eeek - file-not-found - possibly an embedding
 809                 # situation - see if we can locate it in sys.exec_prefix
 810                 w9xpopen = os.path.join(os.path.dirname(sys.exec_prefix),
 811                                         "w9xpopen.exe")
 812                 if not os.path.exists(w9xpopen):
 813                     raise RuntimeError("Cannot locate w9xpopen.exe, which is "
 814                                        "needed for Popen to work with your "
 815                                        "shell or platform.")
 816             return w9xpopen
 817
 818
 819         def _execute_child(self, args, executable, preexec_fn, close_fds,
 820                            cwd, env, universal_newlines,
 821                            startupinfo, creationflags, shell,
 822                            p2cread, p2cwrite,
 823                            c2pread, c2pwrite,
 824                            errread, errwrite):
 825             """Execute program (MS Windows version)"""
 826
 827             if not isinstance(args, types.StringTypes):
 828                 args = list2cmdline(args)
 829
 830             # Process startup details
 831             if startupinfo is None:
 832                 startupinfo = STARTUPINFO()
 833             if None not in (p2cread, c2pwrite, errwrite):
 834                 startupinfo.dwFlags |= STARTF_USESTDHANDLES
 835                 startupinfo.hStdInput = p2cread
 836                 startupinfo.hStdOutput = c2pwrite
 837                 startupinfo.hStdError = errwrite
 838
 839             if shell:
 840                 startupinfo.dwFlags |= STARTF_USESHOWWINDOW
 841                 startupinfo.wShowWindow = SW_HIDE
 842                 comspec = os.environ.get("COMSPEC", "cmd.exe")
 843                 args = comspec + " /c " + args
 844                 if (GetVersion() >= 0x80000000L or
 845                         os.path.basename(comspec).lower() == "command.com"):
 846                     # Win9x, or using command.com on NT. We need to
 847                     # use the w9xpopen intermediate program. For more
 848                     # information, see KB Q150956
 849                     # (http://web.archive.org/web/20011105084002/http://support.microsoft.com/support/kb/articles/Q150/9/56.asp)
 850                     w9xpopen = self._find_w9xpopen()
 851                     args = '"%s" %s' % (w9xpopen, args)
 852                     # Not passing CREATE_NEW_CONSOLE has been known to
 853                     # cause random failures on win9x.  Specifically a
 854                     # dialog: "Your program accessed mem currently in
 855                     # use at xxx" and a hopeful warning about the
 856                     # stability of your system.  Cost is Ctrl+C wont
 857                     # kill children.
 858                     creationflags |= CREATE_NEW_CONSOLE
 859
 860             # Start the process
 861             try:
 862                 hp, ht, pid, tid = CreateProcess(executable, args,
 863                                          # no special security
 864                                          None, None,
 865                                          int(not close_fds),
 866                                          creationflags,
 867                                          env,
 868                                          cwd,
 869                                          startupinfo)
 870             except pywintypes.error, e:
 871                 # Translate pywintypes.error to WindowsError, which is
 872                 # a subclass of OSError.  FIXME: We should really
 873                 # translate errno using _sys_errlist (or simliar), but
 874                 # how can this be done from Python?
 875                 raise WindowsError(*e.args)
 876
 877             # Retain the process handle, but close the thread handle
 878             self._child_created = True
 879             self._handle = hp
 880             self.pid = pid
 881             ht.Close()
 882
 883             # Child is launched. Close the parent's copy of those pipe
 884             # handles that only the child should have open.  You need
 885             # to make sure that no handles to the write end of the
 886             # output pipe are maintained in this process or else the
 887             # pipe will not close when the child process exits and the
 888             # ReadFile will hang.
 889             if p2cread is not None:
 890                 p2cread.Close()
 891             if c2pwrite is not None:
 892                 c2pwrite.Close()
 893             if errwrite is not None:
 894                 errwrite.Close()
 895
 896
 897         def _internal_poll(self, _deadstate=None):
 898             """Check if child process has terminated.  Returns returncode
 899             attribute."""
 900             if self.returncode is None:
 901                 if WaitForSingleObject(self._handle, 0) == WAIT_OBJECT_0:
 902                     self.returncode = GetExitCodeProcess(self._handle)
 903             return self.returncode
 904
 905
 906         def wait(self):
 907             """Wait for child process to terminate.  Returns returncode
 908             attribute."""
 909             if self.returncode is None:
 910                 obj = WaitForSingleObject(self._handle, INFINITE)
 911                 self.returncode = GetExitCodeProcess(self._handle)
 912             return self.returncode
 913
 914
 915         def _readerthread(self, fh, buffer):
 916             buffer.append(fh.read())
 917
 918
 919         def _communicate(self, input):
 920             stdout = None # Return
 921             stderr = None # Return
 922
 923             if self.stdout:
 924                 stdout = []
 925                 stdout_thread = threading.Thread(target=self._readerthread,
 926                                                  args=(self.stdout, stdout))
 927                 stdout_thread.setDaemon(True)
 928                 stdout_thread.start()
 929             if self.stderr:
 930                 stderr = []
 931                 stderr_thread = threading.Thread(target=self._readerthread,
 932                                                  args=(self.stderr, stderr))
 933                 stderr_thread.setDaemon(True)
 934                 stderr_thread.start()
 935
 936             if self.stdin:
 937                 if input is not None:
 938                     self.stdin.write(input)
 939                 self.stdin.close()
 940
 941             if self.stdout:
 942                 stdout_thread.join()
 943             if self.stderr:
 944                 stderr_thread.join()
 945
 946             # All data exchanged.  Translate lists into strings.
 947             if stdout is not None:
 948                 stdout = stdout[0]
 949             if stderr is not None:
 950                 stderr = stderr[0]
 951
 952             # Translate newlines, if requested.  We cannot let the file
 953             # object do the translation: It is based on stdio, which is
 954             # impossible to combine with select (unless forcing no
 955             # buffering).
 956             if self.universal_newlines and hasattr(file, 'newlines'):
 957                 if stdout:
 958                     stdout = self._translate_newlines(stdout)
 959                 if stderr:
 960                     stderr = self._translate_newlines(stderr)
 961
 962             self.wait()
 963             return (stdout, stderr)
 964
 965         def send_signal(self, sig):
 966             """Send a signal to the process
 967             """
 968             if sig == signal.SIGTERM:
 969                 self.terminate()
 970             else:
 971                 raise ValueError("Only SIGTERM is supported on Windows")
 972
 973         def terminate(self):
 974             """Terminates the process
 975             """
 976             TerminateProcess(self._handle, 1)
 977
 978         kill = terminate
 979
 980     else:
 981         #
 982         # POSIX methods
 983         #
 984         def _get_handles(self, stdin, stdout, stderr):
 985             """Construct and return tupel with IO objects:
 986             p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite
 987             """
 988             p2cread, p2cwrite = None, None
 989             c2pread, c2pwrite = None, None
 990             errread, errwrite = None, None
 991
 992             if stdin is None:
 993                 pass
 994             elif stdin == PIPE:
 995                 p2cread, p2cwrite = os.pipe()
 996             elif isinstance(stdin, int):
 997                 p2cread = stdin
 998             else:
 999                 # Assuming file-like object
1000                 p2cread = stdin.fileno()
1001
1002             if stdout is None:
1003                 pass
1004             elif stdout == PIPE:
1005                 c2pread, c2pwrite = os.pipe()
1006             elif isinstance(stdout, int):
1007                 c2pwrite = stdout
1008             else:
1009                 # Assuming file-like object
1010                 c2pwrite = stdout.fileno()
1011
1012             if stderr is None:
1013                 pass
1014             elif stderr == PIPE:
1015                 errread, errwrite = os.pipe()
1016             elif stderr == STDOUT:
1017                 errwrite = c2pwrite
1018             elif isinstance(stderr, int):
1019                 errwrite = stderr
1020             else:
1021                 # Assuming file-like object
1022                 errwrite = stderr.fileno()
1023
1024             return (p2cread, p2cwrite,
1025                     c2pread, c2pwrite,
1026                     errread, errwrite)
1027
1028
1029         def _set_cloexec_flag(self, fd):
1030             try:
1031                 cloexec_flag = fcntl.FD_CLOEXEC
1032             except AttributeError:
1033                 cloexec_flag = 1
1034
1035             old = fcntl.fcntl(fd, fcntl.F_GETFD)
1036             fcntl.fcntl(fd, fcntl.F_SETFD, old | cloexec_flag)
1037
1038
1039         def _close_fds(self, but):
1040             os.closerange(3, but)
1041             os.closerange(but + 1, MAXFD)
1042
1043
1044         def _execute_child(self, args, executable, preexec_fn, close_fds,
1045                            cwd, env, universal_newlines,
1046                            startupinfo, creationflags, shell,
1047                            p2cread, p2cwrite,
1048                            c2pread, c2pwrite,
1049                            errread, errwrite):
1050             """Execute program (POSIX version)"""
1051
1052             if isinstance(args, types.StringTypes):
1053                 args = [args]
1054             else:
1055                 args = list(args)
1056
1057             if shell:
1058                 args = ["/bin/sh", "-c"] + args
1059
1060             if executable is None:
1061                 executable = args[0]
1062
1063             # For transferring possible exec failure from child to parent
1064             # The first char specifies the exception type: 0 means
1065             # OSError, 1 means some other error.
1066             errpipe_read, errpipe_write = os.pipe()
1067             self._set_cloexec_flag(errpipe_write)
1068
1069             gc_was_enabled = gc.isenabled()
1070             # Disable gc to avoid bug where gc -> file_dealloc ->
1071             # write to stderr -> hang.  http://bugs.python.org/issue1336
1072             gc.disable()
1073             try:
1074                 self.pid = os.fork()
1075             except:
1076                 if gc_was_enabled:
1077                     gc.enable()
1078                 raise
1079             self._child_created = True
1080             if self.pid == 0:
1081                 # Child
1082                 try:
1083                     # Close parent's pipe ends
1084                     if p2cwrite is not None:
1085                         os.close(p2cwrite)
1086                     if c2pread is not None:
1087                         os.close(c2pread)
1088                     if errread is not None:
1089                         os.close(errread)
1090                     os.close(errpipe_read)
1091
1092                     # Dup fds for child
1093                     if p2cread is not None:
1094                         os.dup2(p2cread, 0)
1095                     if c2pwrite is not None:
1096                         os.dup2(c2pwrite, 1)
1097                     if errwrite is not None:
1098                         os.dup2(errwrite, 2)
1099
1100                     # Close pipe fds.  Make sure we don't close the same
1101                     # fd more than once, or standard fds.
1102                     if p2cread is not None and p2cread not in (0,):
1103                         os.close(p2cread)
1104                     if c2pwrite is not None and c2pwrite not in (p2cread, 1):
1105                         os.close(c2pwrite)
1106                     if errwrite is not None and errwrite not in (p2cread, c2pwrite, 2):
1107                         os.close(errwrite)
1108
1109                     # Close all other fds, if asked for
1110                     if close_fds:
1111                         self._close_fds(but=errpipe_write)
1112
1113                     if cwd is not None:
1114                         os.chdir(cwd)
1115
1116                     if preexec_fn:
1117                         preexec_fn()
1118
1119                     if env is None:
1120                         os.execvp(executable, args)
1121                     else:
1122                         os.execvpe(executable, args, env)
1123
1124                 except:
1125                     exc_type, exc_value, tb = sys.exc_info()
1126                     # Save the traceback and attach it to the exception object
1127                     exc_lines = traceback.format_exception(exc_type,
1128                                                            exc_value,
1129                                                            tb)
1130                     exc_value.child_traceback = ''.join(exc_lines)
1131                     os.write(errpipe_write, pickle.dumps(exc_value))
1132
1133                 # This exitcode won't be reported to applications, so it
1134                 # really doesn't matter what we return.
1135                 os._exit(255)
1136
1137             # Parent
1138             if gc_was_enabled:
1139                 gc.enable()
1140             os.close(errpipe_write)
1141             if p2cread is not None and p2cwrite is not None:
1142                 os.close(p2cread)
1143             if c2pwrite is not None and c2pread is not None:
1144                 os.close(c2pwrite)
1145             if errwrite is not None and errread is not None:
1146                 os.close(errwrite)
1147
1148             # Wait for exec to fail or succeed; possibly raising exception
1149             data = os.read(errpipe_read, 1048576) # Exceptions limited to 1 MB
1150             os.close(errpipe_read)
1151             if data != "":
1152                 os.waitpid(self.pid, 0)
1153                 child_exception = pickle.loads(data)
1154                 raise child_exception
1155
1156
1157         def _handle_exitstatus(self, sts):
1158             if os.WIFSIGNALED(sts):
1159                 self.returncode = -os.WTERMSIG(sts)
1160             elif os.WIFEXITED(sts):
1161                 self.returncode = os.WEXITSTATUS(sts)
1162             else:
1163                 # Should never happen
1164                 raise RuntimeError("Unknown child exit status!")
1165
1166
1167         def _internal_poll(self, _deadstate=None):
1168             """Check if child process has terminated.  Returns returncode
1169             attribute."""
1170             if self.returncode is None:
1171                 try:
1172                     pid, sts = os.waitpid(self.pid, os.WNOHANG)
1173                     if pid == self.pid:
1174                         self._handle_exitstatus(sts)
1175                 except os.error:
1176                     if _deadstate is not None:
1177                         self.returncode = _deadstate
1178             return self.returncode
1179
1180
1181         def wait(self):
1182             """Wait for child process to terminate.  Returns returncode
1183             attribute."""
1184             if self.returncode is None:
1185                 pid, sts = os.waitpid(self.pid, 0)
1186                 self._handle_exitstatus(sts)
1187             return self.returncode
1188
1189
1190         def _communicate(self, input):
1191             read_set = []
1192             write_set = []
1193             stdout = None # Return
1194             stderr = None # Return
1195
1196             if self.stdin:
1197                 # Flush stdio buffer.  This might block, if the user has
1198                 # been writing to .stdin in an uncontrolled fashion.
1199                 self.stdin.flush()
1200                 if input:
1201                     write_set.append(self.stdin)
1202                 else:
1203                     self.stdin.close()
1204             if self.stdout:
1205                 read_set.append(self.stdout)
1206                 stdout = []
1207             if self.stderr:
1208                 read_set.append(self.stderr)
1209                 stderr = []
1210
1211             input_offset = 0
1212             while read_set or write_set:
1213                 try:
1214                     rlist, wlist, xlist = select.select(read_set, write_set, [])
1215                 except select.error, e:
1216                     if e.args[0] == errno.EINTR:
1217                         continue
1218                     raise
1219
1220                 if self.stdin in wlist:
1221                     # When select has indicated that the file is writable,
1222                     # we can write up to PIPE_BUF bytes without risk
1223                     # blocking.  POSIX defines PIPE_BUF >= 512
1224                     chunk = input[input_offset : input_offset + 512]
1225                     bytes_written = os.write(self.stdin.fileno(), chunk)
1226                     input_offset += bytes_written
1227                     if input_offset >= len(input):
1228                         self.stdin.close()
1229                         write_set.remove(self.stdin)
1230
1231                 if self.stdout in rlist:
1232                     data = os.read(self.stdout.fileno(), 1024)
1233                     if data == "":
1234                         self.stdout.close()
1235                         read_set.remove(self.stdout)
1236                     stdout.append(data)
1237
1238                 if self.stderr in rlist:
1239                     data = os.read(self.stderr.fileno(), 1024)
1240                     if data == "":
1241                         self.stderr.close()
1242                         read_set.remove(self.stderr)
1243                     stderr.append(data)
1244
1245             # All data exchanged.  Translate lists into strings.
1246             if stdout is not None:
1247                 stdout = ''.join(stdout)
1248             if stderr is not None:
1249                 stderr = ''.join(stderr)
1250
1251             # Translate newlines, if requested.  We cannot let the file
1252             # object do the translation: It is based on stdio, which is
1253             # impossible to combine with select (unless forcing no
1254             # buffering).
1255             if self.universal_newlines and hasattr(file, 'newlines'):
1256                 if stdout:
1257                     stdout = self._translate_newlines(stdout)
1258                 if stderr:
1259                     stderr = self._translate_newlines(stderr)
1260
1261             self.wait()
1262             return (stdout, stderr)
1263
1264         def send_signal(self, sig):
1265             """Send a signal to the process
1266             """
1267             os.kill(self.pid, sig)
1268
1269         def terminate(self):
1270             """Terminate the process with SIGTERM
1271             """
1272             self.send_signal(signal.SIGTERM)
1273
1274         def kill(self):
1275             """Kill the process with SIGKILL
1276             """
1277             self.send_signal(signal.SIGKILL)
1278
1279
1280 def _demo_posix():
1281     #
1282     # Example 1: Simple redirection: Get process list
1283     #
1284     plist = Popen(["ps"], stdout=PIPE).communicate()[0]
1285     print "Process list:"
1286     print plist
1287
1288     #
1289     # Example 2: Change uid before executing child
1290     #
1291     if os.getuid() == 0:
1292         p = Popen(["id"], preexec_fn=lambda: os.setuid(100))
1293         p.wait()
1294
1295     #
1296     # Example 3: Connecting several subprocesses
1297     #
1298     print "Looking for 'hda'..."
1299     p1 = Popen(["dmesg"], stdout=PIPE)
1300     p2 = Popen(["grep", "hda"], stdin=p1.stdout, stdout=PIPE)
1301     print repr(p2.communicate()[0])
1302
1303     #
1304     # Example 4: Catch execution error
1305     #
1306     print
1307     print "Trying a weird file..."
1308     try:
1309         print Popen(["/this/path/does/not/exist"]).communicate()
1310     except OSError, e:
1311         if e.errno == errno.ENOENT:
1312             print "The file didn't exist.  I thought so..."
1313             print "Child traceback:"
1314             print e.child_traceback
1315         else:
1316             print "Error", e.errno
1317     else:
1318         print >>sys.stderr, "Gosh.  No error."
1319
1320
1321 def _demo_windows():
1322     #
1323     # Example 1: Connecting several subprocesses
1324     #
1325     print "Looking for 'PROMPT' in set output..."
1326     p1 = Popen("set", stdout=PIPE, shell=True)
1327     p2 = Popen('find "PROMPT"', stdin=p1.stdout, stdout=PIPE)
1328     print repr(p2.communicate()[0])
1329
1330     #
1331     # Example 2: Simple execution of program
1332     #
1333     print "Executing calc..."
1334     p = Popen("calc")
1335     p.wait()
1336
1337
1338 if __name__ == "__main__":
1339     if mswindows:
1340         _demo_windows()
1341     else:
1342         _demo_posix()