Lib/subprocess.py

   1 # subprocess - Subprocesses with accessible I/O streams
   2 #
   3 # For more information about this module, see PEP 324.
   4 #
   5 # This module should remain compatible with Python 2.2, see PEP 291.
   6 #
   7 # Copyright (c) 2003-2005 by Peter Astrand <astrand@lysator.liu.se>
   8 #
   9 # Licensed to PSF under a Contributor Agreement.
  10 # See http://www.python.org/2.4/license for licensing details.
  11
  12 r"""subprocess - Subprocesses with accessible I/O streams
  13
  14 This module allows you to spawn processes, connect to their
  15 input/output/error pipes, and obtain their return codes.  This module
  16 intends to replace several other, older modules and functions, like:
  17
  18 os.system
  19 os.spawn*
  20 os.popen*
  21 popen2.*
  22 commands.*
  23
  24 Information about how the subprocess module can be used to replace these
  25 modules and functions can be found below.
  26
  27
  28
  29 Using the subprocess module
  30 ===========================
  31 This module defines one class called Popen:
  32
  33 class Popen(args, bufsize=0, executable=None,
  34             stdin=None, stdout=None, stderr=None,
  35             preexec_fn=None, close_fds=False, shell=False,
  36             cwd=None, env=None, universal_newlines=False,
  37             startupinfo=None, creationflags=0):
  38
  39
  40 Arguments are:
  41
  42 args should be a string, or a sequence of program arguments.  The
  43 program to execute is normally the first item in the args sequence or
  44 string, but can be explicitly set by using the executable argument.
  45
  46 On UNIX, with shell=False (default): In this case, the Popen class
  47 uses os.execvp() to execute the child program.  args should normally
  48 be a sequence.  A string will be treated as a sequence with the string
  49 as the only item (the program to execute).
  50
  51 On UNIX, with shell=True: If args is a string, it specifies the
  52 command string to execute through the shell.  If args is a sequence,
  53 the first item specifies the command string, and any additional items
  54 will be treated as additional shell arguments.
  55
  56 On Windows: the Popen class uses CreateProcess() to execute the child
  57 program, which operates on strings.  If args is a sequence, it will be
  58 converted to a string using the list2cmdline method.  Please note that
  59 not all MS Windows applications interpret the command line the same
  60 way: The list2cmdline is designed for applications using the same
  61 rules as the MS C runtime.
  62
  63 bufsize, if given, has the same meaning as the corresponding argument
  64 to the built-in open() function: 0 means unbuffered, 1 means line
  65 buffered, any other positive value means use a buffer of
  66 (approximately) that size.  A negative bufsize means to use the system
  67 default, which usually means fully buffered.  The default value for
  68 bufsize is 0 (unbuffered).
  69
  70 stdin, stdout and stderr specify the executed programs' standard
  71 input, standard output and standard error file handles, respectively.
  72 Valid values are PIPE, an existing file descriptor (a positive
  73 integer), an existing file object, and None.  PIPE indicates that a
  74 new pipe to the child should be created.  With None, no redirection
  75 will occur; the child's file handles will be inherited from the
  76 parent.  Additionally, stderr can be STDOUT, which indicates that the
  77 stderr data from the applications should be captured into the same
  78 file handle as for stdout.
  79
  80 If preexec_fn is set to a callable object, this object will be called
  81 in the child process just before the child is executed.
  82
  83 If close_fds is true, all file descriptors except 0, 1 and 2 will be
  84 closed before the child process is executed.
  85
  86 if shell is true, the specified command will be executed through the
  87 shell.
  88
  89 If cwd is not None, the current directory will be changed to cwd
  90 before the child is executed.
  91
  92 If env is not None, it defines the environment variables for the new
  93 process.
  94
  95 If universal_newlines is true, the file objects stdout and stderr are
  96 opened as a text files, but lines may be terminated by any of '\n',
  97 the Unix end-of-line convention, '\r', the Macintosh convention or
  98 '\r\n', the Windows convention.  All of these external representations
  99 are seen as '\n' by the Python program.  Note: This feature is only
 100 available if Python is built with universal newline support (the
 101 default).  Also, the newlines attribute of the file objects stdout,
 102 stdin and stderr are not updated by the communicate() method.
 103
 104 The startupinfo and creationflags, if given, will be passed to the
 105 underlying CreateProcess() function.  They can specify things such as
 106 appearance of the main window and priority for the new process.
 107 (Windows only)
 108
 109
 110 This module also defines two shortcut functions:
 111
 112 call(*popenargs, **kwargs):
 113     Run command with arguments.  Wait for command to complete, then
 114     return the returncode attribute.
 115
 116     The arguments are the same as for the Popen constructor.  Example:
 117
 118     retcode = call(["ls", "-l"])
 119
 120 check_call(*popenargs, **kwargs):
 121     Run command with arguments.  Wait for command to complete.  If the
 122     exit code was zero then return, otherwise raise
 123     CalledProcessError.  The CalledProcessError object will have the
 124     return code in the errno attribute.
 125
 126     The arguments are the same as for the Popen constructor.  Example:
 127
 128     check_call(["ls", "-l"])
 129
 130 Exceptions
 131 ----------
 132 Exceptions raised in the child process, before the new program has
 133 started to execute, will be re-raised in the parent.  Additionally,
 134 the exception object will have one extra attribute called
 135 'child_traceback', which is a string containing traceback information
 136 from the childs point of view.
 137
 138 The most common exception raised is OSError.  This occurs, for
 139 example, when trying to execute a non-existent file.  Applications
 140 should prepare for OSErrors.
 141
 142 A ValueError will be raised if Popen is called with invalid arguments.
 143
 144 check_call() will raise CalledProcessError, which is a subclass of
 145 OSError, if the called process returns a non-zero return code.
 146
 147
 148 Security
 149 --------
 150 Unlike some other popen functions, this implementation will never call
 151 /bin/sh implicitly.  This means that all characters, including shell
 152 metacharacters, can safely be passed to child processes.
 153
 154
 155 Popen objects
 156 =============
 157 Instances of the Popen class have the following methods:
 158
 159 poll()
 160     Check if child process has terminated.  Returns returncode
 161     attribute.
 162
 163 wait()
 164     Wait for child process to terminate.  Returns returncode attribute.
 165
 166 communicate(input=None)
 167     Interact with process: Send data to stdin.  Read data from stdout
 168     and stderr, until end-of-file is reached.  Wait for process to
 169     terminate.  The optional stdin argument should be a string to be
 170     sent to the child process, or None, if no data should be sent to
 171     the child.
 172
 173     communicate() returns a tuple (stdout, stderr).
 174
 175     Note: The data read is buffered in memory, so do not use this
 176     method if the data size is large or unlimited.
 177
 178 The following attributes are also available:
 179
 180 stdin
 181     If the stdin argument is PIPE, this attribute is a file object
 182     that provides input to the child process.  Otherwise, it is None.
 183
 184 stdout
 185     If the stdout argument is PIPE, this attribute is a file object
 186     that provides output from the child process.  Otherwise, it is
 187     None.
 188
 189 stderr
 190     If the stderr argument is PIPE, this attribute is file object that
 191     provides error output from the child process.  Otherwise, it is
 192     None.
 193
 194 pid
 195     The process ID of the child process.
 196
 197 returncode
 198     The child return code.  A None value indicates that the process
 199     hasn't terminated yet.  A negative value -N indicates that the
 200     child was terminated by signal N (UNIX only).
 201
 202
 203 Replacing older functions with the subprocess module
 204 ====================================================
 205 In this section, "a ==> b" means that b can be used as a replacement
 206 for a.
 207
 208 Note: All functions in this section fail (more or less) silently if
 209 the executed program cannot be found; this module raises an OSError
 210 exception.
 211
 212 In the following examples, we assume that the subprocess module is
 213 imported with "from subprocess import *".
 214
 215
 216 Replacing /bin/sh shell backquote
 217 ---------------------------------
 218 output=`mycmd myarg`
 219 ==>
 220 output = Popen(["mycmd", "myarg"], stdout=PIPE).communicate()[0]
 221
 222
 223 Replacing shell pipe line
 224 -------------------------
 225 output=`dmesg | grep hda`
 226 ==>
 227 p1 = Popen(["dmesg"], stdout=PIPE)
 228 p2 = Popen(["grep", "hda"], stdin=p1.stdout, stdout=PIPE)
 229 output = p2.communicate()[0]
 230
 231
 232 Replacing os.system()
 233 ---------------------
 234 sts = os.system("mycmd" + " myarg")
 235 ==>
 236 p = Popen("mycmd" + " myarg", shell=True)
 237 sts = os.waitpid(p.pid, 0)
 238
 239 Note:
 240
 241 * Calling the program through the shell is usually not required.
 242
 243 * It's easier to look at the returncode attribute than the
 244   exitstatus.
 245
 246 A more real-world example would look like this:
 247
 248 try:
 249     retcode = call("mycmd" + " myarg", shell=True)
 250     if retcode < 0:
 251         print >>sys.stderr, "Child was terminated by signal", -retcode
 252     else:
 253         print >>sys.stderr, "Child returned", retcode
 254 except OSError, e:
 255     print >>sys.stderr, "Execution failed:", e
 256
 257
 258 Replacing os.spawn*
 259 -------------------
 260 P_NOWAIT example:
 261
 262 pid = os.spawnlp(os.P_NOWAIT, "/bin/mycmd", "mycmd", "myarg")
 263 ==>
 264 pid = Popen(["/bin/mycmd", "myarg"]).pid
 265
 266
 267 P_WAIT example:
 268
 269 retcode = os.spawnlp(os.P_WAIT, "/bin/mycmd", "mycmd", "myarg")
 270 ==>
 271 retcode = call(["/bin/mycmd", "myarg"])
 272
 273
 274 Vector example:
 275
 276 os.spawnvp(os.P_NOWAIT, path, args)
 277 ==>
 278 Popen([path] + args[1:])
 279
 280
 281 Environment example:
 282
 283 os.spawnlpe(os.P_NOWAIT, "/bin/mycmd", "mycmd", "myarg", env)
 284 ==>
 285 Popen(["/bin/mycmd", "myarg"], env={"PATH": "/usr/bin"})
 286
 287
 288 Replacing os.popen*
 289 -------------------
 290 pipe = os.popen(cmd, mode='r', bufsize)
 291 ==>
 292 pipe = Popen(cmd, shell=True, bufsize=bufsize, stdout=PIPE).stdout
 293
 294 pipe = os.popen(cmd, mode='w', bufsize)
 295 ==>
 296 pipe = Popen(cmd, shell=True, bufsize=bufsize, stdin=PIPE).stdin
 297
 298
 299 (child_stdin, child_stdout) = os.popen2(cmd, mode, bufsize)
 300 ==>
 301 p = Popen(cmd, shell=True, bufsize=bufsize,
 302           stdin=PIPE, stdout=PIPE, close_fds=True)
 303 (child_stdin, child_stdout) = (p.stdin, p.stdout)
 304
 305
 306 (child_stdin,
 307  child_stdout,
 308  child_stderr) = os.popen3(cmd, mode, bufsize)
 309 ==>
 310 p = Popen(cmd, shell=True, bufsize=bufsize,
 311           stdin=PIPE, stdout=PIPE, stderr=PIPE, close_fds=True)
 312 (child_stdin,
 313  child_stdout,
 314  child_stderr) = (p.stdin, p.stdout, p.stderr)
 315
 316
 317 (child_stdin, child_stdout_and_stderr) = os.popen4(cmd, mode, bufsize)
 318 ==>
 319 p = Popen(cmd, shell=True, bufsize=bufsize,
 320           stdin=PIPE, stdout=PIPE, stderr=STDOUT, close_fds=True)
 321 (child_stdin, child_stdout_and_stderr) = (p.stdin, p.stdout)
 322
 323
 324 Replacing popen2.*
 325 ------------------
 326 Note: If the cmd argument to popen2 functions is a string, the command
 327 is executed through /bin/sh.  If it is a list, the command is directly
 328 executed.
 329
 330 (child_stdout, child_stdin) = popen2.popen2("somestring", bufsize, mode)
 331 ==>
 332 p = Popen(["somestring"], shell=True, bufsize=bufsize
 333           stdin=PIPE, stdout=PIPE, close_fds=True)
 334 (child_stdout, child_stdin) = (p.stdout, p.stdin)
 335
 336
 337 (child_stdout, child_stdin) = popen2.popen2(["mycmd", "myarg"], bufsize, mode)
 338 ==>
 339 p = Popen(["mycmd", "myarg"], bufsize=bufsize,
 340           stdin=PIPE, stdout=PIPE, close_fds=True)
 341 (child_stdout, child_stdin) = (p.stdout, p.stdin)
 342
 343 The popen2.Popen3 and popen3.Popen4 basically works as subprocess.Popen,
 344 except that:
 345
 346 * subprocess.Popen raises an exception if the execution fails
 347 * the capturestderr argument is replaced with the stderr argument.
 348 * stdin=PIPE and stdout=PIPE must be specified.
 349 * popen2 closes all filedescriptors by default, but you have to specify
 350   close_fds=True with subprocess.Popen.
 351
 352
 353 """
 354
 355 import sys
 356 mswindows = (sys.platform == "win32")
 357
 358 import os
 359 import types
 360 import traceback
 361
 362 # Exception classes used by this module.
 363 class CalledProcessError(OSError):
 364     """This exception is raised when a process run by check_call() returns
 365     a non-zero exit status.  The exit status will be stored in the
 366     errno attribute.  This exception is a subclass of
 367     OSError."""
 368
 369 if mswindows:
 370     import threading
 371     import msvcrt
 372     if 0: # <-- change this to use pywin32 instead of the _subprocess driver
 373         import pywintypes
 374         from win32api import GetStdHandle, STD_INPUT_HANDLE, \
 375                              STD_OUTPUT_HANDLE, STD_ERROR_HANDLE
 376         from win32api import GetCurrentProcess, DuplicateHandle, \
 377                              GetModuleFileName, GetVersion
 378         from win32con import DUPLICATE_SAME_ACCESS, SW_HIDE
 379         from win32pipe import CreatePipe
 380         from win32process import CreateProcess, STARTUPINFO, \
 381                                  GetExitCodeProcess, STARTF_USESTDHANDLES, \
 382                                  STARTF_USESHOWWINDOW, CREATE_NEW_CONSOLE
 383         from win32event import WaitForSingleObject, INFINITE, WAIT_OBJECT_0
 384     else:
 385         from _subprocess import *
 386         class STARTUPINFO:
 387             dwFlags = 0
 388             hStdInput = None
 389             hStdOutput = None
 390             hStdError = None
 391         class pywintypes:
 392             error = IOError
 393 else:
 394     import select
 395     import errno
 396     import fcntl
 397     import pickle
 398
 399 __all__ = ["Popen", "PIPE", "STDOUT", "call", "check_call", "CalledProcessError"]
 400
 401 try:
 402     MAXFD = os.sysconf("SC_OPEN_MAX")
 403 except:
 404     MAXFD = 256
 405
 406 # True/False does not exist on 2.2.0
 407 try:
 408     False
 409 except NameError:
 410     False = 0
 411     True = 1
 412
 413 _active = []
 414
 415 def _cleanup():
 416     for inst in _active[:]:
 417         inst.poll()
 418
 419 PIPE = -1
 420 STDOUT = -2
 421
 422
 423 def call(*popenargs, **kwargs):
 424     """Run command with arguments.  Wait for command to complete, then
 425     return the returncode attribute.
 426
 427     The arguments are the same as for the Popen constructor.  Example:
 428
 429     retcode = call(["ls", "-l"])
 430     """
 431     return Popen(*popenargs, **kwargs).wait()
 432
 433
 434 def check_call(*popenargs, **kwargs):
 435     """Run command with arguments.  Wait for command to complete.  If
 436     the exit code was zero then return, otherwise raise
 437     CalledProcessError.  The CalledProcessError object will have the
 438     return code in the errno attribute.
 439
 440     The arguments are the same as for the Popen constructor.  Example:
 441
 442     check_call(["ls", "-l"])
 443     """
 444     retcode = call(*popenargs, **kwargs)
 445     cmd = kwargs.get("args")
 446     if cmd is None:
 447         cmd = popenargs[0]
 448     if retcode:
 449         raise CalledProcessError(retcode, "Command %s returned non-zero exit status" % cmd)
 450     return retcode
 451
 452
 453 def list2cmdline(seq):
 454     """
 455     Translate a sequence of arguments into a command line
 456     string, using the same rules as the MS C runtime:
 457
 458     1) Arguments are delimited by white space, which is either a
 459        space or a tab.
 460
 461     2) A string surrounded by double quotation marks is
 462        interpreted as a single argument, regardless of white space
 463        contained within.  A quoted string can be embedded in an
 464        argument.
 465
 466     3) A double quotation mark preceded by a backslash is
 467        interpreted as a literal double quotation mark.
 468
 469     4) Backslashes are interpreted literally, unless they
 470        immediately precede a double quotation mark.
 471
 472     5) If backslashes immediately precede a double quotation mark,
 473        every pair of backslashes is interpreted as a literal
 474        backslash.  If the number of backslashes is odd, the last
 475        backslash escapes the next double quotation mark as
 476        described in rule 3.
 477     """
 478
 479     # See
 480     # http://msdn.microsoft.com/library/en-us/vccelng/htm/progs_12.asp
 481     result = []
 482     needquote = False
 483     for arg in seq:
 484         bs_buf = []
 485
 486         # Add a space to separate this argument from the others
 487         if result:
 488             result.append(' ')
 489
 490         needquote = (" " in arg) or ("\t" in arg)
 491         if needquote:
 492             result.append('"')
 493
 494         for c in arg:
 495             if c == '\\':
 496                 # Don't know if we need to double yet.
 497                 bs_buf.append(c)
 498             elif c == '"':
 499                 # Double backspaces.
 500                 result.append('\\' * len(bs_buf)*2)
 501                 bs_buf = []
 502                 result.append('\\"')
 503             else:
 504                 # Normal char
 505                 if bs_buf:
 506                     result.extend(bs_buf)
 507                     bs_buf = []
 508                 result.append(c)
 509
 510         # Add remaining backspaces, if any.
 511         if bs_buf:
 512             result.extend(bs_buf)
 513
 514         if needquote:
 515             result.extend(bs_buf)
 516             result.append('"')
 517
 518     return ''.join(result)
 519
 520
 521 class Popen(object):
 522     def __init__(self, args, bufsize=0, executable=None,
 523                  stdin=None, stdout=None, stderr=None,
 524                  preexec_fn=None, close_fds=False, shell=False,
 525                  cwd=None, env=None, universal_newlines=False,
 526                  startupinfo=None, creationflags=0):
 527         """Create new Popen instance."""
 528         _cleanup()
 529
 530         if not isinstance(bufsize, (int, long)):
 531             raise TypeError("bufsize must be an integer")
 532
 533         if mswindows:
 534             if preexec_fn is not None:
 535                 raise ValueError("preexec_fn is not supported on Windows "
 536                                  "platforms")
 537             if close_fds:
 538                 raise ValueError("close_fds is not supported on Windows "
 539                                  "platforms")
 540         else:
 541             # POSIX
 542             if startupinfo is not None:
 543                 raise ValueError("startupinfo is only supported on Windows "
 544                                  "platforms")
 545             if creationflags != 0:
 546                 raise ValueError("creationflags is only supported on Windows "
 547                                  "platforms")
 548
 549         self.stdin = None
 550         self.stdout = None
 551         self.stderr = None
 552         self.pid = None
 553         self.returncode = None
 554         self.universal_newlines = universal_newlines
 555
 556         # Input and output objects. The general principle is like
 557         # this:
 558         #
 559         # Parent                   Child
 560         # ------                   -----
 561         # p2cwrite   ---stdin--->  p2cread
 562         # c2pread    <--stdout---  c2pwrite
 563         # errread    <--stderr---  errwrite
 564         #
 565         # On POSIX, the child objects are file descriptors.  On
 566         # Windows, these are Windows file handles.  The parent objects
 567         # are file descriptors on both platforms.  The parent objects
 568         # are None when not using PIPEs. The child objects are None
 569         # when not redirecting.
 570
 571         (p2cread, p2cwrite,
 572          c2pread, c2pwrite,
 573          errread, errwrite) = self._get_handles(stdin, stdout, stderr)
 574
 575         self._execute_child(args, executable, preexec_fn, close_fds,
 576                             cwd, env, universal_newlines,
 577                             startupinfo, creationflags, shell,
 578                             p2cread, p2cwrite,
 579                             c2pread, c2pwrite,
 580                             errread, errwrite)
 581
 582         if p2cwrite:
 583             self.stdin = os.fdopen(p2cwrite, 'wb', bufsize)
 584         if c2pread:
 585             if universal_newlines:
 586                 self.stdout = os.fdopen(c2pread, 'rU', bufsize)
 587             else:
 588                 self.stdout = os.fdopen(c2pread, 'rb', bufsize)
 589         if errread:
 590             if universal_newlines:
 591                 self.stderr = os.fdopen(errread, 'rU', bufsize)
 592             else:
 593                 self.stderr = os.fdopen(errread, 'rb', bufsize)
 594
 595         _active.append(self)
 596
 597
 598     def _translate_newlines(self, data):
 599         data = data.replace("\r\n", "\n")
 600         data = data.replace("\r", "\n")
 601         return data
 602
 603     def communicate(self, input=None):
 604         """Interact with process: Send data to stdin.  Read data from
 605         stdout and stderr, until end-of-file is reached.  Wait for
 606         process to terminate.  The optional input argument should be a
 607         string to be sent to the child process, or None, if no data
 608         should be sent to the child.
 609
 610         communicate() returns a tuple (stdout, stderr)."""
 611
 612         # Optimization: If we are only using one pipe, or no pipe at
 613         # all, using select() or threads is unnecessary.
 614         if [self.stdin, self.stdout, self.stderr].count(None) >= 2:
 615             stdout = None
 616             stderr = None
 617             if self.stdin:
 618                 if input:
 619                     self.stdin.write(input)
 620                 self.stdin.close()
 621             elif self.stdout:
 622                 stdout = self.stdout.read()
 623             elif self.stderr:
 624                 stderr = self.stderr.read()
 625             self.wait()
 626             return (stdout, stderr)
 627
 628         return self._communicate(input)
 629
 630
 631     if mswindows:
 632         #
 633         # Windows methods
 634         #
 635         def _get_handles(self, stdin, stdout, stderr):
 636             """Construct and return tupel with IO objects:
 637             p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite
 638             """
 639             if stdin is None and stdout is None and stderr is None:
 640                 return (None, None, None, None, None, None)
 641
 642             p2cread, p2cwrite = None, None
 643             c2pread, c2pwrite = None, None
 644             errread, errwrite = None, None
 645
 646             if stdin is None:
 647                 p2cread = GetStdHandle(STD_INPUT_HANDLE)
 648             elif stdin == PIPE:
 649                 p2cread, p2cwrite = CreatePipe(None, 0)
 650                 # Detach and turn into fd
 651                 p2cwrite = p2cwrite.Detach()
 652                 p2cwrite = msvcrt.open_osfhandle(p2cwrite, 0)
 653             elif isinstance(stdin, int):
 654                 p2cread = msvcrt.get_osfhandle(stdin)
 655             else:
 656                 # Assuming file-like object
 657                 p2cread = msvcrt.get_osfhandle(stdin.fileno())
 658             p2cread = self._make_inheritable(p2cread)
 659
 660             if stdout is None:
 661                 c2pwrite = GetStdHandle(STD_OUTPUT_HANDLE)
 662             elif stdout == PIPE:
 663                 c2pread, c2pwrite = CreatePipe(None, 0)
 664                 # Detach and turn into fd
 665                 c2pread = c2pread.Detach()
 666                 c2pread = msvcrt.open_osfhandle(c2pread, 0)
 667             elif isinstance(stdout, int):
 668                 c2pwrite = msvcrt.get_osfhandle(stdout)
 669             else:
 670                 # Assuming file-like object
 671                 c2pwrite = msvcrt.get_osfhandle(stdout.fileno())
 672             c2pwrite = self._make_inheritable(c2pwrite)
 673
 674             if stderr is None:
 675                 errwrite = GetStdHandle(STD_ERROR_HANDLE)
 676             elif stderr == PIPE:
 677                 errread, errwrite = CreatePipe(None, 0)
 678                 # Detach and turn into fd
 679                 errread = errread.Detach()
 680                 errread = msvcrt.open_osfhandle(errread, 0)
 681             elif stderr == STDOUT:
 682                 errwrite = c2pwrite
 683             elif isinstance(stderr, int):
 684                 errwrite = msvcrt.get_osfhandle(stderr)
 685             else:
 686                 # Assuming file-like object
 687                 errwrite = msvcrt.get_osfhandle(stderr.fileno())
 688             errwrite = self._make_inheritable(errwrite)
 689
 690             return (p2cread, p2cwrite,
 691                     c2pread, c2pwrite,
 692                     errread, errwrite)
 693
 694
 695         def _make_inheritable(self, handle):
 696             """Return a duplicate of handle, which is inheritable"""
 697             return DuplicateHandle(GetCurrentProcess(), handle,
 698                                    GetCurrentProcess(), 0, 1,
 699                                    DUPLICATE_SAME_ACCESS)
 700
 701
 702         def _find_w9xpopen(self):
 703             """Find and return absolut path to w9xpopen.exe"""
 704             w9xpopen = os.path.join(os.path.dirname(GetModuleFileName(0)),
 705                                     "w9xpopen.exe")
 706             if not os.path.exists(w9xpopen):
 707                 # Eeek - file-not-found - possibly an embedding
 708                 # situation - see if we can locate it in sys.exec_prefix
 709                 w9xpopen = os.path.join(os.path.dirname(sys.exec_prefix),
 710                                         "w9xpopen.exe")
 711                 if not os.path.exists(w9xpopen):
 712                     raise RuntimeError("Cannot locate w9xpopen.exe, which is "
 713                                        "needed for Popen to work with your "
 714                                        "shell or platform.")
 715             return w9xpopen
 716
 717
 718         def _execute_child(self, args, executable, preexec_fn, close_fds,
 719                            cwd, env, universal_newlines,
 720                            startupinfo, creationflags, shell,
 721                            p2cread, p2cwrite,
 722                            c2pread, c2pwrite,
 723                            errread, errwrite):
 724             """Execute program (MS Windows version)"""
 725
 726             if not isinstance(args, types.StringTypes):
 727                 args = list2cmdline(args)
 728
 729             # Process startup details
 730             default_startupinfo = STARTUPINFO()
 731             if startupinfo is None:
 732                 startupinfo = default_startupinfo
 733             if not None in (p2cread, c2pwrite, errwrite):
 734                 startupinfo.dwFlags |= STARTF_USESTDHANDLES
 735                 startupinfo.hStdInput = p2cread
 736                 startupinfo.hStdOutput = c2pwrite
 737                 startupinfo.hStdError = errwrite
 738
 739             if shell:
 740                 default_startupinfo.dwFlags |= STARTF_USESHOWWINDOW
 741                 default_startupinfo.wShowWindow = SW_HIDE
 742                 comspec = os.environ.get("COMSPEC", "cmd.exe")
 743                 args = comspec + " /c " + args
 744                 if (GetVersion() >= 0x80000000L or
 745                         os.path.basename(comspec).lower() == "command.com"):
 746                     # Win9x, or using command.com on NT. We need to
 747                     # use the w9xpopen intermediate program. For more
 748                     # information, see KB Q150956
 749                     # (http://web.archive.org/web/20011105084002/http://support.microsoft.com/support/kb/articles/Q150/9/56.asp)
 750                     w9xpopen = self._find_w9xpopen()
 751                     args = '"%s" %s' % (w9xpopen, args)
 752                     # Not passing CREATE_NEW_CONSOLE has been known to
 753                     # cause random failures on win9x.  Specifically a
 754                     # dialog: "Your program accessed mem currently in
 755                     # use at xxx" and a hopeful warning about the
 756                     # stability of your system.  Cost is Ctrl+C wont
 757                     # kill children.
 758                     creationflags |= CREATE_NEW_CONSOLE
 759
 760             # Start the process
 761             try:
 762                 hp, ht, pid, tid = CreateProcess(executable, args,
 763                                          # no special security
 764                                          None, None,
 765                                          # must inherit handles to pass std
 766                                          # handles
 767                                          1,
 768                                          creationflags,
 769                                          env,
 770                                          cwd,
 771                                          startupinfo)
 772             except pywintypes.error, e:
 773                 # Translate pywintypes.error to WindowsError, which is
 774                 # a subclass of OSError.  FIXME: We should really
 775                 # translate errno using _sys_errlist (or simliar), but
 776                 # how can this be done from Python?
 777                 raise WindowsError(*e.args)
 778
 779             # Retain the process handle, but close the thread handle
 780             self._handle = hp
 781             self.pid = pid
 782             ht.Close()
 783
 784             # Child is launched. Close the parent's copy of those pipe
 785             # handles that only the child should have open.  You need
 786             # to make sure that no handles to the write end of the
 787             # output pipe are maintained in this process or else the
 788             # pipe will not close when the child process exits and the
 789             # ReadFile will hang.
 790             if p2cread is not None:
 791                 p2cread.Close()
 792             if c2pwrite is not None:
 793                 c2pwrite.Close()
 794             if errwrite is not None:
 795                 errwrite.Close()
 796
 797
 798         def poll(self):
 799             """Check if child process has terminated.  Returns returncode
 800             attribute."""
 801             if self.returncode is None:
 802                 if WaitForSingleObject(self._handle, 0) == WAIT_OBJECT_0:
 803                     self.returncode = GetExitCodeProcess(self._handle)
 804                     _active.remove(self)
 805             return self.returncode
 806
 807
 808         def wait(self):
 809             """Wait for child process to terminate.  Returns returncode
 810             attribute."""
 811             if self.returncode is None:
 812                 obj = WaitForSingleObject(self._handle, INFINITE)
 813                 self.returncode = GetExitCodeProcess(self._handle)
 814                 _active.remove(self)
 815             return self.returncode
 816
 817
 818         def _readerthread(self, fh, buffer):
 819             buffer.append(fh.read())
 820
 821
 822         def _communicate(self, input):
 823             stdout = None # Return
 824             stderr = None # Return
 825
 826             if self.stdout:
 827                 stdout = []
 828                 stdout_thread = threading.Thread(target=self._readerthread,
 829                                                  args=(self.stdout, stdout))
 830                 stdout_thread.setDaemon(True)
 831                 stdout_thread.start()
 832             if self.stderr:
 833                 stderr = []
 834                 stderr_thread = threading.Thread(target=self._readerthread,
 835                                                  args=(self.stderr, stderr))
 836                 stderr_thread.setDaemon(True)
 837                 stderr_thread.start()
 838
 839             if self.stdin:
 840                 if input is not None:
 841                     self.stdin.write(input)
 842                 self.stdin.close()
 843
 844             if self.stdout:
 845                 stdout_thread.join()
 846             if self.stderr:
 847                 stderr_thread.join()
 848
 849             # All data exchanged.  Translate lists into strings.
 850             if stdout is not None:
 851                 stdout = stdout[0]
 852             if stderr is not None:
 853                 stderr = stderr[0]
 854
 855             # Translate newlines, if requested.  We cannot let the file
 856             # object do the translation: It is based on stdio, which is
 857             # impossible to combine with select (unless forcing no
 858             # buffering).
 859             if self.universal_newlines and hasattr(open, 'newlines'):
 860                 if stdout:
 861                     stdout = self._translate_newlines(stdout)
 862                 if stderr:
 863                     stderr = self._translate_newlines(stderr)
 864
 865             self.wait()
 866             return (stdout, stderr)
 867
 868     else:
 869         #
 870         # POSIX methods
 871         #
 872         def _get_handles(self, stdin, stdout, stderr):
 873             """Construct and return tupel with IO objects:
 874             p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite
 875             """
 876             p2cread, p2cwrite = None, None
 877             c2pread, c2pwrite = None, None
 878             errread, errwrite = None, None
 879
 880             if stdin is None:
 881                 pass
 882             elif stdin == PIPE:
 883                 p2cread, p2cwrite = os.pipe()
 884             elif isinstance(stdin, int):
 885                 p2cread = stdin
 886             else:
 887                 # Assuming file-like object
 888                 p2cread = stdin.fileno()
 889
 890             if stdout is None:
 891                 pass
 892             elif stdout == PIPE:
 893                 c2pread, c2pwrite = os.pipe()
 894             elif isinstance(stdout, int):
 895                 c2pwrite = stdout
 896             else:
 897                 # Assuming file-like object
 898                 c2pwrite = stdout.fileno()
 899
 900             if stderr is None:
 901                 pass
 902             elif stderr == PIPE:
 903                 errread, errwrite = os.pipe()
 904             elif stderr == STDOUT:
 905                 errwrite = c2pwrite
 906             elif isinstance(stderr, int):
 907                 errwrite = stderr
 908             else:
 909                 # Assuming file-like object
 910                 errwrite = stderr.fileno()
 911
 912             return (p2cread, p2cwrite,
 913                     c2pread, c2pwrite,
 914                     errread, errwrite)
 915
 916
 917         def _set_cloexec_flag(self, fd):
 918             try:
 919                 cloexec_flag = fcntl.FD_CLOEXEC
 920             except AttributeError:
 921                 cloexec_flag = 1
 922
 923             old = fcntl.fcntl(fd, fcntl.F_GETFD)
 924             fcntl.fcntl(fd, fcntl.F_SETFD, old | cloexec_flag)
 925
 926
 927         def _close_fds(self, but):
 928             for i in range(3, MAXFD):
 929                 if i == but:
 930                     continue
 931                 try:
 932                     os.close(i)
 933                 except:
 934                     pass
 935
 936
 937         def _execute_child(self, args, executable, preexec_fn, close_fds,
 938                            cwd, env, universal_newlines,
 939                            startupinfo, creationflags, shell,
 940                            p2cread, p2cwrite,
 941                            c2pread, c2pwrite,
 942                            errread, errwrite):
 943             """Execute program (POSIX version)"""
 944
 945             if isinstance(args, types.StringTypes):
 946                 args = [args]
 947
 948             if shell:
 949                 args = ["/bin/sh", "-c"] + args
 950
 951             if executable is None:
 952                 executable = args[0]
 953
 954             # For transferring possible exec failure from child to parent
 955             # The first char specifies the exception type: 0 means
 956             # OSError, 1 means some other error.
 957             errpipe_read, errpipe_write = os.pipe()
 958             self._set_cloexec_flag(errpipe_write)
 959
 960             self.pid = os.fork()
 961             if self.pid == 0:
 962                 # Child
 963                 try:
 964                     # Close parent's pipe ends
 965                     if p2cwrite:
 966                         os.close(p2cwrite)
 967                     if c2pread:
 968                         os.close(c2pread)
 969                     if errread:
 970                         os.close(errread)
 971                     os.close(errpipe_read)
 972
 973                     # Dup fds for child
 974                     if p2cread:
 975                         os.dup2(p2cread, 0)
 976                     if c2pwrite:
 977                         os.dup2(c2pwrite, 1)
 978                     if errwrite:
 979                         os.dup2(errwrite, 2)
 980
 981                     # Close pipe fds.  Make sure we doesn't close the same
 982                     # fd more than once.
 983                     if p2cread:
 984                         os.close(p2cread)
 985                     if c2pwrite and c2pwrite not in (p2cread,):
 986                         os.close(c2pwrite)
 987                     if errwrite and errwrite not in (p2cread, c2pwrite):
 988                         os.close(errwrite)
 989
 990                     # Close all other fds, if asked for
 991                     if close_fds:
 992                         self._close_fds(but=errpipe_write)
 993
 994                     if cwd is not None:
 995                         os.chdir(cwd)
 996
 997                     if preexec_fn:
 998                         apply(preexec_fn)
 999
1000                     if env is None:
1001                         os.execvp(executable, args)
1002                     else:
1003                         os.execvpe(executable, args, env)
1004
1005                 except:
1006                     exc_type, exc_value, tb = sys.exc_info()
1007                     # Save the traceback and attach it to the exception object
1008                     exc_lines = traceback.format_exception(exc_type,
1009                                                            exc_value,
1010                                                            tb)
1011                     exc_value.child_traceback = ''.join(exc_lines)
1012                     os.write(errpipe_write, pickle.dumps(exc_value))
1013
1014                 # This exitcode won't be reported to applications, so it
1015                 # really doesn't matter what we return.
1016                 os._exit(255)
1017
1018             # Parent
1019             os.close(errpipe_write)
1020             if p2cread and p2cwrite:
1021                 os.close(p2cread)
1022             if c2pwrite and c2pread:
1023                 os.close(c2pwrite)
1024             if errwrite and errread:
1025                 os.close(errwrite)
1026
1027             # Wait for exec to fail or succeed; possibly raising exception
1028             data = os.read(errpipe_read, 1048576) # Exceptions limited to 1 MB
1029             os.close(errpipe_read)
1030             if data != "":
1031                 os.waitpid(self.pid, 0)
1032                 child_exception = pickle.loads(data)
1033                 raise child_exception
1034
1035
1036         def _handle_exitstatus(self, sts):
1037             if os.WIFSIGNALED(sts):
1038                 self.returncode = -os.WTERMSIG(sts)
1039             elif os.WIFEXITED(sts):
1040                 self.returncode = os.WEXITSTATUS(sts)
1041             else:
1042                 # Should never happen
1043                 raise RuntimeError("Unknown child exit status!")
1044
1045             _active.remove(self)
1046
1047
1048         def poll(self):
1049             """Check if child process has terminated.  Returns returncode
1050             attribute."""
1051             if self.returncode is None:
1052                 try:
1053                     pid, sts = os.waitpid(self.pid, os.WNOHANG)
1054                     if pid == self.pid:
1055                         self._handle_exitstatus(sts)
1056                 except os.error:
1057                     pass
1058             return self.returncode
1059
1060
1061         def wait(self):
1062             """Wait for child process to terminate.  Returns returncode
1063             attribute."""
1064             if self.returncode is None:
1065                 pid, sts = os.waitpid(self.pid, 0)
1066                 self._handle_exitstatus(sts)
1067             return self.returncode
1068
1069
1070         def _communicate(self, input):
1071             read_set = []
1072             write_set = []
1073             stdout = None # Return
1074             stderr = None # Return
1075
1076             if self.stdin:
1077                 # Flush stdio buffer.  This might block, if the user has
1078                 # been writing to .stdin in an uncontrolled fashion.
1079                 self.stdin.flush()
1080                 if input:
1081                     write_set.append(self.stdin)
1082                 else:
1083                     self.stdin.close()
1084             if self.stdout:
1085                 read_set.append(self.stdout)
1086                 stdout = []
1087             if self.stderr:
1088                 read_set.append(self.stderr)
1089                 stderr = []
1090
1091             while read_set or write_set:
1092                 rlist, wlist, xlist = select.select(read_set, write_set, [])
1093
1094                 if self.stdin in wlist:
1095                     # When select has indicated that the file is writable,
1096                     # we can write up to PIPE_BUF bytes without risk
1097                     # blocking.  POSIX defines PIPE_BUF >= 512
1098                     bytes_written = os.write(self.stdin.fileno(), input[:512])
1099                     input = input[bytes_written:]
1100                     if not input:
1101                         self.stdin.close()
1102                         write_set.remove(self.stdin)
1103
1104                 if self.stdout in rlist:
1105                     data = os.read(self.stdout.fileno(), 1024)
1106                     if data == "":
1107                         self.stdout.close()
1108                         read_set.remove(self.stdout)
1109                     stdout.append(data)
1110
1111                 if self.stderr in rlist:
1112                     data = os.read(self.stderr.fileno(), 1024)
1113                     if data == "":
1114                         self.stderr.close()
1115                         read_set.remove(self.stderr)
1116                     stderr.append(data)
1117
1118             # All data exchanged.  Translate lists into strings.
1119             if stdout is not None:
1120                 stdout = ''.join(stdout)
1121             if stderr is not None:
1122                 stderr = ''.join(stderr)
1123
1124             # Translate newlines, if requested.  We cannot let the file
1125             # object do the translation: It is based on stdio, which is
1126             # impossible to combine with select (unless forcing no
1127             # buffering).
1128             if self.universal_newlines and hasattr(open, 'newlines'):
1129                 if stdout:
1130                     stdout = self._translate_newlines(stdout)
1131                 if stderr:
1132                     stderr = self._translate_newlines(stderr)
1133
1134             self.wait()
1135             return (stdout, stderr)
1136
1137
1138 def _demo_posix():
1139     #
1140     # Example 1: Simple redirection: Get process list
1141     #
1142     plist = Popen(["ps"], stdout=PIPE).communicate()[0]
1143     print "Process list:"
1144     print plist
1145
1146     #
1147     # Example 2: Change uid before executing child
1148     #
1149     if os.getuid() == 0:
1150         p = Popen(["id"], preexec_fn=lambda: os.setuid(100))
1151         p.wait()
1152
1153     #
1154     # Example 3: Connecting several subprocesses
1155     #
1156     print "Looking for 'hda'..."
1157     p1 = Popen(["dmesg"], stdout=PIPE)
1158     p2 = Popen(["grep", "hda"], stdin=p1.stdout, stdout=PIPE)
1159     print repr(p2.communicate()[0])
1160
1161     #
1162     # Example 4: Catch execution error
1163     #
1164     print
1165     print "Trying a weird file..."
1166     try:
1167         print Popen(["/this/path/does/not/exist"]).communicate()
1168     except OSError, e:
1169         if e.errno == errno.ENOENT:
1170             print "The file didn't exist.  I thought so..."
1171             print "Child traceback:"
1172             print e.child_traceback
1173         else:
1174             print "Error", e.errno
1175     else:
1176         print >>sys.stderr, "Gosh.  No error."
1177
1178
1179 def _demo_windows():
1180     #
1181     # Example 1: Connecting several subprocesses
1182     #
1183     print "Looking for 'PROMPT' in set output..."
1184     p1 = Popen("set", stdout=PIPE, shell=True)
1185     p2 = Popen('find "PROMPT"', stdin=p1.stdout, stdout=PIPE)
1186     print repr(p2.communicate()[0])
1187
1188     #
1189     # Example 2: Simple execution of program
1190     #
1191     print "Executing calc..."
1192     p = Popen("calc")
1193     p.wait()
1194
1195
1196 if __name__ == "__main__":
1197     if mswindows:
1198         _demo_windows()
1199     else:
1200         _demo_posix()