Issue #7632: Fix a serious wrong output bug for string -> float conversion.
[python.git] / Lib / subprocess.py
blobf63e719d7b72f75af423ba1a6ccb324be86976eb
1 # subprocess - Subprocesses with accessible I/O streams
3 # For more information about this module, see PEP 324.
5 # This module should remain compatible with Python 2.2, see PEP 291.
7 # Copyright (c) 2003-2005 by Peter Astrand <astrand@lysator.liu.se>
9 # Licensed to PSF under a Contributor Agreement.
10 # See http://www.python.org/2.4/license for licensing details.
12 r"""subprocess - Subprocesses with accessible I/O streams
14 This module allows you to spawn processes, connect to their
15 input/output/error pipes, and obtain their return codes. This module
16 intends to replace several other, older modules and functions, like:
18 os.system
19 os.spawn*
20 os.popen*
21 popen2.*
22 commands.*
24 Information about how the subprocess module can be used to replace these
25 modules and functions can be found below.
29 Using the subprocess module
30 ===========================
31 This module defines one class called Popen:
33 class Popen(args, bufsize=0, executable=None,
34 stdin=None, stdout=None, stderr=None,
35 preexec_fn=None, close_fds=False, shell=False,
36 cwd=None, env=None, universal_newlines=False,
37 startupinfo=None, creationflags=0):
40 Arguments are:
42 args should be a string, or a sequence of program arguments. The
43 program to execute is normally the first item in the args sequence or
44 string, but can be explicitly set by using the executable argument.
46 On UNIX, with shell=False (default): In this case, the Popen class
47 uses os.execvp() to execute the child program. args should normally
48 be a sequence. A string will be treated as a sequence with the string
49 as the only item (the program to execute).
51 On UNIX, with shell=True: If args is a string, it specifies the
52 command string to execute through the shell. If args is a sequence,
53 the first item specifies the command string, and any additional items
54 will be treated as additional shell arguments.
56 On Windows: the Popen class uses CreateProcess() to execute the child
57 program, which operates on strings. If args is a sequence, it will be
58 converted to a string using the list2cmdline method. Please note that
59 not all MS Windows applications interpret the command line the same
60 way: The list2cmdline is designed for applications using the same
61 rules as the MS C runtime.
63 bufsize, if given, has the same meaning as the corresponding argument
64 to the built-in open() function: 0 means unbuffered, 1 means line
65 buffered, any other positive value means use a buffer of
66 (approximately) that size. A negative bufsize means to use the system
67 default, which usually means fully buffered. The default value for
68 bufsize is 0 (unbuffered).
70 stdin, stdout and stderr specify the executed programs' standard
71 input, standard output and standard error file handles, respectively.
72 Valid values are PIPE, an existing file descriptor (a positive
73 integer), an existing file object, and None. PIPE indicates that a
74 new pipe to the child should be created. With None, no redirection
75 will occur; the child's file handles will be inherited from the
76 parent. Additionally, stderr can be STDOUT, which indicates that the
77 stderr data from the applications should be captured into the same
78 file handle as for stdout.
80 If preexec_fn is set to a callable object, this object will be called
81 in the child process just before the child is executed.
83 If close_fds is true, all file descriptors except 0, 1 and 2 will be
84 closed before the child process is executed.
86 if shell is true, the specified command will be executed through the
87 shell.
89 If cwd is not None, the current directory will be changed to cwd
90 before the child is executed.
92 If env is not None, it defines the environment variables for the new
93 process.
95 If universal_newlines is true, the file objects stdout and stderr are
96 opened as a text files, but lines may be terminated by any of '\n',
97 the Unix end-of-line convention, '\r', the Macintosh convention or
98 '\r\n', the Windows convention. All of these external representations
99 are seen as '\n' by the Python program. Note: This feature is only
100 available if Python is built with universal newline support (the
101 default). Also, the newlines attribute of the file objects stdout,
102 stdin and stderr are not updated by the communicate() method.
104 The startupinfo and creationflags, if given, will be passed to the
105 underlying CreateProcess() function. They can specify things such as
106 appearance of the main window and priority for the new process.
107 (Windows only)
110 This module also defines some shortcut functions:
112 call(*popenargs, **kwargs):
113 Run command with arguments. Wait for command to complete, then
114 return the returncode attribute.
116 The arguments are the same as for the Popen constructor. Example:
118 retcode = call(["ls", "-l"])
120 check_call(*popenargs, **kwargs):
121 Run command with arguments. Wait for command to complete. If the
122 exit code was zero then return, otherwise raise
123 CalledProcessError. The CalledProcessError object will have the
124 return code in the returncode attribute.
126 The arguments are the same as for the Popen constructor. Example:
128 check_call(["ls", "-l"])
130 check_output(*popenargs, **kwargs):
131 Run command with arguments and return its output as a byte string.
133 If the exit code was non-zero it raises a CalledProcessError. The
134 CalledProcessError object will have the return code in the returncode
135 attribute and output in the output attribute.
137 The arguments are the same as for the Popen constructor. Example:
139 output = check_output(["ls", "-l", "/dev/null"])
142 Exceptions
143 ----------
144 Exceptions raised in the child process, before the new program has
145 started to execute, will be re-raised in the parent. Additionally,
146 the exception object will have one extra attribute called
147 'child_traceback', which is a string containing traceback information
148 from the childs point of view.
150 The most common exception raised is OSError. This occurs, for
151 example, when trying to execute a non-existent file. Applications
152 should prepare for OSErrors.
154 A ValueError will be raised if Popen is called with invalid arguments.
156 check_call() and check_output() will raise CalledProcessError, if the
157 called process returns a non-zero return code.
160 Security
161 --------
162 Unlike some other popen functions, this implementation will never call
163 /bin/sh implicitly. This means that all characters, including shell
164 metacharacters, can safely be passed to child processes.
167 Popen objects
168 =============
169 Instances of the Popen class have the following methods:
171 poll()
172 Check if child process has terminated. Returns returncode
173 attribute.
175 wait()
176 Wait for child process to terminate. Returns returncode attribute.
178 communicate(input=None)
179 Interact with process: Send data to stdin. Read data from stdout
180 and stderr, until end-of-file is reached. Wait for process to
181 terminate. The optional input argument should be a string to be
182 sent to the child process, or None, if no data should be sent to
183 the child.
185 communicate() returns a tuple (stdout, stderr).
187 Note: The data read is buffered in memory, so do not use this
188 method if the data size is large or unlimited.
190 The following attributes are also available:
192 stdin
193 If the stdin argument is PIPE, this attribute is a file object
194 that provides input to the child process. Otherwise, it is None.
196 stdout
197 If the stdout argument is PIPE, this attribute is a file object
198 that provides output from the child process. Otherwise, it is
199 None.
201 stderr
202 If the stderr argument is PIPE, this attribute is file object that
203 provides error output from the child process. Otherwise, it is
204 None.
207 The process ID of the child process.
209 returncode
210 The child return code. A None value indicates that the process
211 hasn't terminated yet. A negative value -N indicates that the
212 child was terminated by signal N (UNIX only).
215 Replacing older functions with the subprocess module
216 ====================================================
217 In this section, "a ==> b" means that b can be used as a replacement
218 for a.
220 Note: All functions in this section fail (more or less) silently if
221 the executed program cannot be found; this module raises an OSError
222 exception.
224 In the following examples, we assume that the subprocess module is
225 imported with "from subprocess import *".
228 Replacing /bin/sh shell backquote
229 ---------------------------------
230 output=`mycmd myarg`
232 output = Popen(["mycmd", "myarg"], stdout=PIPE).communicate()[0]
235 Replacing shell pipe line
236 -------------------------
237 output=`dmesg | grep hda`
239 p1 = Popen(["dmesg"], stdout=PIPE)
240 p2 = Popen(["grep", "hda"], stdin=p1.stdout, stdout=PIPE)
241 output = p2.communicate()[0]
244 Replacing os.system()
245 ---------------------
246 sts = os.system("mycmd" + " myarg")
248 p = Popen("mycmd" + " myarg", shell=True)
249 pid, sts = os.waitpid(p.pid, 0)
251 Note:
253 * Calling the program through the shell is usually not required.
255 * It's easier to look at the returncode attribute than the
256 exitstatus.
258 A more real-world example would look like this:
260 try:
261 retcode = call("mycmd" + " myarg", shell=True)
262 if retcode < 0:
263 print >>sys.stderr, "Child was terminated by signal", -retcode
264 else:
265 print >>sys.stderr, "Child returned", retcode
266 except OSError, e:
267 print >>sys.stderr, "Execution failed:", e
270 Replacing os.spawn*
271 -------------------
272 P_NOWAIT example:
274 pid = os.spawnlp(os.P_NOWAIT, "/bin/mycmd", "mycmd", "myarg")
276 pid = Popen(["/bin/mycmd", "myarg"]).pid
279 P_WAIT example:
281 retcode = os.spawnlp(os.P_WAIT, "/bin/mycmd", "mycmd", "myarg")
283 retcode = call(["/bin/mycmd", "myarg"])
286 Vector example:
288 os.spawnvp(os.P_NOWAIT, path, args)
290 Popen([path] + args[1:])
293 Environment example:
295 os.spawnlpe(os.P_NOWAIT, "/bin/mycmd", "mycmd", "myarg", env)
297 Popen(["/bin/mycmd", "myarg"], env={"PATH": "/usr/bin"})
300 Replacing os.popen*
301 -------------------
302 pipe = os.popen("cmd", mode='r', bufsize)
304 pipe = Popen("cmd", shell=True, bufsize=bufsize, stdout=PIPE).stdout
306 pipe = os.popen("cmd", mode='w', bufsize)
308 pipe = Popen("cmd", shell=True, bufsize=bufsize, stdin=PIPE).stdin
311 (child_stdin, child_stdout) = os.popen2("cmd", mode, bufsize)
313 p = Popen("cmd", shell=True, bufsize=bufsize,
314 stdin=PIPE, stdout=PIPE, close_fds=True)
315 (child_stdin, child_stdout) = (p.stdin, p.stdout)
318 (child_stdin,
319 child_stdout,
320 child_stderr) = os.popen3("cmd", mode, bufsize)
322 p = Popen("cmd", shell=True, bufsize=bufsize,
323 stdin=PIPE, stdout=PIPE, stderr=PIPE, close_fds=True)
324 (child_stdin,
325 child_stdout,
326 child_stderr) = (p.stdin, p.stdout, p.stderr)
329 (child_stdin, child_stdout_and_stderr) = os.popen4("cmd", mode,
330 bufsize)
332 p = Popen("cmd", shell=True, bufsize=bufsize,
333 stdin=PIPE, stdout=PIPE, stderr=STDOUT, close_fds=True)
334 (child_stdin, child_stdout_and_stderr) = (p.stdin, p.stdout)
336 On Unix, os.popen2, os.popen3 and os.popen4 also accept a sequence as
337 the command to execute, in which case arguments will be passed
338 directly to the program without shell intervention. This usage can be
339 replaced as follows:
341 (child_stdin, child_stdout) = os.popen2(["/bin/ls", "-l"], mode,
342 bufsize)
344 p = Popen(["/bin/ls", "-l"], bufsize=bufsize, stdin=PIPE, stdout=PIPE)
345 (child_stdin, child_stdout) = (p.stdin, p.stdout)
347 Return code handling translates as follows:
349 pipe = os.popen("cmd", 'w')
351 rc = pipe.close()
352 if rc != None and rc % 256:
353 print "There were some errors"
355 process = Popen("cmd", 'w', shell=True, stdin=PIPE)
357 process.stdin.close()
358 if process.wait() != 0:
359 print "There were some errors"
362 Replacing popen2.*
363 ------------------
364 (child_stdout, child_stdin) = popen2.popen2("somestring", bufsize, mode)
366 p = Popen(["somestring"], shell=True, bufsize=bufsize
367 stdin=PIPE, stdout=PIPE, close_fds=True)
368 (child_stdout, child_stdin) = (p.stdout, p.stdin)
370 On Unix, popen2 also accepts a sequence as the command to execute, in
371 which case arguments will be passed directly to the program without
372 shell intervention. This usage can be replaced as follows:
374 (child_stdout, child_stdin) = popen2.popen2(["mycmd", "myarg"], bufsize,
375 mode)
377 p = Popen(["mycmd", "myarg"], bufsize=bufsize,
378 stdin=PIPE, stdout=PIPE, close_fds=True)
379 (child_stdout, child_stdin) = (p.stdout, p.stdin)
381 The popen2.Popen3 and popen2.Popen4 basically works as subprocess.Popen,
382 except that:
384 * subprocess.Popen raises an exception if the execution fails
385 * the capturestderr argument is replaced with the stderr argument.
386 * stdin=PIPE and stdout=PIPE must be specified.
387 * popen2 closes all filedescriptors by default, but you have to specify
388 close_fds=True with subprocess.Popen.
391 import sys
392 mswindows = (sys.platform == "win32")
394 import os
395 import types
396 import traceback
397 import gc
398 import signal
400 # Exception classes used by this module.
401 class CalledProcessError(Exception):
402 """This exception is raised when a process run by check_call() or
403 check_output() returns a non-zero exit status.
404 The exit status will be stored in the returncode attribute;
405 check_output() will also store the output in the output attribute.
407 def __init__(self, returncode, cmd, output=None):
408 self.returncode = returncode
409 self.cmd = cmd
410 self.output = output
411 def __str__(self):
412 return "Command '%s' returned non-zero exit status %d" % (self.cmd, self.returncode)
415 if mswindows:
416 import threading
417 import msvcrt
418 if 0: # <-- change this to use pywin32 instead of the _subprocess driver
419 import pywintypes
420 from win32api import GetStdHandle, STD_INPUT_HANDLE, \
421 STD_OUTPUT_HANDLE, STD_ERROR_HANDLE
422 from win32api import GetCurrentProcess, DuplicateHandle, \
423 GetModuleFileName, GetVersion
424 from win32con import DUPLICATE_SAME_ACCESS, SW_HIDE
425 from win32pipe import CreatePipe
426 from win32process import CreateProcess, STARTUPINFO, \
427 GetExitCodeProcess, STARTF_USESTDHANDLES, \
428 STARTF_USESHOWWINDOW, CREATE_NEW_CONSOLE
429 from win32process import TerminateProcess
430 from win32event import WaitForSingleObject, INFINITE, WAIT_OBJECT_0
431 else:
432 from _subprocess import *
433 class STARTUPINFO:
434 dwFlags = 0
435 hStdInput = None
436 hStdOutput = None
437 hStdError = None
438 wShowWindow = 0
439 class pywintypes:
440 error = IOError
441 else:
442 import select
443 _has_poll = hasattr(select, 'poll')
444 import errno
445 import fcntl
446 import pickle
448 # When select or poll has indicated that the file is writable,
449 # we can write up to _PIPE_BUF bytes without risk of blocking.
450 # POSIX defines PIPE_BUF as >= 512.
451 _PIPE_BUF = getattr(select, 'PIPE_BUF', 512)
454 __all__ = ["Popen", "PIPE", "STDOUT", "call", "check_call",
455 "check_output", "CalledProcessError"]
457 try:
458 MAXFD = os.sysconf("SC_OPEN_MAX")
459 except:
460 MAXFD = 256
462 _active = []
464 def _cleanup():
465 for inst in _active[:]:
466 res = inst._internal_poll(_deadstate=sys.maxint)
467 if res is not None and res >= 0:
468 try:
469 _active.remove(inst)
470 except ValueError:
471 # This can happen if two threads create a new Popen instance.
472 # It's harmless that it was already removed, so ignore.
473 pass
475 PIPE = -1
476 STDOUT = -2
479 def call(*popenargs, **kwargs):
480 """Run command with arguments. Wait for command to complete, then
481 return the returncode attribute.
483 The arguments are the same as for the Popen constructor. Example:
485 retcode = call(["ls", "-l"])
487 return Popen(*popenargs, **kwargs).wait()
490 def check_call(*popenargs, **kwargs):
491 """Run command with arguments. Wait for command to complete. If
492 the exit code was zero then return, otherwise raise
493 CalledProcessError. The CalledProcessError object will have the
494 return code in the returncode attribute.
496 The arguments are the same as for the Popen constructor. Example:
498 check_call(["ls", "-l"])
500 retcode = call(*popenargs, **kwargs)
501 if retcode:
502 cmd = kwargs.get("args")
503 if cmd is None:
504 cmd = popenargs[0]
505 raise CalledProcessError(retcode, cmd)
506 return 0
509 def check_output(*popenargs, **kwargs):
510 r"""Run command with arguments and return its output as a byte string.
512 If the exit code was non-zero it raises a CalledProcessError. The
513 CalledProcessError object will have the return code in the returncode
514 attribute and output in the output attribute.
516 The arguments are the same as for the Popen constructor. Example:
518 >>> check_output(["ls", "-l", "/dev/null"])
519 'crw-rw-rw- 1 root root 1, 3 Oct 18 2007 /dev/null\n'
521 The stdout argument is not allowed as it is used internally.
522 To capture standard error in the result, use stderr=STDOUT.
524 >>> check_output(["/bin/sh", "-c",
525 ... "ls -l non_existent_file ; exit 0"],
526 ... stderr=STDOUT)
527 'ls: non_existent_file: No such file or directory\n'
529 if 'stdout' in kwargs:
530 raise ValueError('stdout argument not allowed, it will be overridden.')
531 process = Popen(stdout=PIPE, *popenargs, **kwargs)
532 output, unused_err = process.communicate()
533 retcode = process.poll()
534 if retcode:
535 cmd = kwargs.get("args")
536 if cmd is None:
537 cmd = popenargs[0]
538 raise CalledProcessError(retcode, cmd, output=output)
539 return output
542 def list2cmdline(seq):
544 Translate a sequence of arguments into a command line
545 string, using the same rules as the MS C runtime:
547 1) Arguments are delimited by white space, which is either a
548 space or a tab.
550 2) A string surrounded by double quotation marks is
551 interpreted as a single argument, regardless of white space
552 or pipe characters contained within. A quoted string can be
553 embedded in an argument.
555 3) A double quotation mark preceded by a backslash is
556 interpreted as a literal double quotation mark.
558 4) Backslashes are interpreted literally, unless they
559 immediately precede a double quotation mark.
561 5) If backslashes immediately precede a double quotation mark,
562 every pair of backslashes is interpreted as a literal
563 backslash. If the number of backslashes is odd, the last
564 backslash escapes the next double quotation mark as
565 described in rule 3.
568 # See
569 # http://msdn.microsoft.com/en-us/library/17w5ykft.aspx
570 # or search http://msdn.microsoft.com for
571 # "Parsing C++ Command-Line Arguments"
572 result = []
573 needquote = False
574 for arg in seq:
575 bs_buf = []
577 # Add a space to separate this argument from the others
578 if result:
579 result.append(' ')
581 needquote = (" " in arg) or ("\t" in arg) or ("|" in arg) or not arg
582 if needquote:
583 result.append('"')
585 for c in arg:
586 if c == '\\':
587 # Don't know if we need to double yet.
588 bs_buf.append(c)
589 elif c == '"':
590 # Double backslashes.
591 result.append('\\' * len(bs_buf)*2)
592 bs_buf = []
593 result.append('\\"')
594 else:
595 # Normal char
596 if bs_buf:
597 result.extend(bs_buf)
598 bs_buf = []
599 result.append(c)
601 # Add remaining backslashes, if any.
602 if bs_buf:
603 result.extend(bs_buf)
605 if needquote:
606 result.extend(bs_buf)
607 result.append('"')
609 return ''.join(result)
612 class Popen(object):
613 def __init__(self, args, bufsize=0, executable=None,
614 stdin=None, stdout=None, stderr=None,
615 preexec_fn=None, close_fds=False, shell=False,
616 cwd=None, env=None, universal_newlines=False,
617 startupinfo=None, creationflags=0):
618 """Create new Popen instance."""
619 _cleanup()
621 self._child_created = False
622 if not isinstance(bufsize, (int, long)):
623 raise TypeError("bufsize must be an integer")
625 if mswindows:
626 if preexec_fn is not None:
627 raise ValueError("preexec_fn is not supported on Windows "
628 "platforms")
629 if close_fds and (stdin is not None or stdout is not None or
630 stderr is not None):
631 raise ValueError("close_fds is not supported on Windows "
632 "platforms if you redirect stdin/stdout/stderr")
633 else:
634 # POSIX
635 if startupinfo is not None:
636 raise ValueError("startupinfo is only supported on Windows "
637 "platforms")
638 if creationflags != 0:
639 raise ValueError("creationflags is only supported on Windows "
640 "platforms")
642 self.stdin = None
643 self.stdout = None
644 self.stderr = None
645 self.pid = None
646 self.returncode = None
647 self.universal_newlines = universal_newlines
649 # Input and output objects. The general principle is like
650 # this:
652 # Parent Child
653 # ------ -----
654 # p2cwrite ---stdin---> p2cread
655 # c2pread <--stdout--- c2pwrite
656 # errread <--stderr--- errwrite
658 # On POSIX, the child objects are file descriptors. On
659 # Windows, these are Windows file handles. The parent objects
660 # are file descriptors on both platforms. The parent objects
661 # are None when not using PIPEs. The child objects are None
662 # when not redirecting.
664 (p2cread, p2cwrite,
665 c2pread, c2pwrite,
666 errread, errwrite) = self._get_handles(stdin, stdout, stderr)
668 self._execute_child(args, executable, preexec_fn, close_fds,
669 cwd, env, universal_newlines,
670 startupinfo, creationflags, shell,
671 p2cread, p2cwrite,
672 c2pread, c2pwrite,
673 errread, errwrite)
675 if mswindows:
676 if p2cwrite is not None:
677 p2cwrite = msvcrt.open_osfhandle(p2cwrite.Detach(), 0)
678 if c2pread is not None:
679 c2pread = msvcrt.open_osfhandle(c2pread.Detach(), 0)
680 if errread is not None:
681 errread = msvcrt.open_osfhandle(errread.Detach(), 0)
683 if p2cwrite is not None:
684 self.stdin = os.fdopen(p2cwrite, 'wb', bufsize)
685 if c2pread is not None:
686 if universal_newlines:
687 self.stdout = os.fdopen(c2pread, 'rU', bufsize)
688 else:
689 self.stdout = os.fdopen(c2pread, 'rb', bufsize)
690 if errread is not None:
691 if universal_newlines:
692 self.stderr = os.fdopen(errread, 'rU', bufsize)
693 else:
694 self.stderr = os.fdopen(errread, 'rb', bufsize)
697 def _translate_newlines(self, data):
698 data = data.replace("\r\n", "\n")
699 data = data.replace("\r", "\n")
700 return data
703 def __del__(self, sys=sys):
704 if not self._child_created:
705 # We didn't get to successfully create a child process.
706 return
707 # In case the child hasn't been waited on, check if it's done.
708 self._internal_poll(_deadstate=sys.maxint)
709 if self.returncode is None and _active is not None:
710 # Child is still running, keep us alive until we can wait on it.
711 _active.append(self)
714 def communicate(self, input=None):
715 """Interact with process: Send data to stdin. Read data from
716 stdout and stderr, until end-of-file is reached. Wait for
717 process to terminate. The optional input argument should be a
718 string to be sent to the child process, or None, if no data
719 should be sent to the child.
721 communicate() returns a tuple (stdout, stderr)."""
723 # Optimization: If we are only using one pipe, or no pipe at
724 # all, using select() or threads is unnecessary.
725 if [self.stdin, self.stdout, self.stderr].count(None) >= 2:
726 stdout = None
727 stderr = None
728 if self.stdin:
729 if input:
730 self.stdin.write(input)
731 self.stdin.close()
732 elif self.stdout:
733 stdout = self.stdout.read()
734 self.stdout.close()
735 elif self.stderr:
736 stderr = self.stderr.read()
737 self.stderr.close()
738 self.wait()
739 return (stdout, stderr)
741 return self._communicate(input)
744 def poll(self):
745 return self._internal_poll()
748 if mswindows:
750 # Windows methods
752 def _get_handles(self, stdin, stdout, stderr):
753 """Construct and return tuple with IO objects:
754 p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite
756 if stdin is None and stdout is None and stderr is None:
757 return (None, None, None, None, None, None)
759 p2cread, p2cwrite = None, None
760 c2pread, c2pwrite = None, None
761 errread, errwrite = None, None
763 if stdin is None:
764 p2cread = GetStdHandle(STD_INPUT_HANDLE)
765 if p2cread is None:
766 p2cread, _ = CreatePipe(None, 0)
767 elif stdin == PIPE:
768 p2cread, p2cwrite = CreatePipe(None, 0)
769 elif isinstance(stdin, int):
770 p2cread = msvcrt.get_osfhandle(stdin)
771 else:
772 # Assuming file-like object
773 p2cread = msvcrt.get_osfhandle(stdin.fileno())
774 p2cread = self._make_inheritable(p2cread)
776 if stdout is None:
777 c2pwrite = GetStdHandle(STD_OUTPUT_HANDLE)
778 if c2pwrite is None:
779 _, c2pwrite = CreatePipe(None, 0)
780 elif stdout == PIPE:
781 c2pread, c2pwrite = CreatePipe(None, 0)
782 elif isinstance(stdout, int):
783 c2pwrite = msvcrt.get_osfhandle(stdout)
784 else:
785 # Assuming file-like object
786 c2pwrite = msvcrt.get_osfhandle(stdout.fileno())
787 c2pwrite = self._make_inheritable(c2pwrite)
789 if stderr is None:
790 errwrite = GetStdHandle(STD_ERROR_HANDLE)
791 if errwrite is None:
792 _, errwrite = CreatePipe(None, 0)
793 elif stderr == PIPE:
794 errread, errwrite = CreatePipe(None, 0)
795 elif stderr == STDOUT:
796 errwrite = c2pwrite
797 elif isinstance(stderr, int):
798 errwrite = msvcrt.get_osfhandle(stderr)
799 else:
800 # Assuming file-like object
801 errwrite = msvcrt.get_osfhandle(stderr.fileno())
802 errwrite = self._make_inheritable(errwrite)
804 return (p2cread, p2cwrite,
805 c2pread, c2pwrite,
806 errread, errwrite)
809 def _make_inheritable(self, handle):
810 """Return a duplicate of handle, which is inheritable"""
811 return DuplicateHandle(GetCurrentProcess(), handle,
812 GetCurrentProcess(), 0, 1,
813 DUPLICATE_SAME_ACCESS)
816 def _find_w9xpopen(self):
817 """Find and return absolut path to w9xpopen.exe"""
818 w9xpopen = os.path.join(os.path.dirname(GetModuleFileName(0)),
819 "w9xpopen.exe")
820 if not os.path.exists(w9xpopen):
821 # Eeek - file-not-found - possibly an embedding
822 # situation - see if we can locate it in sys.exec_prefix
823 w9xpopen = os.path.join(os.path.dirname(sys.exec_prefix),
824 "w9xpopen.exe")
825 if not os.path.exists(w9xpopen):
826 raise RuntimeError("Cannot locate w9xpopen.exe, which is "
827 "needed for Popen to work with your "
828 "shell or platform.")
829 return w9xpopen
832 def _execute_child(self, args, executable, preexec_fn, close_fds,
833 cwd, env, universal_newlines,
834 startupinfo, creationflags, shell,
835 p2cread, p2cwrite,
836 c2pread, c2pwrite,
837 errread, errwrite):
838 """Execute program (MS Windows version)"""
840 if not isinstance(args, types.StringTypes):
841 args = list2cmdline(args)
843 # Process startup details
844 if startupinfo is None:
845 startupinfo = STARTUPINFO()
846 if None not in (p2cread, c2pwrite, errwrite):
847 startupinfo.dwFlags |= STARTF_USESTDHANDLES
848 startupinfo.hStdInput = p2cread
849 startupinfo.hStdOutput = c2pwrite
850 startupinfo.hStdError = errwrite
852 if shell:
853 startupinfo.dwFlags |= STARTF_USESHOWWINDOW
854 startupinfo.wShowWindow = SW_HIDE
855 comspec = os.environ.get("COMSPEC", "cmd.exe")
856 args = comspec + " /c " + args
857 if (GetVersion() >= 0x80000000L or
858 os.path.basename(comspec).lower() == "command.com"):
859 # Win9x, or using command.com on NT. We need to
860 # use the w9xpopen intermediate program. For more
861 # information, see KB Q150956
862 # (http://web.archive.org/web/20011105084002/http://support.microsoft.com/support/kb/articles/Q150/9/56.asp)
863 w9xpopen = self._find_w9xpopen()
864 args = '"%s" %s' % (w9xpopen, args)
865 # Not passing CREATE_NEW_CONSOLE has been known to
866 # cause random failures on win9x. Specifically a
867 # dialog: "Your program accessed mem currently in
868 # use at xxx" and a hopeful warning about the
869 # stability of your system. Cost is Ctrl+C wont
870 # kill children.
871 creationflags |= CREATE_NEW_CONSOLE
873 # Start the process
874 try:
875 hp, ht, pid, tid = CreateProcess(executable, args,
876 # no special security
877 None, None,
878 int(not close_fds),
879 creationflags,
880 env,
881 cwd,
882 startupinfo)
883 except pywintypes.error, e:
884 # Translate pywintypes.error to WindowsError, which is
885 # a subclass of OSError. FIXME: We should really
886 # translate errno using _sys_errlist (or simliar), but
887 # how can this be done from Python?
888 raise WindowsError(*e.args)
890 # Retain the process handle, but close the thread handle
891 self._child_created = True
892 self._handle = hp
893 self.pid = pid
894 ht.Close()
896 # Child is launched. Close the parent's copy of those pipe
897 # handles that only the child should have open. You need
898 # to make sure that no handles to the write end of the
899 # output pipe are maintained in this process or else the
900 # pipe will not close when the child process exits and the
901 # ReadFile will hang.
902 if p2cread is not None:
903 p2cread.Close()
904 if c2pwrite is not None:
905 c2pwrite.Close()
906 if errwrite is not None:
907 errwrite.Close()
910 def _internal_poll(self, _deadstate=None):
911 """Check if child process has terminated. Returns returncode
912 attribute."""
913 if self.returncode is None:
914 if WaitForSingleObject(self._handle, 0) == WAIT_OBJECT_0:
915 self.returncode = GetExitCodeProcess(self._handle)
916 return self.returncode
919 def wait(self):
920 """Wait for child process to terminate. Returns returncode
921 attribute."""
922 if self.returncode is None:
923 obj = WaitForSingleObject(self._handle, INFINITE)
924 self.returncode = GetExitCodeProcess(self._handle)
925 return self.returncode
928 def _readerthread(self, fh, buffer):
929 buffer.append(fh.read())
932 def _communicate(self, input):
933 stdout = None # Return
934 stderr = None # Return
936 if self.stdout:
937 stdout = []
938 stdout_thread = threading.Thread(target=self._readerthread,
939 args=(self.stdout, stdout))
940 stdout_thread.setDaemon(True)
941 stdout_thread.start()
942 if self.stderr:
943 stderr = []
944 stderr_thread = threading.Thread(target=self._readerthread,
945 args=(self.stderr, stderr))
946 stderr_thread.setDaemon(True)
947 stderr_thread.start()
949 if self.stdin:
950 if input is not None:
951 self.stdin.write(input)
952 self.stdin.close()
954 if self.stdout:
955 stdout_thread.join()
956 if self.stderr:
957 stderr_thread.join()
959 # All data exchanged. Translate lists into strings.
960 if stdout is not None:
961 stdout = stdout[0]
962 if stderr is not None:
963 stderr = stderr[0]
965 # Translate newlines, if requested. We cannot let the file
966 # object do the translation: It is based on stdio, which is
967 # impossible to combine with select (unless forcing no
968 # buffering).
969 if self.universal_newlines and hasattr(file, 'newlines'):
970 if stdout:
971 stdout = self._translate_newlines(stdout)
972 if stderr:
973 stderr = self._translate_newlines(stderr)
975 self.wait()
976 return (stdout, stderr)
978 def send_signal(self, sig):
979 """Send a signal to the process
981 if sig == signal.SIGTERM:
982 self.terminate()
983 else:
984 raise ValueError("Only SIGTERM is supported on Windows")
986 def terminate(self):
987 """Terminates the process
989 TerminateProcess(self._handle, 1)
991 kill = terminate
993 else:
995 # POSIX methods
997 def _get_handles(self, stdin, stdout, stderr):
998 """Construct and return tuple with IO objects:
999 p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite
1001 p2cread, p2cwrite = None, None
1002 c2pread, c2pwrite = None, None
1003 errread, errwrite = None, None
1005 if stdin is None:
1006 pass
1007 elif stdin == PIPE:
1008 p2cread, p2cwrite = os.pipe()
1009 elif isinstance(stdin, int):
1010 p2cread = stdin
1011 else:
1012 # Assuming file-like object
1013 p2cread = stdin.fileno()
1015 if stdout is None:
1016 pass
1017 elif stdout == PIPE:
1018 c2pread, c2pwrite = os.pipe()
1019 elif isinstance(stdout, int):
1020 c2pwrite = stdout
1021 else:
1022 # Assuming file-like object
1023 c2pwrite = stdout.fileno()
1025 if stderr is None:
1026 pass
1027 elif stderr == PIPE:
1028 errread, errwrite = os.pipe()
1029 elif stderr == STDOUT:
1030 errwrite = c2pwrite
1031 elif isinstance(stderr, int):
1032 errwrite = stderr
1033 else:
1034 # Assuming file-like object
1035 errwrite = stderr.fileno()
1037 return (p2cread, p2cwrite,
1038 c2pread, c2pwrite,
1039 errread, errwrite)
1042 def _set_cloexec_flag(self, fd):
1043 try:
1044 cloexec_flag = fcntl.FD_CLOEXEC
1045 except AttributeError:
1046 cloexec_flag = 1
1048 old = fcntl.fcntl(fd, fcntl.F_GETFD)
1049 fcntl.fcntl(fd, fcntl.F_SETFD, old | cloexec_flag)
1052 def _close_fds(self, but):
1053 if hasattr(os, 'closerange'):
1054 os.closerange(3, but)
1055 os.closerange(but + 1, MAXFD)
1056 else:
1057 for i in xrange(3, MAXFD):
1058 if i == but:
1059 continue
1060 try:
1061 os.close(i)
1062 except:
1063 pass
1066 def _execute_child(self, args, executable, preexec_fn, close_fds,
1067 cwd, env, universal_newlines,
1068 startupinfo, creationflags, shell,
1069 p2cread, p2cwrite,
1070 c2pread, c2pwrite,
1071 errread, errwrite):
1072 """Execute program (POSIX version)"""
1074 if isinstance(args, types.StringTypes):
1075 args = [args]
1076 else:
1077 args = list(args)
1079 if shell:
1080 args = ["/bin/sh", "-c"] + args
1082 if executable is None:
1083 executable = args[0]
1085 # For transferring possible exec failure from child to parent
1086 # The first char specifies the exception type: 0 means
1087 # OSError, 1 means some other error.
1088 errpipe_read, errpipe_write = os.pipe()
1089 try:
1090 try:
1091 self._set_cloexec_flag(errpipe_write)
1093 gc_was_enabled = gc.isenabled()
1094 # Disable gc to avoid bug where gc -> file_dealloc ->
1095 # write to stderr -> hang. http://bugs.python.org/issue1336
1096 gc.disable()
1097 try:
1098 self.pid = os.fork()
1099 except:
1100 if gc_was_enabled:
1101 gc.enable()
1102 raise
1103 self._child_created = True
1104 if self.pid == 0:
1105 # Child
1106 try:
1107 # Close parent's pipe ends
1108 if p2cwrite is not None:
1109 os.close(p2cwrite)
1110 if c2pread is not None:
1111 os.close(c2pread)
1112 if errread is not None:
1113 os.close(errread)
1114 os.close(errpipe_read)
1116 # Dup fds for child
1117 if p2cread is not None:
1118 os.dup2(p2cread, 0)
1119 if c2pwrite is not None:
1120 os.dup2(c2pwrite, 1)
1121 if errwrite is not None:
1122 os.dup2(errwrite, 2)
1124 # Close pipe fds. Make sure we don't close the same
1125 # fd more than once, or standard fds.
1126 if p2cread is not None and p2cread not in (0,):
1127 os.close(p2cread)
1128 if c2pwrite is not None and c2pwrite not in (p2cread, 1):
1129 os.close(c2pwrite)
1130 if errwrite is not None and errwrite not in (p2cread, c2pwrite, 2):
1131 os.close(errwrite)
1133 # Close all other fds, if asked for
1134 if close_fds:
1135 self._close_fds(but=errpipe_write)
1137 if cwd is not None:
1138 os.chdir(cwd)
1140 if preexec_fn:
1141 preexec_fn()
1143 if env is None:
1144 os.execvp(executable, args)
1145 else:
1146 os.execvpe(executable, args, env)
1148 except:
1149 exc_type, exc_value, tb = sys.exc_info()
1150 # Save the traceback and attach it to the exception object
1151 exc_lines = traceback.format_exception(exc_type,
1152 exc_value,
1154 exc_value.child_traceback = ''.join(exc_lines)
1155 os.write(errpipe_write, pickle.dumps(exc_value))
1157 # This exitcode won't be reported to applications, so it
1158 # really doesn't matter what we return.
1159 os._exit(255)
1161 # Parent
1162 if gc_was_enabled:
1163 gc.enable()
1164 finally:
1165 # be sure the FD is closed no matter what
1166 os.close(errpipe_write)
1168 if p2cread is not None and p2cwrite is not None:
1169 os.close(p2cread)
1170 if c2pwrite is not None and c2pread is not None:
1171 os.close(c2pwrite)
1172 if errwrite is not None and errread is not None:
1173 os.close(errwrite)
1175 # Wait for exec to fail or succeed; possibly raising exception
1176 data = os.read(errpipe_read, 1048576) # Exception limited to 1M
1177 finally:
1178 # be sure the FD is closed no matter what
1179 os.close(errpipe_read)
1181 if data != "":
1182 os.waitpid(self.pid, 0)
1183 child_exception = pickle.loads(data)
1184 for fd in (p2cwrite, c2pread, errread):
1185 if fd is not None:
1186 os.close(fd)
1187 raise child_exception
1190 def _handle_exitstatus(self, sts):
1191 if os.WIFSIGNALED(sts):
1192 self.returncode = -os.WTERMSIG(sts)
1193 elif os.WIFEXITED(sts):
1194 self.returncode = os.WEXITSTATUS(sts)
1195 else:
1196 # Should never happen
1197 raise RuntimeError("Unknown child exit status!")
1200 def _internal_poll(self, _deadstate=None):
1201 """Check if child process has terminated. Returns returncode
1202 attribute."""
1203 if self.returncode is None:
1204 try:
1205 pid, sts = os.waitpid(self.pid, os.WNOHANG)
1206 if pid == self.pid:
1207 self._handle_exitstatus(sts)
1208 except os.error:
1209 if _deadstate is not None:
1210 self.returncode = _deadstate
1211 return self.returncode
1214 def wait(self):
1215 """Wait for child process to terminate. Returns returncode
1216 attribute."""
1217 if self.returncode is None:
1218 pid, sts = os.waitpid(self.pid, 0)
1219 self._handle_exitstatus(sts)
1220 return self.returncode
1223 def _communicate(self, input):
1224 if self.stdin:
1225 # Flush stdio buffer. This might block, if the user has
1226 # been writing to .stdin in an uncontrolled fashion.
1227 self.stdin.flush()
1228 if not input:
1229 self.stdin.close()
1231 if _has_poll:
1232 stdout, stderr = self._communicate_with_poll(input)
1233 else:
1234 stdout, stderr = self._communicate_with_select(input)
1236 # All data exchanged. Translate lists into strings.
1237 if stdout is not None:
1238 stdout = ''.join(stdout)
1239 if stderr is not None:
1240 stderr = ''.join(stderr)
1242 # Translate newlines, if requested. We cannot let the file
1243 # object do the translation: It is based on stdio, which is
1244 # impossible to combine with select (unless forcing no
1245 # buffering).
1246 if self.universal_newlines and hasattr(file, 'newlines'):
1247 if stdout:
1248 stdout = self._translate_newlines(stdout)
1249 if stderr:
1250 stderr = self._translate_newlines(stderr)
1252 self.wait()
1253 return (stdout, stderr)
1256 def _communicate_with_poll(self, input):
1257 stdout = None # Return
1258 stderr = None # Return
1259 fd2file = {}
1260 fd2output = {}
1262 poller = select.poll()
1263 def register_and_append(file_obj, eventmask):
1264 poller.register(file_obj.fileno(), eventmask)
1265 fd2file[file_obj.fileno()] = file_obj
1267 def close_unregister_and_remove(fd):
1268 poller.unregister(fd)
1269 fd2file[fd].close()
1270 fd2file.pop(fd)
1272 if self.stdin and input:
1273 register_and_append(self.stdin, select.POLLOUT)
1275 select_POLLIN_POLLPRI = select.POLLIN | select.POLLPRI
1276 if self.stdout:
1277 register_and_append(self.stdout, select_POLLIN_POLLPRI)
1278 fd2output[self.stdout.fileno()] = stdout = []
1279 if self.stderr:
1280 register_and_append(self.stderr, select_POLLIN_POLLPRI)
1281 fd2output[self.stderr.fileno()] = stderr = []
1283 input_offset = 0
1284 while fd2file:
1285 try:
1286 ready = poller.poll()
1287 except select.error, e:
1288 if e.args[0] == errno.EINTR:
1289 continue
1290 raise
1292 for fd, mode in ready:
1293 if mode & select.POLLOUT:
1294 chunk = input[input_offset : input_offset + _PIPE_BUF]
1295 input_offset += os.write(fd, chunk)
1296 if input_offset >= len(input):
1297 close_unregister_and_remove(fd)
1298 elif mode & select_POLLIN_POLLPRI:
1299 data = os.read(fd, 4096)
1300 if not data:
1301 close_unregister_and_remove(fd)
1302 fd2output[fd].append(data)
1303 else:
1304 # Ignore hang up or errors.
1305 close_unregister_and_remove(fd)
1307 return (stdout, stderr)
1310 def _communicate_with_select(self, input):
1311 read_set = []
1312 write_set = []
1313 stdout = None # Return
1314 stderr = None # Return
1316 if self.stdin and input:
1317 write_set.append(self.stdin)
1318 if self.stdout:
1319 read_set.append(self.stdout)
1320 stdout = []
1321 if self.stderr:
1322 read_set.append(self.stderr)
1323 stderr = []
1325 input_offset = 0
1326 while read_set or write_set:
1327 try:
1328 rlist, wlist, xlist = select.select(read_set, write_set, [])
1329 except select.error, e:
1330 if e.args[0] == errno.EINTR:
1331 continue
1332 raise
1334 if self.stdin in wlist:
1335 chunk = input[input_offset : input_offset + _PIPE_BUF]
1336 bytes_written = os.write(self.stdin.fileno(), chunk)
1337 input_offset += bytes_written
1338 if input_offset >= len(input):
1339 self.stdin.close()
1340 write_set.remove(self.stdin)
1342 if self.stdout in rlist:
1343 data = os.read(self.stdout.fileno(), 1024)
1344 if data == "":
1345 self.stdout.close()
1346 read_set.remove(self.stdout)
1347 stdout.append(data)
1349 if self.stderr in rlist:
1350 data = os.read(self.stderr.fileno(), 1024)
1351 if data == "":
1352 self.stderr.close()
1353 read_set.remove(self.stderr)
1354 stderr.append(data)
1356 return (stdout, stderr)
1359 def send_signal(self, sig):
1360 """Send a signal to the process
1362 os.kill(self.pid, sig)
1364 def terminate(self):
1365 """Terminate the process with SIGTERM
1367 self.send_signal(signal.SIGTERM)
1369 def kill(self):
1370 """Kill the process with SIGKILL
1372 self.send_signal(signal.SIGKILL)
1375 def _demo_posix():
1377 # Example 1: Simple redirection: Get process list
1379 plist = Popen(["ps"], stdout=PIPE).communicate()[0]
1380 print "Process list:"
1381 print plist
1384 # Example 2: Change uid before executing child
1386 if os.getuid() == 0:
1387 p = Popen(["id"], preexec_fn=lambda: os.setuid(100))
1388 p.wait()
1391 # Example 3: Connecting several subprocesses
1393 print "Looking for 'hda'..."
1394 p1 = Popen(["dmesg"], stdout=PIPE)
1395 p2 = Popen(["grep", "hda"], stdin=p1.stdout, stdout=PIPE)
1396 print repr(p2.communicate()[0])
1399 # Example 4: Catch execution error
1401 print
1402 print "Trying a weird file..."
1403 try:
1404 print Popen(["/this/path/does/not/exist"]).communicate()
1405 except OSError, e:
1406 if e.errno == errno.ENOENT:
1407 print "The file didn't exist. I thought so..."
1408 print "Child traceback:"
1409 print e.child_traceback
1410 else:
1411 print "Error", e.errno
1412 else:
1413 print >>sys.stderr, "Gosh. No error."
1416 def _demo_windows():
1418 # Example 1: Connecting several subprocesses
1420 print "Looking for 'PROMPT' in set output..."
1421 p1 = Popen("set", stdout=PIPE, shell=True)
1422 p2 = Popen('find "PROMPT"', stdin=p1.stdout, stdout=PIPE)
1423 print repr(p2.communicate()[0])
1426 # Example 2: Simple execution of program
1428 print "Executing calc..."
1429 p = Popen("calc")
1430 p.wait()
1433 if __name__ == "__main__":
1434 if mswindows:
1435 _demo_windows()
1436 else:
1437 _demo_posix()