cola/core.py

   1 """This module provides core functions for handling unicode and UNIX quirks
   2
   3 The @interruptable functions retry when system calls are interrupted,
   4 e.g. when python raises an IOError or OSError with errno == EINTR.
   5
   6 """
   7 from __future__ import division, absolute_import, unicode_literals
   8
   9 import os
  10 import sys
  11 import itertools
  12 import platform
  13 import subprocess
  14
  15 from cola.decorators import interruptable
  16 from cola.compat import ustr
  17 from cola.compat import PY2
  18 from cola.compat import PY3
  19 from cola.compat import WIN32
  20
  21 # Some files are not in UTF-8; some other aren't in any codification.
  22 # Remember that GIT doesn't care about encodings (saves binary data)
  23 _encoding_tests = [
  24     'utf-8',
  25     'iso-8859-15',
  26     'windows1252',
  27     'ascii',
  28     # <-- add encodings here
  29 ]
  30
  31 def decode(enc, encoding=None, errors='strict'):
  32     """decode(encoded_string) returns an unencoded unicode string
  33     """
  34     if enc is None or type(enc) is ustr:
  35         return enc
  36
  37     if encoding is None:
  38         encoding_tests = _encoding_tests
  39     else:
  40         encoding_tests = itertools.chain([encoding], _encoding_tests)
  41
  42     for encoding in encoding_tests:
  43         try:
  44             return enc.decode(encoding, errors)
  45         except:
  46             pass
  47     # this shouldn't ever happen... FIXME
  48     return ustr(enc)
  49
  50
  51 def encode(string, encoding=None):
  52     """encode(unencoded_string) returns a string encoded in utf-8
  53     """
  54     if type(string) is not ustr:
  55         return string
  56     return string.encode(encoding or 'utf-8', 'replace')
  57
  58
  59 def mkpath(path, encoding=None):
  60     # The Windows API requires unicode strings regardless of python version
  61     if WIN32:
  62         return decode(path, encoding=encoding)
  63     # UNIX prefers bytes
  64     return encode(path, encoding=encoding)
  65
  66
  67 def read(filename, size=-1, encoding=None, errors='strict'):
  68     """Read filename and return contents"""
  69     with xopen(filename, 'rb') as fh:
  70         return fread(fh, size=size, encoding=encoding, errors=errors)
  71
  72
  73 def write(path, contents, encoding=None):
  74     """Writes a unicode string to a file"""
  75     with xopen(path, 'wb') as fh:
  76         return fwrite(fh, contents, encoding=encoding)
  77
  78
  79 @interruptable
  80 def fread(fh, size=-1, encoding=None, errors='strict'):
  81     """Read from a filehandle and retry when interrupted"""
  82     return decode(fh.read(size), encoding=encoding, errors=errors)
  83
  84
  85 @interruptable
  86 def fwrite(fh, content, encoding=None):
  87     """Write to a filehandle and retry when interrupted"""
  88     return fh.write(encode(content, encoding=encoding))
  89
  90
  91 @interruptable
  92 def wait(proc):
  93     """Wait on a subprocess and retry when interrupted"""
  94     return proc.wait()
  95
  96
  97 @interruptable
  98 def readline(fh, encoding=None):
  99     return decode(fh.readline(), encoding=encoding)
 100
 101
 102 @interruptable
 103 def start_command(cmd, cwd=None, add_env=None,
 104                   universal_newlines=False,
 105                   stdin=subprocess.PIPE,
 106                   stdout=subprocess.PIPE,
 107                   stderr=subprocess.PIPE,
 108                   **extra):
 109     """Start the given command, and return a subprocess object.
 110
 111     This provides a simpler interface to the subprocess module.
 112
 113     """
 114     env = None
 115     if add_env is not None:
 116         env = os.environ.copy()
 117         env.update(add_env)
 118
 119     # Python3 on windows always goes through list2cmdline() internally inside
 120     # of subprocess.py so we must provide unicode strings here otherwise
 121     # Python3 breaks when bytes are provided.
 122     #
 123     # Additionally, the preferred usage on Python3 is to pass unicode
 124     # strings to subprocess.  Python will automatically encode into the
 125     # default encoding (utf-8) when it gets unicode strings.
 126     cmd = prep_for_subprocess(cmd)
 127
 128     if WIN32 and cwd == getcwd():
 129         # Windows cannot deal with passing a cwd that contains unicode
 130         # but we luckily can pass None when the supplied cwd is the same
 131         # as our current directory and get the same effect.
 132         # Not doing this causes unicode encoding errors when launching
 133         # the subprocess.
 134         cwd = None
 135
 136     if WIN32:
 137         CREATE_NO_WINDOW = 0x08000000
 138         extra['creationflags'] = CREATE_NO_WINDOW
 139
 140     return subprocess.Popen(cmd, bufsize=1, stdin=stdin, stdout=stdout,
 141                             stderr=stderr, cwd=cwd, env=env,
 142                             universal_newlines=universal_newlines, **extra)
 143
 144
 145 def prep_for_subprocess(cmd):
 146     """Decode on Python3, encode on Python2"""
 147     # See the comment in start_command()
 148     if PY3:
 149         cmd = [decode(c) for c in cmd]
 150     else:
 151         cmd = [encode(c) for c in cmd]
 152     return cmd
 153
 154
 155 @interruptable
 156 def communicate(proc):
 157     return proc.communicate()
 158
 159
 160 def run_command(cmd, encoding=None, *args, **kwargs):
 161     """Run the given command to completion, and return its results.
 162
 163     This provides a simpler interface to the subprocess module.
 164     The results are formatted as a 3-tuple: (exit_code, output, errors)
 165     The other arguments are passed on to start_command().
 166
 167     """
 168     process = start_command(cmd, *args, **kwargs)
 169     (output, errors) = communicate(process)
 170     output = decode(output, encoding=encoding)
 171     errors = decode(errors, encoding=encoding)
 172     exit_code = process.returncode
 173     return (exit_code, output or '', errors or '')
 174
 175
 176 @interruptable
 177 def _fork_posix(args, cwd=None):
 178     """Launch a process in the background."""
 179     encoded_args = [encode(arg) for arg in args]
 180     return subprocess.Popen(encoded_args, cwd=cwd).pid
 181
 182
 183 def _fork_win32(args, cwd=None):
 184     """Launch a background process using crazy win32 voodoo."""
 185     # This is probably wrong, but it works.  Windows.. wow.
 186     if args[0] == 'git-dag':
 187         # win32 can't exec python scripts
 188         args = [sys.executable] + args
 189     args[0] = _win32_find_exe(args[0])
 190
 191     if PY3:
 192         # see comment in start_command()
 193         argv = [decode(arg) for arg in args]
 194     else:
 195         argv = [encode(arg) for arg in args]
 196     DETACHED_PROCESS = 0x00000008 # Amazing!
 197     return subprocess.Popen(argv, cwd=cwd, creationflags=DETACHED_PROCESS).pid
 198
 199
 200 def _win32_find_exe(exe):
 201     """Find the actual file for a Windows executable.
 202
 203     This function goes through the same process that the Windows shell uses to
 204     locate an executable, taking into account the PATH and PATHEXT environment
 205     variables.  This allows us to avoid passing shell=True to subprocess.Popen.
 206
 207     For reference, see:
 208     http://technet.microsoft.com/en-us/library/cc723564.aspx#XSLTsection127121120120
 209
 210     """
 211     # try the argument itself
 212     candidates = [exe]
 213     # if argument does not have an extension, also try it with each of the
 214     # extensions specified in PATHEXT
 215     if '.' not in exe:
 216         extensions = getenv('PATHEXT', '').split(os.pathsep)
 217         candidates.extend([exe+ext for ext in extensions
 218                             if ext.startswith('.')])
 219     # search the current directory first
 220     for candidate in candidates:
 221         if exists(candidate):
 222             return candidate
 223     # if the argument does not include a path separator, search each of the
 224     # directories on the PATH
 225     if not os.path.dirname(exe):
 226         for path in getenv('PATH').split(os.pathsep):
 227             if path:
 228                 for candidate in candidates:
 229                     full_path = os.path.join(path, candidate)
 230                     if exists(full_path):
 231                         return full_path
 232     # not found, punt and return the argument unchanged
 233     return exe
 234
 235
 236 # Portability wrappers
 237 if sys.platform == 'win32' or sys.platform == 'cygwin':
 238     fork = _fork_win32
 239 else:
 240     fork = _fork_posix
 241
 242
 243 def wrap(action, fn, decorator=None):
 244     """Wrap arguments with `action`, optionally decorate the result"""
 245     if decorator is None:
 246         decorator = lambda x: x
 247     def wrapped(*args, **kwargs):
 248         return decorator(fn(action(*args, **kwargs)))
 249     return wrapped
 250
 251
 252 def decorate(decorator, fn):
 253     """Decorate the result of `fn` with `action`"""
 254     def decorated(*args, **kwargs):
 255         return decorator(fn(*args, **kwargs))
 256     return decorated
 257
 258
 259 def getenv(name, default=None):
 260     return decode(os.getenv(name, default))
 261
 262
 263 def xopen(path, mode='r', encoding=None):
 264     return open(mkpath(path, encoding=encoding), mode)
 265
 266
 267 def stdout(msg):
 268     msg = msg + '\n'
 269     if PY2:
 270         msg = encode(msg, encoding='utf-8')
 271     sys.stdout.write(msg)
 272
 273
 274 def stderr(msg):
 275     msg = msg + '\n'
 276     if PY2:
 277         msg = encode(msg, encoding='utf-8')
 278     sys.stderr.write(msg)
 279
 280
 281 @interruptable
 282 def node():
 283     return platform.node()
 284
 285
 286 abspath = wrap(mkpath, os.path.abspath, decorator=decode)
 287 chdir = wrap(mkpath, os.chdir)
 288 exists = wrap(mkpath, os.path.exists)
 289 expanduser = wrap(encode, os.path.expanduser, decorator=decode)
 290 try:  # Python 2
 291     getcwd = os.getcwdu
 292 except AttributeError:
 293     getcwd = os.getcwd
 294 isdir = wrap(mkpath, os.path.isdir)
 295 isfile = wrap(mkpath, os.path.isfile)
 296 islink = wrap(mkpath, os.path.islink)
 297 makedirs = wrap(mkpath, os.makedirs)
 298 try:
 299     readlink = wrap(mkpath, os.readlink, decorator=decode)
 300 except AttributeError:
 301     readlink = lambda p: p
 302 realpath = wrap(mkpath, os.path.realpath, decorator=decode)
 303 relpath = wrap(mkpath, os.path.relpath, decorator=decode)
 304 stat = wrap(mkpath, os.stat)
 305 unlink = wrap(mkpath, os.unlink)
 306 walk = wrap(mkpath, os.walk)