zeroinstall/zerostore/unpack.py

   1 """Unpacking archives of various formats."""
   2
   3 # Copyright (C) 2009, Thomas Leonard
   4 # See the README file for details, or visit http://0install.net.
   5
   6 from zeroinstall import _
   7 import os, subprocess
   8 import shutil
   9 import glob
  10 import traceback
  11 from tempfile import mkdtemp, mkstemp
  12 import re
  13 from logging import debug, warn
  14 import errno
  15 from zeroinstall import SafeException
  16 from zeroinstall.support import find_in_path, ro_rmtree
  17
  18 _cpio_version = None
  19 def _get_cpio_version():
  20         global _cpio_version
  21         if _cpio_version is None:
  22                 _cpio_version = os.popen('cpio --version 2>&1').next()
  23                 debug(_("cpio version = %s"), _cpio_version)
  24         return _cpio_version
  25
  26 def _gnu_cpio():
  27         gnu_cpio = '(GNU cpio)' in _get_cpio_version()
  28         debug(_("Is GNU cpio = %s"), gnu_cpio)
  29         return gnu_cpio
  30
  31 _tar_version = None
  32 def _get_tar_version():
  33         global _tar_version
  34         if _tar_version is None:
  35                 _tar_version = os.popen('tar --version 2>&1').next().strip()
  36                 debug(_("tar version = %s"), _tar_version)
  37         return _tar_version
  38
  39 def _gnu_tar():
  40         gnu_tar = '(GNU tar)' in _get_tar_version()
  41         debug(_("Is GNU tar = %s"), gnu_tar)
  42         return gnu_tar
  43
  44 def recent_gnu_tar():
  45         """@deprecated: should be private"""
  46         recent_gnu_tar = False
  47         if _gnu_tar():
  48                 version = re.search(r'\)\s*(\d+(\.\d+)*)', _get_tar_version())
  49                 if version:
  50                         version = map(int, version.group(1).split('.'))
  51                         recent_gnu_tar = version > [1, 13, 92]
  52                 else:
  53                         warn(_("Failed to extract GNU tar version number"))
  54         debug(_("Recent GNU tar = %s"), recent_gnu_tar)
  55         return recent_gnu_tar
  56
  57 # Disabled, as Plash does not currently support fchmod(2).
  58 _pola_run = None
  59 #_pola_run = find_in_path('pola-run')
  60 #if _pola_run:
  61 #       info('Found pola-run: %s', _pola_run)
  62 #else:
  63 #       info('pola-run not found; archive extraction will not be sandboxed')
  64
  65 def type_from_url(url):
  66         """Guess the MIME type for this resource based on its URL. Returns None if we don't know what it is."""
  67         url = url.lower()
  68         if url.endswith('.rpm'): return 'application/x-rpm'
  69         if url.endswith('.deb'): return 'application/x-deb'
  70         if url.endswith('.tar.bz2'): return 'application/x-bzip-compressed-tar'
  71         if url.endswith('.tar.gz'): return 'application/x-compressed-tar'
  72         if url.endswith('.tar.lzma'): return 'application/x-lzma-compressed-tar'
  73         if url.endswith('.tar.xz'): return 'application/x-xz-compressed-tar'
  74         if url.endswith('.tbz'): return 'application/x-bzip-compressed-tar'
  75         if url.endswith('.tgz'): return 'application/x-compressed-tar'
  76         if url.endswith('.tlz'): return 'application/x-lzma-compressed-tar'
  77         if url.endswith('.txz'): return 'application/x-xz-compressed-tar'
  78         if url.endswith('.tar'): return 'application/x-tar'
  79         if url.endswith('.zip'): return 'application/zip'
  80         if url.endswith('.cab'): return 'application/vnd.ms-cab-compressed'
  81         if url.endswith('.dmg'): return 'application/x-apple-diskimage'
  82         if url.endswith('.gem'): return 'application/x-ruby-gem'
  83         return None
  84
  85 def check_type_ok(mime_type):
  86         """Check we have the needed software to extract from an archive of the given type.
  87         @raise SafeException: if the needed software is not available"""
  88         assert mime_type
  89         if mime_type == 'application/x-rpm':
  90                 if not find_in_path('rpm2cpio'):
  91                         raise SafeException(_("This package looks like an RPM, but you don't have the rpm2cpio command "
  92                                         "I need to extract it. Install the 'rpm' package first (this works even if "
  93                                         "you're on a non-RPM-based distribution such as Debian)."))
  94         elif mime_type == 'application/x-deb':
  95                 if not find_in_path('ar'):
  96                         raise SafeException(_("This package looks like a Debian package, but you don't have the 'ar' command "
  97                                         "I need to extract it. Install the package containing it (sometimes called 'binutils') "
  98                                         "first. This works even if you're on a non-Debian-based distribution such as Red Hat)."))
  99         elif mime_type == 'application/x-bzip-compressed-tar':
 100                 pass    # We'll fall back to Python's built-in tar.bz2 support
 101         elif mime_type == 'application/zip':
 102                 if not find_in_path('unzip'):
 103                         raise SafeException(_("This package looks like a zip-compressed archive, but you don't have the 'unzip' command "
 104                                         "I need to extract it. Install the package containing it first."))
 105         elif mime_type == 'application/vnd.ms-cab-compressed':
 106                 if not find_in_path('cabextract'):
 107                         raise SafeException(_("This package looks like a Microsoft Cabinet archive, but you don't have the 'cabextract' command "
 108                                         "I need to extract it. Install the package containing it first."))
 109         elif mime_type == 'application/x-apple-diskimage':
 110                 if not find_in_path('hdiutil'):
 111                         raise SafeException(_("This package looks like a Apple Disk Image, but you don't have the 'hdiutil' command "
 112                                         "I need to extract it."))
 113         elif mime_type == 'application/x-lzma-compressed-tar':
 114                 pass    # We can get it through Zero Install
 115         elif mime_type == 'application/x-xz-compressed-tar':
 116                 if not find_in_path('unxz'):
 117                         raise SafeException(_("This package looks like a xz-compressed package, but you don't have the 'unxz' command "
 118                                         "I need to extract it. Install the package containing it (it's probably called 'xz-utils') "
 119                                         "first."))
 120         elif mime_type in ('application/x-compressed-tar', 'application/x-tar', 'application/x-ruby-gem'):
 121                 pass
 122         else:
 123                 from zeroinstall import version
 124                 raise SafeException(_("Unsupported archive type '%(type)s' (for injector version %(version)s)") % {'type': mime_type, 'version': version})
 125
 126 def _exec_maybe_sandboxed(writable, prog, *args):
 127         """execlp prog, with (only) the 'writable' directory writable if sandboxing is available.
 128         If no sandbox is available, run without a sandbox."""
 129         prog_path = find_in_path(prog)
 130         if not prog_path: raise Exception(_("'%s' not found in $PATH") % prog)
 131         if _pola_run is None:
 132                 os.execlp(prog_path, prog_path, *args)
 133         # We have pola-shell :-)
 134         pola_args = ['--prog', prog_path, '-f', '/']
 135         for a in args:
 136                 pola_args += ['-a', a]
 137         if writable:
 138                 pola_args += ['-fw', writable]
 139         os.execl(_pola_run, _pola_run, *pola_args)
 140
 141 def unpack_archive_over(url, data, destdir, extract = None, type = None, start_offset = 0):
 142         """Like unpack_archive, except that we unpack to a temporary directory first and
 143         then move things over, checking that we're not following symlinks at each stage.
 144         Use this when you want to unpack an unarchive into a directory which already has
 145         stuff in it.
 146         @note: Since 0.49, the leading "extract" component is removed (unlike unpack_archive).
 147         @since: 0.28"""
 148         import stat
 149         tmpdir = mkdtemp(dir = destdir)
 150         assert extract is None or os.sep not in extract, extract
 151         try:
 152                 mtimes = []
 153
 154                 unpack_archive(url, data, tmpdir, extract, type, start_offset)
 155
 156                 if extract is None:
 157                         srcdir = tmpdir
 158                 else:
 159                         srcdir = os.path.join(tmpdir, extract)
 160                         assert not os.path.islink(srcdir)
 161
 162                 stem_len = len(srcdir)
 163                 for root, dirs, files in os.walk(srcdir):
 164                         relative_root = root[stem_len + 1:] or '.'
 165                         target_root = os.path.join(destdir, relative_root)
 166                         try:
 167                                 info = os.lstat(target_root)
 168                         except OSError as ex:
 169                                 if ex.errno != errno.ENOENT:
 170                                         raise   # Some odd error.
 171                                 # Doesn't exist. OK.
 172                                 os.mkdir(target_root)
 173                         else:
 174                                 if stat.S_ISLNK(info.st_mode):
 175                                         raise SafeException(_('Attempt to unpack dir over symlink "%s"!') % relative_root)
 176                                 elif not stat.S_ISDIR(info.st_mode):
 177                                         raise SafeException(_('Attempt to unpack dir over non-directory "%s"!') % relative_root)
 178                         mtimes.append((relative_root, os.lstat(os.path.join(srcdir, root)).st_mtime))
 179
 180                         for s in dirs:  # Symlinks are counted as directories
 181                                 src = os.path.join(srcdir, relative_root, s)
 182                                 if os.path.islink(src):
 183                                         files.append(s)
 184
 185                         for f in files:
 186                                 src = os.path.join(srcdir, relative_root, f)
 187                                 dest = os.path.join(destdir, relative_root, f)
 188                                 if os.path.islink(dest):
 189                                         raise SafeException(_('Attempt to unpack file over symlink "%s"!') %
 190                                                         os.path.join(relative_root, f))
 191                                 os.rename(src, dest)
 192
 193                 for path, mtime in mtimes[1:]:
 194                         os.utime(os.path.join(destdir, path), (mtime, mtime))
 195         finally:
 196                 ro_rmtree(tmpdir)
 197
 198 def unpack_archive(url, data, destdir, extract = None, type = None, start_offset = 0):
 199         """Unpack stream 'data' into directory 'destdir'. If extract is given, extract just
 200         that sub-directory from the archive (i.e. destdir/extract will exist afterwards).
 201         Works out the format from the name."""
 202         if type is None: type = type_from_url(url)
 203         if type is None: raise SafeException(_("Unknown extension (and no MIME type given) in '%s'") % url)
 204         if type == 'application/x-bzip-compressed-tar':
 205                 extract_tar(data, destdir, extract, 'bzip2', start_offset)
 206         elif type == 'application/x-deb':
 207                 extract_deb(data, destdir, extract, start_offset)
 208         elif type == 'application/x-rpm':
 209                 extract_rpm(data, destdir, extract, start_offset)
 210         elif type == 'application/zip':
 211                 extract_zip(data, destdir, extract, start_offset)
 212         elif type == 'application/x-tar':
 213                 extract_tar(data, destdir, extract, None, start_offset)
 214         elif type == 'application/x-lzma-compressed-tar':
 215                 extract_tar(data, destdir, extract, 'lzma', start_offset)
 216         elif type == 'application/x-xz-compressed-tar':
 217                 extract_tar(data, destdir, extract, 'xz', start_offset)
 218         elif type == 'application/x-compressed-tar':
 219                 extract_tar(data, destdir, extract, 'gzip', start_offset)
 220         elif type == 'application/vnd.ms-cab-compressed':
 221                 extract_cab(data, destdir, extract, start_offset)
 222         elif type == 'application/x-apple-diskimage':
 223                 extract_dmg(data, destdir, extract, start_offset)
 224         elif type == 'application/x-ruby-gem':
 225                 extract_gem(data, destdir, extract, start_offset)
 226         else:
 227                 raise SafeException(_('Unknown MIME type "%(type)s" for "%(url)s"') % {'type': type, 'url': url})
 228
 229 def extract_deb(stream, destdir, extract = None, start_offset = 0):
 230         if extract:
 231                 raise SafeException(_('Sorry, but the "extract" attribute is not yet supported for Debs'))
 232
 233         stream.seek(start_offset)
 234         # ar can't read from stdin, so make a copy...
 235         deb_copy_name = os.path.join(destdir, 'archive.deb')
 236         deb_copy = open(deb_copy_name, 'w')
 237         shutil.copyfileobj(stream, deb_copy)
 238         deb_copy.close()
 239
 240         data_tar = None
 241         p = subprocess.Popen(('ar', 't', 'archive.deb'), stdout=subprocess.PIPE, cwd=destdir, universal_newlines=True)
 242         o = p.communicate()[0]
 243         for line in o.split('\n'):
 244                 if line == 'data.tar':
 245                         data_compression = None
 246                 elif line == 'data.tar.gz':
 247                         data_compression = 'gzip'
 248                 elif line == 'data.tar.bz2':
 249                         data_compression = 'bzip2'
 250                 elif line == 'data.tar.lzma':
 251                         data_compression = 'lzma'
 252                 else:
 253                         continue
 254                 data_tar = line
 255                 break
 256         else:
 257                 raise SafeException(_("File is not a Debian package."))
 258
 259         _extract(stream, destdir, ('ar', 'x', 'archive.deb', data_tar))
 260         os.unlink(deb_copy_name)
 261         data_name = os.path.join(destdir, data_tar)
 262         data_stream = open(data_name)
 263         os.unlink(data_name)
 264         extract_tar(data_stream, destdir, None, data_compression)
 265
 266 def extract_rpm(stream, destdir, extract = None, start_offset = 0):
 267         if extract:
 268                 raise SafeException(_('Sorry, but the "extract" attribute is not yet supported for RPMs'))
 269         fd, cpiopath = mkstemp('-rpm-tmp')
 270         try:
 271                 child = os.fork()
 272                 if child == 0:
 273                         try:
 274                                 try:
 275                                         os.dup2(stream.fileno(), 0)
 276                                         os.lseek(0, start_offset, 0)
 277                                         os.dup2(fd, 1)
 278                                         _exec_maybe_sandboxed(None, 'rpm2cpio', '-')
 279                                 except:
 280                                         traceback.print_exc()
 281                         finally:
 282                                 os._exit(1)
 283                 id, status = os.waitpid(child, 0)
 284                 assert id == child
 285                 if status != 0:
 286                         raise SafeException(_("rpm2cpio failed; can't unpack RPM archive; exit code %d") % status)
 287                 os.close(fd)
 288                 fd = None
 289
 290                 args = ['cpio', '-mid']
 291                 if _gnu_cpio():
 292                         args.append('--quiet')
 293
 294                 _extract(open(cpiopath), destdir, args)
 295                 # Set the mtime of every directory under 'tmp' to 0, since cpio doesn't
 296                 # preserve directory mtimes.
 297                 for root, dirs, files in os.walk(destdir):
 298                         os.utime(root, (0, 0))
 299         finally:
 300                 if fd is not None:
 301                         os.close(fd)
 302                 os.unlink(cpiopath)
 303
 304 def extract_gem(stream, destdir, extract = None, start_offset = 0):
 305         "@since: 0.53"
 306         stream.seek(start_offset)
 307         payload = 'data.tar.gz'
 308         payload_stream = None
 309         tmpdir = mkdtemp(dir = destdir)
 310         try:
 311                 extract_tar(stream, destdir=tmpdir, extract=payload, decompress=None)
 312                 payload_stream = open(os.path.join(tmpdir, payload))
 313                 extract_tar(payload_stream, destdir=destdir, extract=extract, decompress='gzip')
 314         finally:
 315                 if payload_stream:
 316                         payload_stream.close()
 317                 ro_rmtree(tmpdir)
 318
 319 def extract_cab(stream, destdir, extract, start_offset = 0):
 320         "@since: 0.24"
 321         if extract:
 322                 raise SafeException(_('Sorry, but the "extract" attribute is not yet supported for Cabinet files'))
 323
 324         stream.seek(start_offset)
 325         # cabextract can't read from stdin, so make a copy...
 326         cab_copy_name = os.path.join(destdir, 'archive.cab')
 327         cab_copy = open(cab_copy_name, 'w')
 328         shutil.copyfileobj(stream, cab_copy)
 329         cab_copy.close()
 330
 331         _extract(stream, destdir, ['cabextract', '-s', '-q', 'archive.cab'])
 332         os.unlink(cab_copy_name)
 333
 334 def extract_dmg(stream, destdir, extract, start_offset = 0):
 335         "@since: 0.46"
 336         if extract:
 337                 raise SafeException(_('Sorry, but the "extract" attribute is not yet supported for DMGs'))
 338
 339         stream.seek(start_offset)
 340         # hdiutil can't read from stdin, so make a copy...
 341         dmg_copy_name = os.path.join(destdir, 'archive.dmg')
 342         dmg_copy = open(dmg_copy_name, 'w')
 343         shutil.copyfileobj(stream, dmg_copy)
 344         dmg_copy.close()
 345
 346         mountpoint = mkdtemp(prefix='archive')
 347         subprocess.check_call(["hdiutil", "attach", "-quiet", "-mountpoint", mountpoint, "-nobrowse", dmg_copy_name])
 348         subprocess.check_call(["cp", "-pR"] + glob.glob("%s/*" % mountpoint) + [destdir])
 349         subprocess.check_call(["hdiutil", "detach", "-quiet", mountpoint])
 350         os.rmdir(mountpoint)
 351         os.unlink(dmg_copy_name)
 352
 353 def extract_zip(stream, destdir, extract, start_offset = 0):
 354         if extract:
 355                 # Limit the characters we accept, to avoid sending dodgy
 356                 # strings to zip
 357                 if not re.match('^[a-zA-Z0-9][- _a-zA-Z0-9.]*$', extract):
 358                         raise SafeException(_('Illegal character in extract attribute'))
 359
 360         stream.seek(start_offset)
 361         # unzip can't read from stdin, so make a copy...
 362         zip_copy_name = os.path.join(destdir, 'archive.zip')
 363         zip_copy = open(zip_copy_name, 'w')
 364         shutil.copyfileobj(stream, zip_copy)
 365         zip_copy.close()
 366
 367         args = ['unzip', '-q', '-o', 'archive.zip']
 368
 369         if extract:
 370                 args.append(extract + '/*')
 371
 372         _extract(stream, destdir, args)
 373         os.unlink(zip_copy_name)
 374
 375 def extract_tar(stream, destdir, extract, decompress, start_offset = 0):
 376         if extract:
 377                 # Limit the characters we accept, to avoid sending dodgy
 378                 # strings to tar
 379                 if not re.match('^[a-zA-Z0-9][- _a-zA-Z0-9.]*$', extract):
 380                         raise SafeException(_('Illegal character in extract attribute'))
 381
 382         assert decompress in [None, 'bzip2', 'gzip', 'lzma', 'xz']
 383
 384         if _gnu_tar():
 385                 ext_cmd = ['tar']
 386                 if decompress:
 387                         if decompress == 'bzip2':
 388                                 ext_cmd.append('--bzip2')
 389                         elif decompress == 'gzip':
 390                                 ext_cmd.append('-z')
 391                         elif decompress == 'lzma':
 392                                 unlzma = find_in_path('unlzma')
 393                                 if not unlzma:
 394                                         unlzma = os.path.abspath(os.path.join(os.path.dirname(__file__), '_unlzma'))
 395                                 ext_cmd.append('--use-compress-program=' + unlzma)
 396                         elif decompress == 'xz':
 397                                 unxz = find_in_path('unxz')
 398                                 if not unxz:
 399                                         unxz = os.path.abspath(os.path.join(os.path.dirname(__file__), '_unxz'))
 400                                 ext_cmd.append('--use-compress-program=' + unxz)
 401
 402                 if recent_gnu_tar():
 403                         ext_cmd.extend(('-x', '--no-same-owner', '--no-same-permissions'))
 404                 else:
 405                         ext_cmd.extend(('xf', '-'))
 406
 407                 if extract:
 408                         ext_cmd.append(extract)
 409
 410                 _extract(stream, destdir, ext_cmd, start_offset)
 411         else:
 412                 import tempfile
 413
 414                 # Since we don't have GNU tar, use python's tarfile module. This will probably
 415                 # be a lot slower and we do not support lzma and xz; however, it is portable.
 416                 # (lzma and xz are handled by first uncompressing stream to a temporary file.
 417                 # this is simple to do, but less efficient than piping through the program)
 418                 if decompress is None:
 419                         rmode = 'r|'
 420                 elif decompress == 'bzip2':
 421                         rmode = 'r|bz2'
 422                 elif decompress == 'gzip':
 423                         rmode = 'r|gz'
 424                 elif decompress == 'lzma':
 425                         unlzma = find_in_path('unlzma')
 426                         if not unlzma:
 427                                 unlzma = os.path.abspath(os.path.join(os.path.dirname(__file__), '_unlzma'))
 428                         temp = tempfile.NamedTemporaryFile(suffix='.tar')
 429                         subprocess.check_call((unlzma), stdin=stream, stdout=temp)
 430                         rmode = 'r|'
 431                         stream = temp
 432                 elif decompress == 'xz':
 433                         unxz = find_in_path('unxz')
 434                         if not unxz:
 435                                 unxz = os.path.abspath(os.path.join(os.path.dirname(__file__), '_unxz'))
 436                         temp = tempfile.NamedTemporaryFile(suffix='.tar')
 437                         subprocess.check_call((unxz), stdin=stream, stdout=temp)
 438                         rmode = 'r|'
 439                         stream = temp
 440                 else:
 441                         raise SafeException(_('GNU tar unavailable; unsupported compression format: %s') % decompress)
 442
 443                 import tarfile
 444
 445                 stream.seek(start_offset)
 446                 # Python 2.5.1 crashes if name is None; see Python bug #1706850
 447                 tar = tarfile.open(name = '', mode = rmode, fileobj = stream)
 448
 449                 current_umask = os.umask(0)
 450                 os.umask(current_umask)
 451
 452                 uid = gid = None
 453                 try:
 454                         uid = os.geteuid()
 455                         gid = os.getegid()
 456                 except:
 457                         debug(_("Can't get uid/gid"))
 458
 459                 def chmod_extract(tarinfo):
 460                         # If any X bit is set, they all must be
 461                         if tarinfo.mode & 0o111:
 462                                 tarinfo.mode |= 0o111
 463
 464                         # Everyone gets read and write (subject to the umask)
 465                         # No special bits are allowed.
 466                         tarinfo.mode = ((tarinfo.mode | 0o666) & ~current_umask) & 0o777
 467
 468                         # Don't change owner, even if run as root
 469                         if uid:
 470                                 tarinfo.uid = uid
 471                         if gid:
 472                                 tarinfo.gid = gid
 473                         tar.extract(tarinfo, destdir)
 474
 475                 extracted_anything = False
 476                 ext_dirs = []
 477
 478                 for tarinfo in tar:
 479                         if extract is None or \
 480                            tarinfo.name.startswith(extract + '/') or \
 481                            tarinfo.name == extract:
 482                                 if tarinfo.isdir():
 483                                         ext_dirs.append(tarinfo)
 484
 485                                 chmod_extract(tarinfo)
 486                                 extracted_anything = True
 487
 488                 # Due to a bug in tarfile (python versions < 2.5), we have to manually
 489                 # set the mtime of each directory that we extract after extracting everything.
 490
 491                 for tarinfo in ext_dirs:
 492                         dirname = os.path.join(destdir, tarinfo.name)
 493                         os.utime(dirname, (tarinfo.mtime, tarinfo.mtime))
 494
 495                 tar.close()
 496
 497                 if extract and not extracted_anything:
 498                         raise SafeException(_('Unable to find specified file = %s in archive') % extract)
 499
 500 def _extract(stream, destdir, command, start_offset = 0):
 501         """Run execvp('command') inside destdir in a child process, with
 502         stream seeked to 'start_offset' as stdin."""
 503
 504         # Some zip archives are missing timezone information; force consistent results
 505         child_env = os.environ.copy()
 506         child_env['TZ'] = 'GMT'
 507
 508         stream.seek(start_offset)
 509
 510         # TODO: use pola-run if available, once it supports fchmod
 511         child = subprocess.Popen(command, cwd = destdir, stdin = stream, stderr = subprocess.PIPE, env = child_env)
 512
 513         unused, cerr = child.communicate()
 514
 515         status = child.wait()
 516         if status != 0:
 517                 raise SafeException(_('Failed to extract archive (using %(command)s); exit code %(status)d:\n%(err)s') % {'command': command, 'status': status, 'err': cerr.strip()})