zeroinstall/zerostore/unpack.py

   1 """Unpacking archives of various formats."""
   2
   3 # Copyright (C) 2009, Thomas Leonard
   4 # See the README file for details, or visit http://0install.net.
   5
   6 from zeroinstall import _
   7 import os, subprocess
   8 import shutil
   9 import glob
  10 import traceback
  11 from tempfile import mkdtemp, mkstemp
  12 import re
  13 from logging import debug, warn
  14 from zeroinstall import SafeException
  15 from zeroinstall.support import find_in_path, ro_rmtree
  16
  17 _cpio_version = None
  18 def _get_cpio_version():
  19         global _cpio_version
  20         if _cpio_version is None:
  21                 _cpio_version = os.popen('cpio --version 2>&1').next()
  22                 debug(_("cpio version = %s"), _cpio_version)
  23         return _cpio_version
  24
  25 def _gnu_cpio():
  26         gnu_cpio = '(GNU cpio)' in _get_cpio_version()
  27         debug(_("Is GNU cpio = %s"), gnu_cpio)
  28         return gnu_cpio
  29
  30 _tar_version = None
  31 def _get_tar_version():
  32         global _tar_version
  33         if _tar_version is None:
  34                 _tar_version = os.popen('tar --version 2>&1').next().strip()
  35                 debug(_("tar version = %s"), _tar_version)
  36         return _tar_version
  37
  38 def _gnu_tar():
  39         gnu_tar = '(GNU tar)' in _get_tar_version()
  40         debug(_("Is GNU tar = %s"), gnu_tar)
  41         return gnu_tar
  42
  43 def recent_gnu_tar():
  44         """@deprecated: should be private"""
  45         recent_gnu_tar = False
  46         if _gnu_tar():
  47                 version = re.search(r'\)\s*(\d+(\.\d+)*)', _get_tar_version())
  48                 if version:
  49                         version = map(int, version.group(1).split('.'))
  50                         recent_gnu_tar = version > [1, 13, 92]
  51                 else:
  52                         warn(_("Failed to extract GNU tar version number"))
  53         debug(_("Recent GNU tar = %s"), recent_gnu_tar)
  54         return recent_gnu_tar
  55
  56 # Disabled, as Plash does not currently support fchmod(2).
  57 _pola_run = None
  58 #_pola_run = find_in_path('pola-run')
  59 #if _pola_run:
  60 #       info('Found pola-run: %s', _pola_run)
  61 #else:
  62 #       info('pola-run not found; archive extraction will not be sandboxed')
  63
  64 def type_from_filename(name):
  65         '''
  66         Guess the MIME type for this resource based on its name.
  67
  68         Returns None if we don't know what it is.
  69
  70         @ivar name string that ends in a filename
  71         '''
  72         name = name.lower()
  73         if name.endswith('.rpm'): return 'application/x-rpm'
  74         if name.endswith('.deb'): return 'application/x-deb'
  75         if name.endswith('.tar.bz2'): return 'application/x-bzip-compressed-tar'
  76         if name.endswith('.tar.gz'): return 'application/x-compressed-tar'
  77         if name.endswith('.tar.lzma'): return 'application/x-lzma-compressed-tar'
  78         if name.endswith('.tar.xz'): return 'application/x-xz-compressed-tar'
  79         if name.endswith('.tbz'): return 'application/x-bzip-compressed-tar'
  80         if name.endswith('.tgz'): return 'application/x-compressed-tar'
  81         if name.endswith('.tlz'): return 'application/x-lzma-compressed-tar'
  82         if name.endswith('.txz'): return 'application/x-xz-compressed-tar'
  83         if name.endswith('.tar'): return 'application/x-tar'
  84         if name.endswith('.zip'): return 'application/zip'
  85         if name.endswith('.cab'): return 'application/vnd.ms-cab-compressed'
  86         if name.endswith('.dmg'): return 'application/x-apple-diskimage'
  87         if name.endswith('.gem'): return 'application/x-ruby-gem'
  88         return None
  89
  90 def check_type_ok(mime_type):
  91         """Check we have the needed software to extract from an archive of the given type.
  92         @raise SafeException: if the needed software is not available"""
  93         assert mime_type
  94         if mime_type == 'application/x-rpm':
  95                 if not find_in_path('rpm2cpio'):
  96                         raise SafeException(_("This package looks like an RPM, but you don't have the rpm2cpio command "
  97                                         "I need to extract it. Install the 'rpm' package first (this works even if "
  98                                         "you're on a non-RPM-based distribution such as Debian)."))
  99         elif mime_type == 'application/x-deb':
 100                 if not find_in_path('ar'):
 101                         raise SafeException(_("This package looks like a Debian package, but you don't have the 'ar' command "
 102                                         "I need to extract it. Install the package containing it (sometimes called 'binutils') "
 103                                         "first. This works even if you're on a non-Debian-based distribution such as Red Hat)."))
 104         elif mime_type == 'application/x-bzip-compressed-tar':
 105                 pass    # We'll fall back to Python's built-in tar.bz2 support
 106         elif mime_type == 'application/zip':
 107                 if not find_in_path('unzip'):
 108                         raise SafeException(_("This package looks like a zip-compressed archive, but you don't have the 'unzip' command "
 109                                         "I need to extract it. Install the package containing it first."))
 110         elif mime_type == 'application/vnd.ms-cab-compressed':
 111                 if not find_in_path('cabextract'):
 112                         raise SafeException(_("This package looks like a Microsoft Cabinet archive, but you don't have the 'cabextract' command "
 113                                         "I need to extract it. Install the package containing it first."))
 114         elif mime_type == 'application/x-apple-diskimage':
 115                 if not find_in_path('hdiutil'):
 116                         raise SafeException(_("This package looks like a Apple Disk Image, but you don't have the 'hdiutil' command "
 117                                         "I need to extract it."))
 118         elif mime_type == 'application/x-lzma-compressed-tar':
 119                 pass    # We can get it through Zero Install
 120         elif mime_type == 'application/x-xz-compressed-tar':
 121                 if not find_in_path('unxz'):
 122                         raise SafeException(_("This package looks like a xz-compressed package, but you don't have the 'unxz' command "
 123                                         "I need to extract it. Install the package containing it (it's probably called 'xz-utils') "
 124                                         "first."))
 125         elif mime_type in ('application/x-compressed-tar', 'application/x-tar', 'application/x-ruby-gem'):
 126                 pass
 127         else:
 128                 from zeroinstall import version
 129                 raise SafeException(_("Unsupported archive type '%(type)s' (for injector version %(version)s)") % {'type': mime_type, 'version': version})
 130
 131 def _exec_maybe_sandboxed(writable, prog, *args):
 132         """execlp prog, with (only) the 'writable' directory writable if sandboxing is available.
 133         If no sandbox is available, run without a sandbox."""
 134         prog_path = find_in_path(prog)
 135         if not prog_path: raise Exception(_("'%s' not found in $PATH") % prog)
 136         if _pola_run is None:
 137                 os.execlp(prog_path, prog_path, *args)
 138         # We have pola-shell :-)
 139         pola_args = ['--prog', prog_path, '-f', '/']
 140         for a in args:
 141                 pola_args += ['-a', a]
 142         if writable:
 143                 pola_args += ['-fw', writable]
 144         os.execl(_pola_run, _pola_run, *pola_args)
 145
 146 def unpack_archive_over(archive, data, destdir, extract = None, type = None, start_offset = 0):
 147         """Like unpack_archive, except that we unpack to a temporary directory first and
 148         then move things over, checking that we're not following symlinks at each stage.
 149         Use this when you want to unpack an unarchive into a directory which already has
 150         stuff in it.
 151         @note: Since 0.49, the leading "extract" component is removed (unlike unpack_archive).
 152         @since: 0.28"""
 153         import stat
 154         tmpdir = mkdtemp(dir = destdir)
 155         assert extract is None or os.sep not in extract, extract
 156         try:
 157                 mtimes = []
 158
 159                 unpack_archive(archive, data, tmpdir, extract, type, start_offset)
 160
 161                 if extract is None:
 162                         srcdir = tmpdir
 163                 else:
 164                         srcdir = os.path.join(tmpdir, extract)
 165                         assert not os.path.islink(srcdir)
 166
 167                 stem_len = len(srcdir)
 168                 for root, dirs, files in os.walk(srcdir):
 169                         relative_root = root[stem_len + 1:] or '.'
 170                         target_root = os.path.join(destdir, relative_root)
 171                         try:
 172                                 info = os.lstat(target_root)
 173                         except OSError, ex:
 174                                 if ex.errno != 2:
 175                                         raise   # Some odd error.
 176                                 # Doesn't exist. OK.
 177                                 os.mkdir(target_root)
 178                         else:
 179                                 if stat.S_ISLNK(info.st_mode):
 180                                         raise SafeException(_('Attempt to unpack dir over symlink "%s"!') % relative_root)
 181                                 elif not stat.S_ISDIR(info.st_mode):
 182                                         raise SafeException(_('Attempt to unpack dir over non-directory "%s"!') % relative_root)
 183                         mtimes.append((relative_root, os.lstat(os.path.join(srcdir, root)).st_mtime))
 184
 185                         for s in dirs:  # Symlinks are counted as directories
 186                                 src = os.path.join(srcdir, relative_root, s)
 187                                 if os.path.islink(src):
 188                                         files.append(s)
 189
 190                         for f in files:
 191                                 src = os.path.join(srcdir, relative_root, f)
 192                                 dest = os.path.join(destdir, relative_root, f)
 193                                 if os.path.islink(dest):
 194                                         raise SafeException(_('Attempt to unpack file over symlink "%s"!') %
 195                                                         os.path.join(relative_root, f))
 196                                 os.rename(src, dest)
 197
 198                 for path, mtime in mtimes[1:]:
 199                         os.utime(os.path.join(destdir, path), (mtime, mtime))
 200         finally:
 201                 ro_rmtree(tmpdir)
 202
 203 def unpack_archive(archive, data, destdir, extract = None, type = None, start_offset = 0):
 204         """Unpack stream 'data' into directory 'destdir'. If extract is given, extract just
 205         that sub-directory from the archive (i.e. destdir/extract will exist afterwards).
 206         Works out the format from the name.
 207
 208         @ivar archive Path or url to the archive
 209         """
 210         if type is None: type = type_from_filename(archive)
 211         if type is None: raise SafeException(_("Unknown extension (and no MIME type given) in '%s'") % archive)
 212         if type == 'application/x-bzip-compressed-tar':
 213                 extract_tar(data, destdir, extract, 'bzip2', start_offset)
 214         elif type == 'application/x-deb':
 215                 extract_deb(data, destdir, extract, start_offset)
 216         elif type == 'application/x-rpm':
 217                 extract_rpm(data, destdir, extract, start_offset)
 218         elif type == 'application/zip':
 219                 extract_zip(data, destdir, extract, start_offset)
 220         elif type == 'application/x-tar':
 221                 extract_tar(data, destdir, extract, None, start_offset)
 222         elif type == 'application/x-lzma-compressed-tar':
 223                 extract_tar(data, destdir, extract, 'lzma', start_offset)
 224         elif type == 'application/x-xz-compressed-tar':
 225                 extract_tar(data, destdir, extract, 'xz', start_offset)
 226         elif type == 'application/x-compressed-tar':
 227                 extract_tar(data, destdir, extract, 'gzip', start_offset)
 228         elif type == 'application/vnd.ms-cab-compressed':
 229                 extract_cab(data, destdir, extract, start_offset)
 230         elif type == 'application/x-apple-diskimage':
 231                 extract_dmg(data, destdir, extract, start_offset)
 232         elif type == 'application/x-ruby-gem':
 233                 extract_gem(data, destdir, extract, start_offset)
 234         else:
 235                 raise SafeException(_('Unknown MIME type "%(type)s" for "%(url)s"') % {'type': type, 'url': archive})
 236
 237 def extract_deb(stream, destdir, extract = None, start_offset = 0):
 238         if extract:
 239                 raise SafeException(_('Sorry, but the "extract" attribute is not yet supported for Debs'))
 240
 241         stream.seek(start_offset)
 242         # ar can't read from stdin, so make a copy...
 243         deb_copy_name = os.path.join(destdir, 'archive.deb')
 244         deb_copy = file(deb_copy_name, 'w')
 245         shutil.copyfileobj(stream, deb_copy)
 246         deb_copy.close()
 247
 248         data_tar = None
 249         p = subprocess.Popen(('ar', 't', 'archive.deb'), stdout=subprocess.PIPE, cwd=destdir, universal_newlines=True)
 250         o = p.communicate()[0]
 251         for line in o.split('\n'):
 252                 if line == 'data.tar':
 253                         data_compression = None
 254                 elif line == 'data.tar.gz':
 255                         data_compression = 'gzip'
 256                 elif line == 'data.tar.bz2':
 257                         data_compression = 'bzip2'
 258                 elif line == 'data.tar.lzma':
 259                         data_compression = 'lzma'
 260                 else:
 261                         continue
 262                 data_tar = line
 263                 break
 264         else:
 265                 raise SafeException(_("File is not a Debian package."))
 266
 267         _extract(stream, destdir, ('ar', 'x', 'archive.deb', data_tar))
 268         os.unlink(deb_copy_name)
 269         data_name = os.path.join(destdir, data_tar)
 270         data_stream = file(data_name)
 271         os.unlink(data_name)
 272         extract_tar(data_stream, destdir, None, data_compression)
 273
 274 def extract_rpm(stream, destdir, extract = None, start_offset = 0):
 275         if extract:
 276                 raise SafeException(_('Sorry, but the "extract" attribute is not yet supported for RPMs'))
 277         fd, cpiopath = mkstemp('-rpm-tmp')
 278         try:
 279                 child = os.fork()
 280                 if child == 0:
 281                         try:
 282                                 try:
 283                                         os.dup2(stream.fileno(), 0)
 284                                         os.lseek(0, start_offset, 0)
 285                                         os.dup2(fd, 1)
 286                                         _exec_maybe_sandboxed(None, 'rpm2cpio', '-')
 287                                 except:
 288                                         traceback.print_exc()
 289                         finally:
 290                                 os._exit(1)
 291                 id, status = os.waitpid(child, 0)
 292                 assert id == child
 293                 if status != 0:
 294                         raise SafeException(_("rpm2cpio failed; can't unpack RPM archive; exit code %d") % status)
 295                 os.close(fd)
 296                 fd = None
 297
 298                 args = ['cpio', '-mid']
 299                 if _gnu_cpio():
 300                         args.append('--quiet')
 301
 302                 _extract(file(cpiopath), destdir, args)
 303                 # Set the mtime of every directory under 'tmp' to 0, since cpio doesn't
 304                 # preserve directory mtimes.
 305                 os.path.walk(destdir, lambda arg, dirname, names: os.utime(dirname, (0, 0)), None)
 306         finally:
 307                 if fd is not None:
 308                         os.close(fd)
 309                 os.unlink(cpiopath)
 310
 311 def extract_gem(stream, destdir, extract = None, start_offset = 0):
 312         "@since: 0.53"
 313         stream.seek(start_offset)
 314         payload = 'data.tar.gz'
 315         payload_stream = None
 316         tmpdir = mkdtemp(dir = destdir)
 317         try:
 318                 extract_tar(stream, destdir=tmpdir, extract=payload, decompress=None)
 319                 payload_stream = file(os.path.join(tmpdir, payload))
 320                 extract_tar(payload_stream, destdir=destdir, extract=extract, decompress='gzip')
 321         finally:
 322                 if payload_stream:
 323                         payload_stream.close()
 324                 ro_rmtree(tmpdir)
 325
 326 def extract_cab(stream, destdir, extract, start_offset = 0):
 327         "@since: 0.24"
 328         if extract:
 329                 raise SafeException(_('Sorry, but the "extract" attribute is not yet supported for Cabinet files'))
 330
 331         stream.seek(start_offset)
 332         # cabextract can't read from stdin, so make a copy...
 333         cab_copy_name = os.path.join(destdir, 'archive.cab')
 334         cab_copy = file(cab_copy_name, 'w')
 335         shutil.copyfileobj(stream, cab_copy)
 336         cab_copy.close()
 337
 338         _extract(stream, destdir, ['cabextract', '-s', '-q', 'archive.cab'])
 339         os.unlink(cab_copy_name)
 340
 341 def extract_dmg(stream, destdir, extract, start_offset = 0):
 342         "@since: 0.46"
 343         if extract:
 344                 raise SafeException(_('Sorry, but the "extract" attribute is not yet supported for DMGs'))
 345
 346         stream.seek(start_offset)
 347         # hdiutil can't read from stdin, so make a copy...
 348         dmg_copy_name = os.path.join(destdir, 'archive.dmg')
 349         dmg_copy = file(dmg_copy_name, 'w')
 350         shutil.copyfileobj(stream, dmg_copy)
 351         dmg_copy.close()
 352
 353         mountpoint = mkdtemp(prefix='archive')
 354         subprocess.check_call(["hdiutil", "attach", "-quiet", "-mountpoint", mountpoint, "-nobrowse", dmg_copy_name])
 355         subprocess.check_call(["cp", "-pR"] + glob.glob("%s/*" % mountpoint) + [destdir])
 356         subprocess.check_call(["hdiutil", "detach", "-quiet", mountpoint])
 357         os.rmdir(mountpoint)
 358         os.unlink(dmg_copy_name)
 359
 360 def extract_zip(stream, destdir, extract, start_offset = 0):
 361         if extract:
 362                 # Limit the characters we accept, to avoid sending dodgy
 363                 # strings to zip
 364                 if not re.match('^[a-zA-Z0-9][- _a-zA-Z0-9.]*$', extract):
 365                         raise SafeException(_('Illegal character in extract attribute'))
 366
 367         stream.seek(start_offset)
 368         # unzip can't read from stdin, so make a copy...
 369         zip_copy_name = os.path.join(destdir, 'archive.zip')
 370         zip_copy = file(zip_copy_name, 'w')
 371         shutil.copyfileobj(stream, zip_copy)
 372         zip_copy.close()
 373
 374         args = ['unzip', '-q', '-o', 'archive.zip']
 375
 376         if extract:
 377                 args.append(extract + '/*')
 378
 379         _extract(stream, destdir, args)
 380         os.unlink(zip_copy_name)
 381
 382 def extract_tar(stream, destdir, extract, decompress, start_offset = 0):
 383         if extract:
 384                 # Limit the characters we accept, to avoid sending dodgy
 385                 # strings to tar
 386                 if not re.match('^[a-zA-Z0-9][- _a-zA-Z0-9.]*$', extract):
 387                         raise SafeException(_('Illegal character in extract attribute'))
 388
 389         assert decompress in [None, 'bzip2', 'gzip', 'lzma', 'xz']
 390
 391         if _gnu_tar():
 392                 ext_cmd = ['tar']
 393                 if decompress:
 394                         if decompress == 'bzip2':
 395                                 ext_cmd.append('--bzip2')
 396                         elif decompress == 'gzip':
 397                                 ext_cmd.append('-z')
 398                         elif decompress == 'lzma':
 399                                 unlzma = find_in_path('unlzma')
 400                                 if not unlzma:
 401                                         unlzma = os.path.abspath(os.path.join(os.path.dirname(__file__), '_unlzma'))
 402                                 ext_cmd.append('--use-compress-program=' + unlzma)
 403                         elif decompress == 'xz':
 404                                 unxz = find_in_path('unxz')
 405                                 if not unxz:
 406                                         unxz = os.path.abspath(os.path.join(os.path.dirname(__file__), '_unxz'))
 407                                 ext_cmd.append('--use-compress-program=' + unxz)
 408
 409                 if recent_gnu_tar():
 410                         ext_cmd.extend(('-x', '--no-same-owner', '--no-same-permissions'))
 411                 else:
 412                         ext_cmd.extend(('xf', '-'))
 413
 414                 if extract:
 415                         ext_cmd.append(extract)
 416
 417                 _extract(stream, destdir, ext_cmd, start_offset)
 418         else:
 419                 import tempfile
 420
 421                 # Since we don't have GNU tar, use python's tarfile module. This will probably
 422                 # be a lot slower and we do not support lzma and xz; however, it is portable.
 423                 # (lzma and xz are handled by first uncompressing stream to a temporary file.
 424                 # this is simple to do, but less efficient than piping through the program)
 425                 if decompress is None:
 426                         rmode = 'r|'
 427                 elif decompress == 'bzip2':
 428                         rmode = 'r|bz2'
 429                 elif decompress == 'gzip':
 430                         rmode = 'r|gz'
 431                 elif decompress == 'lzma':
 432                         unlzma = find_in_path('unlzma')
 433                         if not unlzma:
 434                                 unlzma = os.path.abspath(os.path.join(os.path.dirname(__file__), '_unlzma'))
 435                         temp = tempfile.NamedTemporaryFile(suffix='.tar')
 436                         subprocess.check_call((unlzma), stdin=stream, stdout=temp)
 437                         rmode = 'r|'
 438                         stream = temp
 439                 elif decompress == 'xz':
 440                         unxz = find_in_path('unxz')
 441                         if not unxz:
 442                                 unxz = os.path.abspath(os.path.join(os.path.dirname(__file__), '_unxz'))
 443                         temp = tempfile.NamedTemporaryFile(suffix='.tar')
 444                         subprocess.check_call((unxz), stdin=stream, stdout=temp)
 445                         rmode = 'r|'
 446                         stream = temp
 447                 else:
 448                         raise SafeException(_('GNU tar unavailable; unsupported compression format: %s') % decompress)
 449
 450                 import tarfile
 451
 452                 stream.seek(start_offset)
 453                 # Python 2.5.1 crashes if name is None; see Python bug #1706850
 454                 tar = tarfile.open(name = '', mode = rmode, fileobj = stream)
 455
 456                 current_umask = os.umask(0)
 457                 os.umask(current_umask)
 458
 459                 uid = gid = None
 460                 try:
 461                         uid = os.geteuid()
 462                         gid = os.getegid()
 463                 except:
 464                         debug(_("Can't get uid/gid"))
 465
 466                 def chmod_extract(tarinfo):
 467                         # If any X bit is set, they all must be
 468                         if tarinfo.mode & 0111:
 469                                 tarinfo.mode |= 0111
 470
 471                         # Everyone gets read and write (subject to the umask)
 472                         # No special bits are allowed.
 473                         tarinfo.mode = ((tarinfo.mode | 0666) & ~current_umask) & 0777
 474
 475                         # Don't change owner, even if run as root
 476                         if uid:
 477                                 tarinfo.uid = uid
 478                         if gid:
 479                                 tarinfo.gid = gid
 480                         tar.extract(tarinfo, destdir)
 481
 482                 extracted_anything = False
 483                 ext_dirs = []
 484
 485                 for tarinfo in tar:
 486                         if extract is None or \
 487                            tarinfo.name.startswith(extract + '/') or \
 488                            tarinfo.name == extract:
 489                                 if tarinfo.isdir():
 490                                         ext_dirs.append(tarinfo)
 491
 492                                 chmod_extract(tarinfo)
 493                                 extracted_anything = True
 494
 495                 # Due to a bug in tarfile (python versions < 2.5), we have to manually
 496                 # set the mtime of each directory that we extract after extracting everything.
 497
 498                 for tarinfo in ext_dirs:
 499                         dirname = os.path.join(destdir, tarinfo.name)
 500                         os.utime(dirname, (tarinfo.mtime, tarinfo.mtime))
 501
 502                 tar.close()
 503
 504                 if extract and not extracted_anything:
 505                         raise SafeException(_('Unable to find specified file = %s in archive') % extract)
 506
 507 def _extract(stream, destdir, command, start_offset = 0):
 508         """Run execvp('command') inside destdir in a child process, with
 509         stream seeked to 'start_offset' as stdin."""
 510
 511         # Some zip archives are missing timezone information; force consistent results
 512         child_env = os.environ.copy()
 513         child_env['TZ'] = 'GMT'
 514
 515         stream.seek(start_offset)
 516
 517         # TODO: use pola-run if available, once it supports fchmod
 518         child = subprocess.Popen(command, cwd = destdir, stdin = stream, stderr = subprocess.PIPE, env = child_env)
 519
 520         unused, cerr = child.communicate()
 521
 522         status = child.wait()
 523         if status != 0:
 524                 raise SafeException(_('Failed to extract archive (using %(command)s); exit code %(status)d:\n%(err)s') % {'command': command, 'status': status, 'err': cerr.strip()})