zeroinstall/zerostore/unpack.py

   1 """Unpacking archives of various formats."""
   2
   3 # Copyright (C) 2009, Thomas Leonard
   4 # See the README file for details, or visit http://0install.net.
   5
   6 from zeroinstall import _
   7 import os, subprocess
   8 import shutil
   9 import glob
  10 import traceback
  11 from tempfile import mkdtemp, mkstemp
  12 import re
  13 from logging import debug, warn
  14 from zeroinstall import SafeException
  15 from zeroinstall.support import find_in_path, ro_rmtree
  16
  17 _cpio_version = None
  18 def _get_cpio_version():
  19         global _cpio_version
  20         if _cpio_version is None:
  21                 _cpio_version = os.popen('cpio --version 2>&1').next()
  22                 debug(_("cpio version = %s"), _cpio_version)
  23         return _cpio_version
  24
  25 def _gnu_cpio():
  26         gnu_cpio = '(GNU cpio)' in _get_cpio_version()
  27         debug(_("Is GNU cpio = %s"), gnu_cpio)
  28         return gnu_cpio
  29
  30 _tar_version = None
  31 def _get_tar_version():
  32         global _tar_version
  33         if _tar_version is None:
  34                 _tar_version = os.popen('tar --version 2>&1').next().strip()
  35                 debug(_("tar version = %s"), _tar_version)
  36         return _tar_version
  37
  38 def _gnu_tar():
  39         gnu_tar = '(GNU tar)' in _get_tar_version()
  40         debug(_("Is GNU tar = %s"), gnu_tar)
  41         return gnu_tar
  42
  43 def recent_gnu_tar():
  44         """@deprecated: should be private"""
  45         recent_gnu_tar = False
  46         if _gnu_tar():
  47                 version = re.search(r'\)\s*(\d+(\.\d+)*)', _get_tar_version())
  48                 if version:
  49                         version = map(int, version.group(1).split('.'))
  50                         recent_gnu_tar = version > [1, 13, 92]
  51                 else:
  52                         warn(_("Failed to extract GNU tar version number"))
  53         debug(_("Recent GNU tar = %s"), recent_gnu_tar)
  54         return recent_gnu_tar
  55
  56 # Disabled, as Plash does not currently support fchmod(2).
  57 _pola_run = None
  58 #_pola_run = find_in_path('pola-run')
  59 #if _pola_run:
  60 #       info('Found pola-run: %s', _pola_run)
  61 #else:
  62 #       info('pola-run not found; archive extraction will not be sandboxed')
  63
  64 def type_from_url(url):
  65         """Guess the MIME type for this resource based on its URL. Returns None if we don't know what it is."""
  66         url = url.lower()
  67         if url.endswith('.rpm'): return 'application/x-rpm'
  68         if url.endswith('.deb'): return 'application/x-deb'
  69         if url.endswith('.tar.bz2'): return 'application/x-bzip-compressed-tar'
  70         if url.endswith('.tar.gz'): return 'application/x-compressed-tar'
  71         if url.endswith('.tar.lzma'): return 'application/x-lzma-compressed-tar'
  72         if url.endswith('.tar.xz'): return 'application/x-xz-compressed-tar'
  73         if url.endswith('.tgz'): return 'application/x-compressed-tar'
  74         if url.endswith('.tar'): return 'application/x-tar'
  75         if url.endswith('.zip'): return 'application/zip'
  76         if url.endswith('.cab'): return 'application/vnd.ms-cab-compressed'
  77         if url.endswith('.dmg'): return 'application/x-apple-diskimage'
  78         if url.endswith('.gem'): return 'application/x-ruby-gem'
  79         return None
  80
  81 def check_type_ok(mime_type):
  82         """Check we have the needed software to extract from an archive of the given type.
  83         @raise SafeException: if the needed software is not available"""
  84         assert mime_type
  85         if mime_type == 'application/x-rpm':
  86                 if not find_in_path('rpm2cpio'):
  87                         raise SafeException(_("This package looks like an RPM, but you don't have the rpm2cpio command "
  88                                         "I need to extract it. Install the 'rpm' package first (this works even if "
  89                                         "you're on a non-RPM-based distribution such as Debian)."))
  90         elif mime_type == 'application/x-deb':
  91                 if not find_in_path('ar'):
  92                         raise SafeException(_("This package looks like a Debian package, but you don't have the 'ar' command "
  93                                         "I need to extract it. Install the package containing it (sometimes called 'binutils') "
  94                                         "first. This works even if you're on a non-Debian-based distribution such as Red Hat)."))
  95         elif mime_type == 'application/x-bzip-compressed-tar':
  96                 pass    # We'll fall back to Python's built-in tar.bz2 support
  97         elif mime_type == 'application/zip':
  98                 if not find_in_path('unzip'):
  99                         raise SafeException(_("This package looks like a zip-compressed archive, but you don't have the 'unzip' command "
 100                                         "I need to extract it. Install the package containing it first."))
 101         elif mime_type == 'application/vnd.ms-cab-compressed':
 102                 if not find_in_path('cabextract'):
 103                         raise SafeException(_("This package looks like a Microsoft Cabinet archive, but you don't have the 'cabextract' command "
 104                                         "I need to extract it. Install the package containing it first."))
 105         elif mime_type == 'application/x-apple-diskimage':
 106                 if not find_in_path('hdiutil'):
 107                         raise SafeException(_("This package looks like a Apple Disk Image, but you don't have the 'hdiutil' command "
 108                                         "I need to extract it."))
 109         elif mime_type == 'application/x-lzma-compressed-tar':
 110                 pass    # We can get it through Zero Install
 111         elif mime_type == 'application/x-xz-compressed-tar':
 112                 if not find_in_path('unxz'):
 113                         raise SafeException(_("This package looks like a xz-compressed package, but you don't have the 'unxz' command "
 114                                         "I need to extract it. Install the package containing it (it's probably called 'xz-utils') "
 115                                         "first."))
 116         elif mime_type in ('application/x-compressed-tar', 'application/x-tar'):
 117                 pass
 118         else:
 119                 from zeroinstall import version
 120                 raise SafeException(_("Unsupported archive type '%(type)s' (for injector version %(version)s)") % {'type': mime_type, 'version': version})
 121
 122 def _exec_maybe_sandboxed(writable, prog, *args):
 123         """execlp prog, with (only) the 'writable' directory writable if sandboxing is available.
 124         If no sandbox is available, run without a sandbox."""
 125         prog_path = find_in_path(prog)
 126         if not prog_path: raise Exception(_("'%s' not found in $PATH") % prog)
 127         if _pola_run is None:
 128                 os.execlp(prog_path, prog_path, *args)
 129         # We have pola-shell :-)
 130         pola_args = ['--prog', prog_path, '-f', '/']
 131         for a in args:
 132                 pola_args += ['-a', a]
 133         if writable:
 134                 pola_args += ['-fw', writable]
 135         os.execl(_pola_run, _pola_run, *pola_args)
 136
 137 def unpack_archive_over(url, data, destdir, extract = None, type = None, start_offset = 0):
 138         """Like unpack_archive, except that we unpack to a temporary directory first and
 139         then move things over, checking that we're not following symlinks at each stage.
 140         Use this when you want to unpack an unarchive into a directory which already has
 141         stuff in it.
 142         @note: Since 0.49, the leading "extract" component is removed (unlike unpack_archive).
 143         @since: 0.28"""
 144         import stat
 145         tmpdir = mkdtemp(dir = destdir)
 146         assert extract is None or os.sep not in extract, extract
 147         try:
 148                 mtimes = []
 149
 150                 unpack_archive(url, data, tmpdir, extract, type, start_offset)
 151
 152                 if extract is None:
 153                         srcdir = tmpdir
 154                 else:
 155                         srcdir = os.path.join(tmpdir, extract)
 156                         assert not os.path.islink(srcdir)
 157
 158                 stem_len = len(srcdir)
 159                 for root, dirs, files in os.walk(srcdir):
 160                         relative_root = root[stem_len + 1:] or '.'
 161                         target_root = os.path.join(destdir, relative_root)
 162                         try:
 163                                 info = os.lstat(target_root)
 164                         except OSError, ex:
 165                                 if ex.errno != 2:
 166                                         raise   # Some odd error.
 167                                 # Doesn't exist. OK.
 168                                 os.mkdir(target_root)
 169                         else:
 170                                 if stat.S_ISLNK(info.st_mode):
 171                                         raise SafeException(_('Attempt to unpack dir over symlink "%s"!') % relative_root)
 172                                 elif not stat.S_ISDIR(info.st_mode):
 173                                         raise SafeException(_('Attempt to unpack dir over non-directory "%s"!') % relative_root)
 174                         mtimes.append((relative_root, os.lstat(os.path.join(srcdir, root)).st_mtime))
 175
 176                         for s in dirs:  # Symlinks are counted as directories
 177                                 src = os.path.join(srcdir, relative_root, s)
 178                                 if os.path.islink(src):
 179                                         files.append(s)
 180
 181                         for f in files:
 182                                 src = os.path.join(srcdir, relative_root, f)
 183                                 dest = os.path.join(destdir, relative_root, f)
 184                                 if os.path.islink(dest):
 185                                         raise SafeException(_('Attempt to unpack file over symlink "%s"!') %
 186                                                         os.path.join(relative_root, f))
 187                                 os.rename(src, dest)
 188
 189                 for path, mtime in mtimes[1:]:
 190                         os.utime(os.path.join(destdir, path), (mtime, mtime))
 191         finally:
 192                 ro_rmtree(tmpdir)
 193
 194 def unpack_archive(url, data, destdir, extract = None, type = None, start_offset = 0):
 195         """Unpack stream 'data' into directory 'destdir'. If extract is given, extract just
 196         that sub-directory from the archive (i.e. destdir/extract will exist afterwards).
 197         Works out the format from the name."""
 198         if type is None: type = type_from_url(url)
 199         if type is None: raise SafeException(_("Unknown extension (and no MIME type given) in '%s'") % url)
 200         if type == 'application/x-bzip-compressed-tar':
 201                 extract_tar(data, destdir, extract, 'bzip2', start_offset)
 202         elif type == 'application/x-deb':
 203                 extract_deb(data, destdir, extract, start_offset)
 204         elif type == 'application/x-rpm':
 205                 extract_rpm(data, destdir, extract, start_offset)
 206         elif type == 'application/zip':
 207                 extract_zip(data, destdir, extract, start_offset)
 208         elif type == 'application/x-tar':
 209                 extract_tar(data, destdir, extract, None, start_offset)
 210         elif type == 'application/x-lzma-compressed-tar':
 211                 extract_tar(data, destdir, extract, 'lzma', start_offset)
 212         elif type == 'application/x-xz-compressed-tar':
 213                 extract_tar(data, destdir, extract, 'xz', start_offset)
 214         elif type == 'application/x-compressed-tar':
 215                 extract_tar(data, destdir, extract, 'gzip', start_offset)
 216         elif type == 'application/vnd.ms-cab-compressed':
 217                 extract_cab(data, destdir, extract, start_offset)
 218         elif type == 'application/x-apple-diskimage':
 219                 extract_dmg(data, destdir, extract, start_offset)
 220         elif type == 'application/x-ruby-gem':
 221                 extract_gem(data, destdir, extract, start_offset)
 222         else:
 223                 raise SafeException(_('Unknown MIME type "%(type)s" for "%(url)s"') % {'type': type, 'url': url})
 224
 225 def extract_deb(stream, destdir, extract = None, start_offset = 0):
 226         if extract:
 227                 raise SafeException(_('Sorry, but the "extract" attribute is not yet supported for Debs'))
 228
 229         stream.seek(start_offset)
 230         # ar can't read from stdin, so make a copy...
 231         deb_copy_name = os.path.join(destdir, 'archive.deb')
 232         deb_copy = file(deb_copy_name, 'w')
 233         shutil.copyfileobj(stream, deb_copy)
 234         deb_copy.close()
 235
 236         data_tar = None
 237         p = subprocess.Popen(('ar', 't', 'archive.deb'), stdout=subprocess.PIPE, cwd=destdir, universal_newlines=True)
 238         o = p.communicate()[0]
 239         for line in o.split('\n'):
 240                 if line == 'data.tar':
 241                         data_compression = None
 242                 elif line == 'data.tar.gz':
 243                         data_compression = 'gzip'
 244                 elif line == 'data.tar.bz2':
 245                         data_compression = 'bzip2'
 246                 elif line == 'data.tar.lzma':
 247                         data_compression = 'lzma'
 248                 else:
 249                         continue
 250                 data_tar = line
 251                 break
 252         else:
 253                 raise SafeException(_("File is not a Debian package."))
 254
 255         _extract(stream, destdir, ('ar', 'x', 'archive.deb', data_tar))
 256         os.unlink(deb_copy_name)
 257         data_name = os.path.join(destdir, data_tar)
 258         data_stream = file(data_name)
 259         os.unlink(data_name)
 260         extract_tar(data_stream, destdir, None, data_compression)
 261
 262 def extract_rpm(stream, destdir, extract = None, start_offset = 0):
 263         if extract:
 264                 raise SafeException(_('Sorry, but the "extract" attribute is not yet supported for RPMs'))
 265         fd, cpiopath = mkstemp('-rpm-tmp')
 266         try:
 267                 child = os.fork()
 268                 if child == 0:
 269                         try:
 270                                 try:
 271                                         os.dup2(stream.fileno(), 0)
 272                                         os.lseek(0, start_offset, 0)
 273                                         os.dup2(fd, 1)
 274                                         _exec_maybe_sandboxed(None, 'rpm2cpio', '-')
 275                                 except:
 276                                         traceback.print_exc()
 277                         finally:
 278                                 os._exit(1)
 279                 id, status = os.waitpid(child, 0)
 280                 assert id == child
 281                 if status != 0:
 282                         raise SafeException(_("rpm2cpio failed; can't unpack RPM archive; exit code %d") % status)
 283                 os.close(fd)
 284                 fd = None
 285
 286                 args = ['cpio', '-mid']
 287                 if _gnu_cpio():
 288                         args.append('--quiet')
 289
 290                 _extract(file(cpiopath), destdir, args)
 291                 # Set the mtime of every directory under 'tmp' to 0, since cpio doesn't
 292                 # preserve directory mtimes.
 293                 os.path.walk(destdir, lambda arg, dirname, names: os.utime(dirname, (0, 0)), None)
 294         finally:
 295                 if fd is not None:
 296                         os.close(fd)
 297                 os.unlink(cpiopath)
 298
 299 def extract_gem(stream, destdir, extract = None, start_offset = 0):
 300         "@since 0.53"
 301         stream.seek(start_offset)
 302         payload = 'data.tar.gz'
 303         payload_stream = None
 304         tmpdir = mkdtemp(dir = destdir)
 305         try:
 306                 extract_tar(stream, destdir=tmpdir, extract=payload, decompress=None)
 307                 payload_stream = file(os.path.join(tmpdir, payload))
 308                 extract_tar(payload_stream, destdir=destdir, extract=extract, decompress='gzip')
 309         finally:
 310                 if payload_stream:
 311                         payload_stream.close()
 312                 ro_rmtree(tmpdir)
 313
 314 def extract_cab(stream, destdir, extract, start_offset = 0):
 315         "@since: 0.24"
 316         if extract:
 317                 raise SafeException(_('Sorry, but the "extract" attribute is not yet supported for Cabinet files'))
 318
 319         stream.seek(start_offset)
 320         # cabextract can't read from stdin, so make a copy...
 321         cab_copy_name = os.path.join(destdir, 'archive.cab')
 322         cab_copy = file(cab_copy_name, 'w')
 323         shutil.copyfileobj(stream, cab_copy)
 324         cab_copy.close()
 325
 326         _extract(stream, destdir, ['cabextract', '-s', '-q', 'archive.cab'])
 327         os.unlink(cab_copy_name)
 328
 329 def extract_dmg(stream, destdir, extract, start_offset = 0):
 330         "@since: 0.46"
 331         if extract:
 332                 raise SafeException(_('Sorry, but the "extract" attribute is not yet supported for DMGs'))
 333
 334         stream.seek(start_offset)
 335         # hdiutil can't read from stdin, so make a copy...
 336         dmg_copy_name = os.path.join(destdir, 'archive.dmg')
 337         dmg_copy = file(dmg_copy_name, 'w')
 338         shutil.copyfileobj(stream, dmg_copy)
 339         dmg_copy.close()
 340
 341         mountpoint = mkdtemp(prefix='archive')
 342         subprocess.check_call(["hdiutil", "attach", "-quiet", "-mountpoint", mountpoint, "-nobrowse", dmg_copy_name])
 343         subprocess.check_call(["cp", "-pR"] + glob.glob("%s/*" % mountpoint) + [destdir])
 344         subprocess.check_call(["hdiutil", "detach", "-quiet", mountpoint])
 345         os.rmdir(mountpoint)
 346         os.unlink(dmg_copy_name)
 347
 348 def extract_zip(stream, destdir, extract, start_offset = 0):
 349         if extract:
 350                 # Limit the characters we accept, to avoid sending dodgy
 351                 # strings to zip
 352                 if not re.match('^[a-zA-Z0-9][- _a-zA-Z0-9.]*$', extract):
 353                         raise SafeException(_('Illegal character in extract attribute'))
 354
 355         stream.seek(start_offset)
 356         # unzip can't read from stdin, so make a copy...
 357         zip_copy_name = os.path.join(destdir, 'archive.zip')
 358         zip_copy = file(zip_copy_name, 'w')
 359         shutil.copyfileobj(stream, zip_copy)
 360         zip_copy.close()
 361
 362         args = ['unzip', '-q', '-o', 'archive.zip']
 363
 364         if extract:
 365                 args.append(extract + '/*')
 366
 367         _extract(stream, destdir, args)
 368         os.unlink(zip_copy_name)
 369
 370 def extract_tar(stream, destdir, extract, decompress, start_offset = 0):
 371         if extract:
 372                 # Limit the characters we accept, to avoid sending dodgy
 373                 # strings to tar
 374                 if not re.match('^[a-zA-Z0-9][- _a-zA-Z0-9.]*$', extract):
 375                         raise SafeException(_('Illegal character in extract attribute'))
 376
 377         assert decompress in [None, 'bzip2', 'gzip', 'lzma', 'xz']
 378
 379         if _gnu_tar():
 380                 ext_cmd = ['tar']
 381                 if decompress:
 382                         if decompress == 'bzip2':
 383                                 ext_cmd.append('--bzip2')
 384                         elif decompress == 'gzip':
 385                                 ext_cmd.append('-z')
 386                         elif decompress == 'lzma':
 387                                 unlzma = find_in_path('unlzma')
 388                                 if not unlzma:
 389                                         unlzma = os.path.abspath(os.path.join(os.path.dirname(__file__), '_unlzma'))
 390                                 ext_cmd.append('--use-compress-program=' + unlzma)
 391                         elif decompress == 'xz':
 392                                 ext_cmd.append('--use-compress-program=unxz')
 393
 394                 if recent_gnu_tar():
 395                         ext_cmd.extend(('-x', '--no-same-owner', '--no-same-permissions'))
 396                 else:
 397                         ext_cmd.extend(('xf', '-'))
 398
 399                 if extract:
 400                         ext_cmd.append(extract)
 401
 402                 _extract(stream, destdir, ext_cmd, start_offset)
 403         else:
 404                 # Since we don't have GNU tar, use python's tarfile module. This will probably
 405                 # be a lot slower and we do not support lzma and xz; however, it is portable.
 406                 if decompress is None:
 407                         rmode = 'r|'
 408                 elif decompress == 'bzip2':
 409                         rmode = 'r|bz2'
 410                 elif decompress == 'gzip':
 411                         rmode = 'r|gz'
 412                 else:
 413                         raise SafeException(_('GNU tar unavailable; unsupported compression format: %s') % decompress)
 414
 415                 import tarfile
 416
 417                 stream.seek(start_offset)
 418                 # Python 2.5.1 crashes if name is None; see Python bug #1706850
 419                 tar = tarfile.open(name = '', mode = rmode, fileobj = stream)
 420
 421                 current_umask = os.umask(0)
 422                 os.umask(current_umask)
 423
 424                 uid = gid = None
 425                 try:
 426                         uid = os.geteuid()
 427                         gid = os.getegid()
 428                 except:
 429                         debug(_("Can't get uid/gid"))
 430
 431                 def chmod_extract(tarinfo):
 432                         # If any X bit is set, they all must be
 433                         if tarinfo.mode & 0111:
 434                                 tarinfo.mode |= 0111
 435
 436                         # Everyone gets read and write (subject to the umask)
 437                         # No special bits are allowed.
 438                         tarinfo.mode = ((tarinfo.mode | 0666) & ~current_umask) & 0777
 439
 440                         # Don't change owner, even if run as root
 441                         if uid:
 442                                 tarinfo.uid = uid
 443                         if gid:
 444                                 tarinfo.gid = gid
 445                         tar.extract(tarinfo, destdir)
 446
 447                 extracted_anything = False
 448                 ext_dirs = []
 449
 450                 for tarinfo in tar:
 451                         if extract is None or \
 452                            tarinfo.name.startswith(extract + '/') or \
 453                            tarinfo.name == extract:
 454                                 if tarinfo.isdir():
 455                                         ext_dirs.append(tarinfo)
 456
 457                                 chmod_extract(tarinfo)
 458                                 extracted_anything = True
 459
 460                 # Due to a bug in tarfile (python versions < 2.5), we have to manually
 461                 # set the mtime of each directory that we extract after extracting everything.
 462
 463                 for tarinfo in ext_dirs:
 464                         dirname = os.path.join(destdir, tarinfo.name)
 465                         os.utime(dirname, (tarinfo.mtime, tarinfo.mtime))
 466
 467                 tar.close()
 468
 469                 if extract and not extracted_anything:
 470                         raise SafeException(_('Unable to find specified file = %s in archive') % extract)
 471
 472 def _extract(stream, destdir, command, start_offset = 0):
 473         """Run execvp('command') inside destdir in a child process, with
 474         stream seeked to 'start_offset' as stdin."""
 475
 476         # Some zip archives are missing timezone information; force consistent results
 477         child_env = os.environ.copy()
 478         child_env['TZ'] = 'GMT'
 479
 480         stream.seek(start_offset)
 481
 482         # TODO: use pola-run if available, once it supports fchmod
 483         child = subprocess.Popen(command, cwd = destdir, stdin = stream, stderr = subprocess.PIPE, env = child_env)
 484
 485         unused, cerr = child.communicate()
 486
 487         status = child.wait()
 488         if status != 0:
 489                 raise SafeException(_('Failed to extract archive (using %(command)s); exit code %(status)d:\n%(err)s') % {'command': command, 'status': status, 'err': cerr.strip()})