zeroinstall/zerostore/unpack.py

   1 """Unpacking archives of various formats."""
   2
   3 # Copyright (C) 2006, Thomas Leonard
   4 # See the README file for details, or visit http://0install.net.
   5
   6 import os
   7 import shutil
   8 import traceback
   9 from tempfile import mkdtemp, mkstemp
  10 import re
  11 from logging import debug, warn
  12 from zeroinstall import SafeException
  13 from zeroinstall.support import find_in_path, ro_rmtree
  14
  15 _cpio_version = None
  16 def _get_cpio_version():
  17         global _cpio_version
  18         if _cpio_version is None:
  19                 _cpio_version = os.popen('cpio --version 2>&1').next()
  20                 debug("cpio version = %s", _cpio_version)
  21         return _cpio_version
  22
  23 def _gnu_cpio():
  24         gnu_cpio = '(GNU cpio)' in _get_cpio_version()
  25         debug("Is GNU cpio = %s", gnu_cpio)
  26         return gnu_cpio
  27
  28 _tar_version = None
  29 def _get_tar_version():
  30         global _tar_version
  31         if _tar_version is None:
  32                 _tar_version = os.popen('tar --version 2>&1').next().strip()
  33                 debug("tar version = %s", _tar_version)
  34         return _tar_version
  35
  36 def _gnu_tar():
  37         gnu_tar = '(GNU tar)' in _get_tar_version()
  38         debug("Is GNU tar = %s", gnu_tar)
  39         return gnu_tar
  40
  41 def recent_gnu_tar():
  42         """@deprecated: should be private"""
  43         recent_gnu_tar = False
  44         if _gnu_tar():
  45                 version = re.search(r'\)\s*(\d+(\.\d+)*)', _get_tar_version())
  46                 if version:
  47                         version = map(int, version.group(1).split('.'))
  48                         recent_gnu_tar = version > [1, 13, 92]
  49                 else:
  50                         warn("Failed to extract GNU tar version number")
  51         debug("Recent GNU tar = %s", recent_gnu_tar)
  52         return recent_gnu_tar
  53
  54 # Disabled, as Plash does not currently support fchmod(2).
  55 _pola_run = None
  56 #_pola_run = find_in_path('pola-run')
  57 #if _pola_run:
  58 #       info('Found pola-run: %s', _pola_run)
  59 #else:
  60 #       info('pola-run not found; archive extraction will not be sandboxed')
  61
  62 def type_from_url(url):
  63         """Guess the MIME type for this resource based on its URL. Returns None if we don't know what it is."""
  64         url = url.lower()
  65         if url.endswith('.rpm'): return 'application/x-rpm'
  66         if url.endswith('.deb'): return 'application/x-deb'
  67         if url.endswith('.tar.bz2'): return 'application/x-bzip-compressed-tar'
  68         if url.endswith('.tar.gz'): return 'application/x-compressed-tar'
  69         if url.endswith('.tar.lzma'): return 'application/x-lzma-compressed-tar'        # XXX: No registered MIME type!
  70         if url.endswith('.tgz'): return 'application/x-compressed-tar'
  71         if url.endswith('.tar'): return 'application/x-tar'
  72         if url.endswith('.zip'): return 'application/zip'
  73         if url.endswith('.cab'): return 'application/vnd.ms-cab-compressed'
  74         return None
  75
  76 def check_type_ok(mime_type):
  77         """Check we have the needed software to extract from an archive of the given type.
  78         @raise SafeException: if the needed software is not available"""
  79         assert mime_type
  80         if mime_type == 'application/x-rpm':
  81                 if not find_in_path('rpm2cpio'):
  82                         raise SafeException("This package looks like an RPM, but you don't have the rpm2cpio command "
  83                                         "I need to extract it. Install the 'rpm' package first (this works even if "
  84                                         "you're on a non-RPM-based distribution such as Debian).")
  85         elif mime_type == 'application/x-deb':
  86                 if not find_in_path('ar'):
  87                         raise SafeException("This package looks like a Debian package, but you don't have the 'ar' command "
  88                                         "I need to extract it. Install the package containing it (sometimes called 'binutils') "
  89                                         "first. This works even if you're on a non-Debian-based distribution such as Red Hat).")
  90         elif mime_type == 'application/x-bzip-compressed-tar':
  91                 if not find_in_path('bunzip2'):
  92                         raise SafeException("This package looks like a bzip2-compressed package, but you don't have the 'bunzip2' command "
  93                                         "I need to extract it. Install the package containing it (it's probably called 'bzip2') "
  94                                         "first.")
  95         elif mime_type == 'application/zip':
  96                 if not find_in_path('unzip'):
  97                         raise SafeException("This package looks like a zip-compressed archive, but you don't have the 'unzip' command "
  98                                         "I need to extract it. Install the package containing it first.")
  99         elif mime_type == 'application/vnd.ms-cab-compressed':
 100                 if not find_in_path('cabextract'):
 101                         raise SafeException("This package looks like a Microsoft Cabinet archive, but you don't have the 'cabextract' command "
 102                                         "I need to extract it. Install the package containing it first.")
 103         elif mime_type == 'application/x-lzma-compressed-tar':
 104                 if not find_in_path('unlzma'):
 105                         raise SafeException("This package looks like an LZMA archive, but you don't have the 'unlzma' command "
 106                                         "I need to extract it. Install the package containing it (it's probably called 'lzma') first.")
 107         elif mime_type in ('application/x-compressed-tar', 'application/x-tar'):
 108                 pass
 109         else:
 110                 from zeroinstall import version
 111                 raise SafeException("Unsupported archive type '%s' (for injector version %s)" % (mime_type, version))
 112
 113 def _exec_maybe_sandboxed(writable, prog, *args):
 114         """execlp prog, with (only) the 'writable' directory writable if sandboxing is available.
 115         If no sandbox is available, run without a sandbox."""
 116         prog_path = find_in_path(prog)
 117         if not prog_path: raise Exception("'%s' not found in $PATH" % prog)
 118         if _pola_run is None:
 119                 os.execlp(prog_path, prog_path, *args)
 120         # We have pola-shell :-)
 121         pola_args = ['--prog', prog_path, '-f', '/']
 122         for a in args:
 123                 pola_args += ['-a', a]
 124         if writable:
 125                 pola_args += ['-fw', writable]
 126         os.execl(_pola_run, _pola_run, *pola_args)
 127
 128 def unpack_archive_over(url, data, destdir, extract = None, type = None, start_offset = 0):
 129         """Like unpack_archive, except that we unpack to a temporary directory first and
 130         then move things over, checking that we're not following symlinks at each stage.
 131         Use this when you want to unpack an unarchive into a directory which already has
 132         stuff in it.
 133         @since: 0.28"""
 134         import stat
 135         tmpdir = mkdtemp(dir = destdir)
 136         try:
 137                 mtimes = []
 138
 139                 unpack_archive(url, data, tmpdir, extract, type, start_offset)
 140
 141                 stem_len = len(tmpdir)
 142                 for root, dirs, files in os.walk(tmpdir):
 143                         relative_root = root[stem_len + 1:] or '.'
 144                         target_root = os.path.join(destdir, relative_root)
 145                         try:
 146                                 info = os.lstat(target_root)
 147                         except OSError, ex:
 148                                 if ex.errno != 2:
 149                                         raise   # Some odd error.
 150                                 # Doesn't exist. OK.
 151                                 os.mkdir(target_root)
 152                         else:
 153                                 if stat.S_ISLNK(info.st_mode):
 154                                         raise SafeException('Attempt to unpack dir over symlink "%s"!' % relative_root)
 155                                 elif not stat.S_ISDIR(info.st_mode):
 156                                         raise SafeException('Attempt to unpack dir over non-directory "%s"!' % relative_root)
 157                         mtimes.append((relative_root, os.lstat(os.path.join(tmpdir, root)).st_mtime))
 158
 159                         for s in dirs:  # Symlinks are counted as directories
 160                                 src = os.path.join(tmpdir, relative_root, s)
 161                                 if os.path.islink(src):
 162                                         files.append(s)
 163
 164                         for f in files:
 165                                 src = os.path.join(tmpdir, relative_root, f)
 166                                 dest = os.path.join(destdir, relative_root, f)
 167                                 if os.path.islink(dest):
 168                                         raise SafeException('Attempt to unpack file over symlink "%s"!' %
 169                                                         os.path.join(relative_root, f))
 170                                 os.rename(src, dest)
 171
 172                 for path, mtime in mtimes[1:]:
 173                         os.utime(os.path.join(destdir, path), (mtime, mtime))
 174         finally:
 175                 ro_rmtree(tmpdir)
 176
 177 def unpack_archive(url, data, destdir, extract = None, type = None, start_offset = 0):
 178         """Unpack stream 'data' into directory 'destdir'. If extract is given, extract just
 179         that sub-directory from the archive. Works out the format from the name."""
 180         if type is None: type = type_from_url(url)
 181         if type is None: raise SafeException("Unknown extension (and no MIME type given) in '%s'" % url)
 182         if type == 'application/x-bzip-compressed-tar':
 183                 extract_tar(data, destdir, extract, 'bzip2', start_offset)
 184         elif type == 'application/x-deb':
 185                 extract_deb(data, destdir, extract, start_offset)
 186         elif type == 'application/x-rpm':
 187                 extract_rpm(data, destdir, extract, start_offset)
 188         elif type == 'application/zip':
 189                 extract_zip(data, destdir, extract, start_offset)
 190         elif type == 'application/x-tar':
 191                 extract_tar(data, destdir, extract, None, start_offset)
 192         elif type == 'application/x-lzma-compressed-tar':
 193                 extract_tar(data, destdir, extract, 'lzma', start_offset)
 194         elif type == 'application/x-compressed-tar':
 195                 extract_tar(data, destdir, extract, 'gzip', start_offset)
 196         elif type == 'application/vnd.ms-cab-compressed':
 197                 extract_cab(data, destdir, extract, start_offset)
 198         else:
 199                 raise SafeException('Unknown MIME type "%s" for "%s"' % (type, url))
 200
 201 def extract_deb(stream, destdir, extract = None, start_offset = 0):
 202         if extract:
 203                 raise SafeException('Sorry, but the "extract" attribute is not yet supported for Debs')
 204
 205         stream.seek(start_offset)
 206         # ar can't read from stdin, so make a copy...
 207         deb_copy_name = os.path.join(destdir, 'archive.deb')
 208         deb_copy = file(deb_copy_name, 'w')
 209         shutil.copyfileobj(stream, deb_copy)
 210         deb_copy.close()
 211         _extract(stream, destdir, ('ar', 'x', 'archive.deb', 'data.tar.gz'))
 212         os.unlink(deb_copy_name)
 213         data_name = os.path.join(destdir, 'data.tar.gz')
 214         data_stream = file(data_name)
 215         os.unlink(data_name)
 216         extract_tar(data_stream, destdir, None, 'gzip')
 217
 218 def extract_rpm(stream, destdir, extract = None, start_offset = 0):
 219         if extract:
 220                 raise SafeException('Sorry, but the "extract" attribute is not yet supported for RPMs')
 221         fd, cpiopath = mkstemp('-rpm-tmp')
 222         try:
 223                 child = os.fork()
 224                 if child == 0:
 225                         try:
 226                                 try:
 227                                         os.dup2(stream.fileno(), 0)
 228                                         os.lseek(0, start_offset, 0)
 229                                         os.dup2(fd, 1)
 230                                         _exec_maybe_sandboxed(None, 'rpm2cpio', '-')
 231                                 except:
 232                                         traceback.print_exc()
 233                         finally:
 234                                 os._exit(1)
 235                 id, status = os.waitpid(child, 0)
 236                 assert id == child
 237                 if status != 0:
 238                         raise SafeException("rpm2cpio failed; can't unpack RPM archive; exit code %d" % status)
 239                 os.close(fd)
 240                 fd = None
 241
 242                 args = ['cpio', '-mid']
 243                 if _gnu_cpio():
 244                         args.append('--quiet')
 245
 246                 _extract(file(cpiopath), destdir, args)
 247                 # Set the mtime of every directory under 'tmp' to 0, since cpio doesn't
 248                 # preserve directory mtimes.
 249                 os.path.walk(destdir, lambda arg, dirname, names: os.utime(dirname, (0, 0)), None)
 250         finally:
 251                 if fd is not None:
 252                         os.close(fd)
 253                 os.unlink(cpiopath)
 254
 255 def extract_cab(stream, destdir, extract, start_offset = 0):
 256         "@since: 0.24"
 257         if extract:
 258                 raise SafeException('Sorry, but the "extract" attribute is not yet supported for Cabinet files')
 259
 260         stream.seek(start_offset)
 261         # cabextract can't read from stdin, so make a copy...
 262         cab_copy_name = os.path.join(destdir, 'archive.cab')
 263         cab_copy = file(cab_copy_name, 'w')
 264         shutil.copyfileobj(stream, cab_copy)
 265         cab_copy.close()
 266
 267         _extract(stream, destdir, ['cabextract', '-s', '-q', 'archive.cab'])
 268         os.unlink(cab_copy_name)
 269
 270 def extract_zip(stream, destdir, extract, start_offset = 0):
 271         if extract:
 272                 # Limit the characters we accept, to avoid sending dodgy
 273                 # strings to zip
 274                 if not re.match('^[a-zA-Z0-9][- _a-zA-Z0-9.]*$', extract):
 275                         raise SafeException('Illegal character in extract attribute')
 276
 277         stream.seek(start_offset)
 278         # unzip can't read from stdin, so make a copy...
 279         zip_copy_name = os.path.join(destdir, 'archive.zip')
 280         zip_copy = file(zip_copy_name, 'w')
 281         shutil.copyfileobj(stream, zip_copy)
 282         zip_copy.close()
 283
 284         args = ['unzip', '-q', '-o', 'archive.zip']
 285
 286         if extract:
 287                 args.append(extract + '/*')
 288
 289         _extract(stream, destdir, args)
 290         os.unlink(zip_copy_name)
 291
 292         if extract:
 293                 # unzip uses extract just as a filter, so we still need to move things
 294                 extracted_dir = os.path.join(destdir, extract)
 295                 for x in os.listdir(extracted_dir):
 296                         os.rename(os.path.join(extracted_dir, x), os.path.join(destdir, x))
 297                 os.rmdir(extracted_dir)
 298
 299 def extract_tar(stream, destdir, extract, decompress, start_offset = 0):
 300         if extract:
 301                 # Limit the characters we accept, to avoid sending dodgy
 302                 # strings to tar
 303                 if not re.match('^[a-zA-Z0-9][- _a-zA-Z0-9.]*$', extract):
 304                         raise SafeException('Illegal character in extract attribute')
 305
 306         assert decompress in [None, 'bzip2', 'gzip', 'lzma']
 307
 308         if _gnu_tar():
 309                 ext_cmd = ['tar']
 310                 if decompress:
 311                         if decompress == 'bzip2':
 312                                 ext_cmd.append('--bzip2')
 313                         elif decompress == 'gzip':
 314                                 ext_cmd.append('-z')
 315                         elif decompress == 'lzma':
 316                                 ext_cmd.append('--use-compress-program=unlzma')
 317
 318                 if recent_gnu_tar():
 319                         ext_cmd.extend(('-x', '--no-same-owner', '--no-same-permissions'))
 320                 else:
 321                         ext_cmd.extend(('xf', '-'))
 322
 323                 if extract:
 324                         ext_cmd.append(extract)
 325
 326                 _extract(stream, destdir, ext_cmd, start_offset)
 327         else:
 328                 # Since we don't have GNU tar, use python's tarfile module. This will probably
 329                 # be a lot slower and we do not support lzma; however, it is portable.
 330                 if decompress is None:
 331                         rmode = 'r|'
 332                 elif decompress == 'bzip2':
 333                         rmode = 'r|bz2'
 334                 elif decompress == 'gzip':
 335                         rmode = 'r|gz'
 336                 else:
 337                         raise SafeException('GNU tar unavailable; unsupported compression format: ' + decompress)
 338
 339                 import tarfile
 340
 341                 stream.seek(start_offset)
 342                 # Python 2.5.1 crashes if name is None; see Python bug #1706850
 343                 tar = tarfile.open(name = '', mode = rmode, fileobj = stream)
 344
 345                 current_umask = os.umask(0)
 346                 os.umask(current_umask)
 347
 348                 uid = gid = None
 349                 try:
 350                         uid = os.geteuid()
 351                         gid = os.getegid()
 352                 except:
 353                         debug("Can't get uid/gid")
 354
 355                 def chmod_extract(tarinfo):
 356                         # If any X bit is set, they all must be
 357                         if tarinfo.mode & 0111:
 358                                 tarinfo.mode |= 0111
 359
 360                         # Everyone gets read and write (subject to the umask)
 361                         # No special bits are allowed.
 362                         tarinfo.mode = ((tarinfo.mode | 0666) & ~current_umask) & 0777
 363
 364                         # Don't change owner, even if run as root
 365                         if uid:
 366                                 tarinfo.uid = uid
 367                         if gid:
 368                                 tarinfo.gid = gid
 369                         tar.extract(tarinfo, destdir)
 370
 371                 extracted_anything = False
 372                 ext_dirs = []
 373
 374                 for tarinfo in tar:
 375                         if extract is None or \
 376                            tarinfo.name.startswith(extract + '/') or \
 377                            tarinfo.name == extract:
 378                                 if tarinfo.isdir():
 379                                         ext_dirs.append(tarinfo)
 380
 381                                 chmod_extract(tarinfo)
 382                                 extracted_anything = True
 383
 384                 # Due to a bug in tarfile (python versions < 2.5), we have to manually
 385                 # set the mtime of each directory that we extract after extracting everything.
 386
 387                 for tarinfo in ext_dirs:
 388                         dirname = os.path.join(destdir, tarinfo.name)
 389                         os.utime(dirname, (tarinfo.mtime, tarinfo.mtime))
 390
 391                 tar.close()
 392
 393                 if extract and not extracted_anything:
 394                         raise SafeException('Unable to find specified file = %s in archive' % extract)
 395
 396 def _extract(stream, destdir, command, start_offset = 0):
 397         """Run execvp('command') inside destdir in a child process, with
 398         stream seeked to 'start_offset' as stdin."""
 399         child = os.fork()
 400         if child == 0:
 401                 try:
 402                         try:
 403                                 # Some zip archives are missing timezone information; force consistent results
 404                                 os.environ['TZ'] = 'GMT'
 405
 406                                 os.chdir(destdir)
 407                                 stream.seek(start_offset)
 408                                 os.dup2(stream.fileno(), 0)
 409                                 _exec_maybe_sandboxed(destdir, *command)
 410                         except:
 411                                 traceback.print_exc()
 412                 finally:
 413                         os._exit(1)
 414         id, status = os.waitpid(child, 0)
 415         assert id == child
 416         if status != 0:
 417                 raise SafeException('Failed to extract archive; exit code %d' % status)