zeroinstall/zerostore/unpack.py

   1 """Unpacking archives of various formats."""
   2
   3 # Copyright (C) 2006, Thomas Leonard
   4 # See the README file for details, or visit http://0install.net.
   5
   6 import os
   7 import shutil
   8 import traceback
   9 from tempfile import mkdtemp, mkstemp
  10 import sha
  11 import re
  12 from logging import debug, info, warn
  13 from zeroinstall import SafeException
  14 from zeroinstall.support import find_in_path, ro_rmtree
  15
  16 _cpio_version = None
  17 def _get_cpio_version():
  18         global _cpio_version
  19         if _cpio_version is None:
  20                 _cpio_version = os.popen('cpio --version 2>&1').next()
  21                 debug("cpio version = %s", _cpio_version)
  22         return _cpio_version
  23
  24 def _gnu_cpio():
  25         gnu_cpio = '(GNU cpio)' in _get_cpio_version()
  26         debug("Is GNU cpio = %s", gnu_cpio)
  27         return gnu_cpio
  28
  29 _tar_version = None
  30 def _get_tar_version():
  31         global _tar_version
  32         if _tar_version is None:
  33                 _tar_version = os.popen('tar --version 2>&1').next().strip()
  34                 debug("tar version = %s", _tar_version)
  35         return _tar_version
  36
  37 def _gnu_tar():
  38         gnu_tar = '(GNU tar)' in _get_tar_version()
  39         debug("Is GNU tar = %s", gnu_tar)
  40         return gnu_tar
  41
  42 def recent_gnu_tar():
  43         """@deprecated: should be private"""
  44         recent_gnu_tar = False
  45         if _gnu_tar():
  46                 version = _get_tar_version()
  47                 try:
  48                         version = version.split(')', 1)[1].strip()
  49                         assert version
  50                         version = map(int, version.split('.'))
  51                         recent_gnu_tar = version > [1, 13, 92]
  52                 except:
  53                         warn("Failed to extract GNU tar version number")
  54         debug("Recent GNU tar = %s", recent_gnu_tar)
  55         return recent_gnu_tar
  56
  57 _pola_run = find_in_path('pola-run')
  58 if _pola_run:
  59         info('Found pola-run: %s', _pola_run)
  60 else:
  61         info('pola-run not found; archive extraction will not be sandboxed')
  62
  63 def type_from_url(url):
  64         """Guess the MIME type for this resource based on its URL. Returns None if we don't know what it is."""
  65         url = url.lower()
  66         if url.endswith('.rpm'): return 'application/x-rpm'
  67         if url.endswith('.deb'): return 'application/x-deb'
  68         if url.endswith('.tar.bz2'): return 'application/x-bzip-compressed-tar'
  69         if url.endswith('.tar.gz'): return 'application/x-compressed-tar'
  70         if url.endswith('.tar.lzma'): return 'application/x-lzma-compressed-tar'        # XXX: No registered MIME type!
  71         if url.endswith('.tgz'): return 'application/x-compressed-tar'
  72         if url.endswith('.tar'): return 'application/x-tar'
  73         if url.endswith('.zip'): return 'application/zip'
  74         if url.endswith('.cab'): return 'application/vnd.ms-cab-compressed'
  75         return None
  76
  77 def check_type_ok(mime_type):
  78         """Check we have the needed software to extract from an archive of the given type.
  79         @raise SafeException: if the needed software is not available"""
  80         assert mime_type
  81         if mime_type == 'application/x-rpm':
  82                 if not find_in_path('rpm2cpio'):
  83                         raise SafeException("This package looks like an RPM, but you don't have the rpm2cpio command "
  84                                         "I need to extract it. Install the 'rpm' package first (this works even if "
  85                                         "you're on a non-RPM-based distribution such as Debian).")
  86         elif mime_type == 'application/x-deb':
  87                 if not find_in_path('ar'):
  88                         raise SafeException("This package looks like a Debian package, but you don't have the 'ar' command "
  89                                         "I need to extract it. Install the package containing it (sometimes called 'binutils') "
  90                                         "first. This works even if you're on a non-Debian-based distribution such as Red Hat).")
  91         elif mime_type == 'application/x-bzip-compressed-tar':
  92                 if not find_in_path('bunzip2'):
  93                         raise SafeException("This package looks like a bzip2-compressed package, but you don't have the 'bunzip2' command "
  94                                         "I need to extract it. Install the package containing it (it's probably called 'bzip2') "
  95                                         "first.")
  96         elif mime_type == 'application/zip':
  97                 if not find_in_path('unzip'):
  98                         raise SafeException("This package looks like a zip-compressed archive, but you don't have the 'unzip' command "
  99                                         "I need to extract it. Install the package containing it first.")
 100         elif mime_type == 'application/vnd.ms-cab-compressed':
 101                 if not find_in_path('cabextract'):
 102                         raise SafeException("This package looks like a Microsoft Cabinet archive, but you don't have the 'cabextract' command "
 103                                         "I need to extract it. Install the package containing it first.")
 104         elif mime_type == 'application/x-lzma-compressed-tar':
 105                 if not find_in_path('unlzma'):
 106                         raise SafeException("This package looks like an LZMA archive, but you don't have the 'unlzma' command "
 107                                         "I need to extract it. Install the package containing it (it's probably called 'lzma') first.")
 108         elif mime_type in ('application/x-compressed-tar', 'application/x-tar'):
 109                 pass
 110         else:
 111                 from zeroinstall import version
 112                 raise SafeException("Unsupported archive type '%s' (for injector version %s)" % (mime_type, version))
 113
 114 def _exec_maybe_sandboxed(writable, prog, *args):
 115         """execlp prog, with (only) the 'writable' directory writable if sandboxing is available.
 116         If no sandbox is available, run without a sandbox."""
 117         prog_path = find_in_path(prog)
 118         if not prog_path: raise Exception("'%s' not found in $PATH" % prog)
 119         if _pola_run is None:
 120                 os.execlp(prog_path, prog_path, *args)
 121         # We have pola-shell :-)
 122         pola_args = ['--prog', prog_path, '-f', '/']
 123         for a in args:
 124                 pola_args += ['-a', a]
 125         if writable:
 126                 pola_args += ['-fw', writable]
 127         os.execl(_pola_run, _pola_run, *pola_args)
 128
 129 def unpack_archive_over(url, data, destdir, extract = None, type = None, start_offset = 0):
 130         """Like unpack_archive, except that we unpack to a temporary directory first and
 131         then move things over, checking that we're not following symlinks at each stage.
 132         Use this when you want to unpack an unarchive into a directory which already has
 133         stuff in it.
 134         @since: 0.28"""
 135         import stat
 136         tmpdir = mkdtemp(dir = destdir)
 137         try:
 138                 mtimes = []
 139
 140                 unpack_archive(url, data, tmpdir, extract, type, start_offset)
 141
 142                 stem_len = len(tmpdir)
 143                 for root, dirs, files in os.walk(tmpdir):
 144                         relative_root = root[stem_len + 1:] or '.'
 145                         target_root = os.path.join(destdir, relative_root)
 146                         try:
 147                                 info = os.lstat(target_root)
 148                         except OSError, ex:
 149                                 if ex.errno != 2:
 150                                         raise   # Some odd error.
 151                                 # Doesn't exist. OK.
 152                                 os.mkdir(target_root)
 153                         else:
 154                                 if stat.S_ISLNK(info.st_mode):
 155                                         raise SafeException('Attempt to unpack dir over symlink "%s"!' % relative_root)
 156                                 elif not stat.S_ISDIR(info.st_mode):
 157                                         raise SafeException('Attempt to unpack dir over non-directory "%s"!' % relative_root)
 158                         mtimes.append((relative_root, os.lstat(os.path.join(tmpdir, root)).st_mtime))
 159
 160                         for s in dirs:  # Symlinks are counted as directories
 161                                 src = os.path.join(tmpdir, relative_root, s)
 162                                 if os.path.islink(src):
 163                                         files.append(s)
 164
 165                         for f in files:
 166                                 src = os.path.join(tmpdir, relative_root, f)
 167                                 dest = os.path.join(destdir, relative_root, f)
 168                                 if os.path.islink(dest):
 169                                         raise SafeException('Attempt to unpack file over symlink "%s"!' %
 170                                                         os.path.join(relative_root, f))
 171                                 os.rename(src, dest)
 172
 173                 for path, mtime in mtimes[1:]:
 174                         os.utime(os.path.join(destdir, path), (mtime, mtime))
 175         finally:
 176                 ro_rmtree(tmpdir)
 177
 178 def unpack_archive(url, data, destdir, extract = None, type = None, start_offset = 0):
 179         """Unpack stream 'data' into directory 'destdir'. If extract is given, extract just
 180         that sub-directory from the archive. Works out the format from the name."""
 181         if type is None: type = type_from_url(url)
 182         if type is None: raise SafeException("Unknown extension (and no MIME type given) in '%s'" % url)
 183         if type == 'application/x-bzip-compressed-tar':
 184                 extract_tar(data, destdir, extract, 'bzip2', start_offset)
 185         elif type == 'application/x-deb':
 186                 extract_deb(data, destdir, extract, start_offset)
 187         elif type == 'application/x-rpm':
 188                 extract_rpm(data, destdir, extract, start_offset)
 189         elif type == 'application/zip':
 190                 extract_zip(data, destdir, extract, start_offset)
 191         elif type == 'application/x-tar':
 192                 extract_tar(data, destdir, extract, None, start_offset)
 193         elif type == 'application/x-lzma-compressed-tar':
 194                 extract_tar(data, destdir, extract, 'lzma', start_offset)
 195         elif type == 'application/x-compressed-tar':
 196                 extract_tar(data, destdir, extract, 'gzip', start_offset)
 197         elif type == 'application/vnd.ms-cab-compressed':
 198                 extract_cab(data, destdir, extract, start_offset)
 199         else:
 200                 raise SafeException('Unknown MIME type "%s" for "%s"' % (type, url))
 201
 202 def extract_deb(stream, destdir, extract = None, start_offset = 0):
 203         if extract:
 204                 raise SafeException('Sorry, but the "extract" attribute is not yet supported for Debs')
 205
 206         stream.seek(start_offset)
 207         # ar can't read from stdin, so make a copy...
 208         deb_copy_name = os.path.join(destdir, 'archive.deb')
 209         deb_copy = file(deb_copy_name, 'w')
 210         shutil.copyfileobj(stream, deb_copy)
 211         deb_copy.close()
 212         _extract(stream, destdir, ('ar', 'x', 'archive.deb', 'data.tar.gz'))
 213         os.unlink(deb_copy_name)
 214         data_name = os.path.join(destdir, 'data.tar.gz')
 215         data_stream = file(data_name)
 216         os.unlink(data_name)
 217         extract_tar(data_stream, destdir, None, 'gzip')
 218
 219 def extract_rpm(stream, destdir, extract = None, start_offset = 0):
 220         if extract:
 221                 raise SafeException('Sorry, but the "extract" attribute is not yet supported for RPMs')
 222         fd, cpiopath = mkstemp('-rpm-tmp')
 223         try:
 224                 child = os.fork()
 225                 if child == 0:
 226                         try:
 227                                 try:
 228                                         os.dup2(stream.fileno(), 0)
 229                                         os.lseek(0, start_offset, 0)
 230                                         os.dup2(fd, 1)
 231                                         _exec_maybe_sandboxed(None, 'rpm2cpio', '-')
 232                                 except:
 233                                         traceback.print_exc()
 234                         finally:
 235                                 os._exit(1)
 236                 id, status = os.waitpid(child, 0)
 237                 assert id == child
 238                 if status != 0:
 239                         raise SafeException("rpm2cpio failed; can't unpack RPM archive; exit code %d" % status)
 240                 os.close(fd)
 241                 fd = None
 242
 243                 args = ['cpio', '-mid']
 244                 if _gnu_cpio():
 245                         args.append('--quiet')
 246
 247                 _extract(file(cpiopath), destdir, args)
 248                 # Set the mtime of every directory under 'tmp' to 0, since cpio doesn't
 249                 # preserve directory mtimes.
 250                 os.path.walk(destdir, lambda arg, dirname, names: os.utime(dirname, (0, 0)), None)
 251         finally:
 252                 if fd is not None:
 253                         os.close(fd)
 254                 os.unlink(cpiopath)
 255
 256 def extract_cab(stream, destdir, extract, start_offset = 0):
 257         "@since: 0.24"
 258         if extract:
 259                 raise SafeException('Sorry, but the "extract" attribute is not yet supported for Cabinet files')
 260
 261         stream.seek(start_offset)
 262         # cabextract can't read from stdin, so make a copy...
 263         cab_copy_name = os.path.join(destdir, 'archive.cab')
 264         cab_copy = file(cab_copy_name, 'w')
 265         shutil.copyfileobj(stream, cab_copy)
 266         cab_copy.close()
 267
 268         _extract(stream, destdir, ['cabextract', '-s', '-q', 'archive.cab'])
 269         os.unlink(cab_copy_name)
 270
 271 def extract_zip(stream, destdir, extract, start_offset = 0):
 272         if extract:
 273                 # Limit the characters we accept, to avoid sending dodgy
 274                 # strings to zip
 275                 if not re.match('^[a-zA-Z0-9][- _a-zA-Z0-9.]*$', extract):
 276                         raise SafeException('Illegal character in extract attribute')
 277
 278         stream.seek(start_offset)
 279         # unzip can't read from stdin, so make a copy...
 280         zip_copy_name = os.path.join(destdir, 'archive.zip')
 281         zip_copy = file(zip_copy_name, 'w')
 282         shutil.copyfileobj(stream, zip_copy)
 283         zip_copy.close()
 284
 285         args = ['unzip', '-q', '-o', 'archive.zip']
 286
 287         if extract:
 288                 args.append(extract + '/*')
 289
 290         _extract(stream, destdir, args)
 291         os.unlink(zip_copy_name)
 292
 293         if extract:
 294                 # unzip uses extract just as a filter, so we still need to move things
 295                 extracted_dir = os.path.join(destdir, extract)
 296                 for x in os.listdir(extracted_dir):
 297                         os.rename(os.path.join(extracted_dir, x), os.path.join(destdir, x))
 298                 os.rmdir(extracted_dir)
 299
 300 def extract_tar(stream, destdir, extract, decompress, start_offset = 0):
 301         if extract:
 302                 # Limit the characters we accept, to avoid sending dodgy
 303                 # strings to tar
 304                 if not re.match('^[a-zA-Z0-9][- _a-zA-Z0-9.]*$', extract):
 305                         raise SafeException('Illegal character in extract attribute')
 306
 307         assert decompress in [None, 'bzip2', 'gzip', 'lzma']
 308
 309         if _gnu_tar():
 310                 ext_cmd = ['tar']
 311                 if decompress:
 312                         if decompress == 'bzip2':
 313                                 ext_cmd.append('--bzip2')
 314                         elif decompress == 'gzip':
 315                                 ext_cmd.append('-z')
 316                         elif decompress == 'lzma':
 317                                 ext_cmd.append('--use-compress-program=unlzma')
 318
 319                 if recent_gnu_tar():
 320                         ext_cmd.extend(('-x', '--no-same-owner', '--no-same-permissions'))
 321                 else:
 322                         ext_cmd.extend(('xf', '-'))
 323
 324                 if extract:
 325                         ext_cmd.append(extract)
 326
 327                 _extract(stream, destdir, ext_cmd, start_offset)
 328         else:
 329                 # Since we don't have GNU tar, use python's tarfile module. This will probably
 330                 # be a lot slower and we do not support lzma; however, it is portable.
 331                 if decompress is None:
 332                         rmode = 'r|'
 333                 elif decompress == 'bzip2':
 334                         rmode = 'r|bz2'
 335                 elif decompress == 'gzip':
 336                         rmode = 'r|gz'
 337                 else:
 338                         raise SafeException('GNU tar unavailable; unsupported compression format: ' + decompress)
 339
 340                 import tarfile
 341
 342                 stream.seek(start_offset)
 343                 # Python 2.5.1 crashes if name is None; see Python bug #1706850
 344                 tar = tarfile.open(name = '', mode = rmode, fileobj = stream)
 345
 346                 current_umask = os.umask(0)
 347                 os.umask(current_umask)
 348
 349                 uid = gid = None
 350                 try:
 351                         uid = os.geteuid()
 352                         gid = os.getegid()
 353                 except:
 354                         debug("Can't get uid/gid")
 355
 356                 def chmod_extract(tarinfo):
 357                         # If any X bit is set, they all must be
 358                         if tarinfo.mode & 0111:
 359                                 tarinfo.mode |= 0111
 360
 361                         # Everyone gets read and write (subject to the umask)
 362                         # No special bits are allowed.
 363                         tarinfo.mode = ((tarinfo.mode | 0666) & ~current_umask) & 0777
 364
 365                         # Don't change owner, even if run as root
 366                         if uid:
 367                                 tarinfo.uid = uid
 368                         if gid:
 369                                 tarinfo.gid = gid
 370                         tar.extract(tarinfo, destdir)
 371
 372                 extracted_anything = False
 373                 ext_dirs = []
 374
 375                 for tarinfo in tar:
 376                         if extract is None or \
 377                            tarinfo.name.startswith(extract + '/') or \
 378                            tarinfo.name == extract:
 379                                 if tarinfo.isdir():
 380                                         ext_dirs.append(tarinfo)
 381
 382                                 chmod_extract(tarinfo)
 383                                 extracted_anything = True
 384
 385                 # Due to a bug in tarfile (python versions < 2.5), we have to manually
 386                 # set the mtime of each directory that we extract after extracting everything.
 387
 388                 for tarinfo in ext_dirs:
 389                         dirname = os.path.join(destdir, tarinfo.name)
 390                         os.utime(dirname, (tarinfo.mtime, tarinfo.mtime))
 391
 392                 tar.close()
 393
 394                 if extract and not extracted_anything:
 395                         raise SafeException('Unable to find specified file = %s in archive' % extract)
 396
 397 def _extract(stream, destdir, command, start_offset = 0):
 398         """Run execvp('command') inside destdir in a child process, with
 399         stream seeked to 'start_offset' as stdin."""
 400         child = os.fork()
 401         if child == 0:
 402                 try:
 403                         try:
 404                                 # Some zip archives are missing timezone information; force consistent results
 405                                 os.environ['TZ'] = 'GMT'
 406
 407                                 os.chdir(destdir)
 408                                 stream.seek(start_offset)
 409                                 os.dup2(stream.fileno(), 0)
 410                                 _exec_maybe_sandboxed(destdir, *command)
 411                         except:
 412                                 traceback.print_exc()
 413                 finally:
 414                         os._exit(1)
 415         id, status = os.waitpid(child, 0)
 416         assert id == child
 417         if status != 0:
 418                 raise SafeException('Failed to extract archive; exit code %d' % status)