Lib/tarfile.py

   1 #!/usr/bin/env python
   2 # -*- coding: iso-8859-1 -*-
   3 #-------------------------------------------------------------------
   4 # tarfile.py
   5 #-------------------------------------------------------------------
   6 # Copyright (C) 2002 Lars Gustäbel <lars@gustaebel.de>
   7 # All rights reserved.
   8 #
   9 # Permission  is  hereby granted,  free  of charge,  to  any person
  10 # obtaining a  copy of  this software  and associated documentation
  11 # files  (the  "Software"),  to   deal  in  the  Software   without
  12 # restriction,  including  without limitation  the  rights to  use,
  13 # copy, modify, merge, publish, distribute, sublicense, and/or sell
  14 # copies  of  the  Software,  and to  permit  persons  to  whom the
  15 # Software  is  furnished  to  do  so,  subject  to  the  following
  16 # conditions:
  17 #
  18 # The above copyright  notice and this  permission notice shall  be
  19 # included in all copies or substantial portions of the Software.
  20 #
  21 # THE SOFTWARE IS PROVIDED "AS  IS", WITHOUT WARRANTY OF ANY  KIND,
  22 # EXPRESS OR IMPLIED, INCLUDING  BUT NOT LIMITED TO  THE WARRANTIES
  23 # OF  MERCHANTABILITY,  FITNESS   FOR  A  PARTICULAR   PURPOSE  AND
  24 # NONINFRINGEMENT.  IN  NO  EVENT SHALL  THE  AUTHORS  OR COPYRIGHT
  25 # HOLDERS  BE LIABLE  FOR ANY  CLAIM, DAMAGES  OR OTHER  LIABILITY,
  26 # WHETHER  IN AN  ACTION OF  CONTRACT, TORT  OR OTHERWISE,  ARISING
  27 # FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  28 # OTHER DEALINGS IN THE SOFTWARE.
  29 #
  30 """Read from and write to tar format archives.
  31 """
  32
  33 __version__ = "$Revision$"
  34 # $Source$
  35
  36 version     = "0.6.4"
  37 __author__  = "Lars Gustäbel (lars@gustaebel.de)"
  38 __date__    = "$Date$"
  39 __cvsid__   = "$Id$"
  40 __credits__ = "Gustavo Niemeyer, Niels Gustäbel, Richard Townsend."
  41
  42 #---------
  43 # Imports
  44 #---------
  45 import sys
  46 import os
  47 import shutil
  48 import stat
  49 import errno
  50 import time
  51 import struct
  52
  53 if sys.platform == 'mac':
  54     # This module needs work for MacOS9, especially in the area of pathname
  55     # handling. In many places it is assumed a simple substitution of / by the
  56     # local os.path.sep is good enough to convert pathnames, but this does not
  57     # work with the mac rooted:path:name versus :nonrooted:path:name syntax
  58     raise ImportError, "tarfile does not work for platform==mac"
  59
  60 try:
  61     import grp, pwd
  62 except ImportError:
  63     grp = pwd = None
  64
  65 # from tarfile import *
  66 __all__ = ["TarFile", "TarInfo", "is_tarfile", "TarError"]
  67
  68 #---------------------------------------------------------
  69 # tar constants
  70 #---------------------------------------------------------
  71 NUL        = "\0"               # the null character
  72 BLOCKSIZE  = 512                # length of processing blocks
  73 RECORDSIZE = BLOCKSIZE * 20     # length of records
  74 MAGIC      = "ustar"            # magic tar string
  75 VERSION    = "00"               # version number
  76
  77 LENGTH_NAME    = 100            # maximum length of a filename
  78 LENGTH_LINK    = 100            # maximum length of a linkname
  79 LENGTH_PREFIX  = 155            # maximum length of the prefix field
  80 MAXSIZE_MEMBER = 077777777777L  # maximum size of a file (11 octal digits)
  81
  82 REGTYPE  = "0"                  # regular file
  83 AREGTYPE = "\0"                 # regular file
  84 LNKTYPE  = "1"                  # link (inside tarfile)
  85 SYMTYPE  = "2"                  # symbolic link
  86 CHRTYPE  = "3"                  # character special device
  87 BLKTYPE  = "4"                  # block special device
  88 DIRTYPE  = "5"                  # directory
  89 FIFOTYPE = "6"                  # fifo special device
  90 CONTTYPE = "7"                  # contiguous file
  91
  92 GNUTYPE_LONGNAME = "L"          # GNU tar extension for longnames
  93 GNUTYPE_LONGLINK = "K"          # GNU tar extension for longlink
  94 GNUTYPE_SPARSE   = "S"          # GNU tar extension for sparse file
  95
  96 #---------------------------------------------------------
  97 # tarfile constants
  98 #---------------------------------------------------------
  99 SUPPORTED_TYPES = (REGTYPE, AREGTYPE, LNKTYPE,  # file types that tarfile
 100                    SYMTYPE, DIRTYPE, FIFOTYPE,  # can cope with.
 101                    CONTTYPE, CHRTYPE, BLKTYPE,
 102                    GNUTYPE_LONGNAME, GNUTYPE_LONGLINK,
 103                    GNUTYPE_SPARSE)
 104
 105 REGULAR_TYPES = (REGTYPE, AREGTYPE,             # file types that somehow
 106                  CONTTYPE, GNUTYPE_SPARSE)      # represent regular files
 107
 108 #---------------------------------------------------------
 109 # Bits used in the mode field, values in octal.
 110 #---------------------------------------------------------
 111 S_IFLNK = 0120000        # symbolic link
 112 S_IFREG = 0100000        # regular file
 113 S_IFBLK = 0060000        # block device
 114 S_IFDIR = 0040000        # directory
 115 S_IFCHR = 0020000        # character device
 116 S_IFIFO = 0010000        # fifo
 117
 118 TSUID   = 04000          # set UID on execution
 119 TSGID   = 02000          # set GID on execution
 120 TSVTX   = 01000          # reserved
 121
 122 TUREAD  = 0400           # read by owner
 123 TUWRITE = 0200           # write by owner
 124 TUEXEC  = 0100           # execute/search by owner
 125 TGREAD  = 0040           # read by group
 126 TGWRITE = 0020           # write by group
 127 TGEXEC  = 0010           # execute/search by group
 128 TOREAD  = 0004           # read by other
 129 TOWRITE = 0002           # write by other
 130 TOEXEC  = 0001           # execute/search by other
 131
 132 #---------------------------------------------------------
 133 # Some useful functions
 134 #---------------------------------------------------------
 135 def nts(s):
 136     """Convert a null-terminated string buffer to a python string.
 137     """
 138     return s.rstrip(NUL)
 139
 140 def calc_chksum(buf):
 141     """Calculate the checksum for a member's header. It's a simple addition
 142        of all bytes, treating the chksum field as if filled with spaces.
 143        buf is a 512 byte long string buffer which holds the header.
 144     """
 145     chk = 256                           # chksum field is treated as blanks,
 146                                         # so the initial value is 8 * ord(" ")
 147     for c in buf[:148]: chk += ord(c)   # sum up all bytes before chksum
 148     for c in buf[156:]: chk += ord(c)   # sum up all bytes after chksum
 149     return chk
 150
 151 def copyfileobj(src, dst, length=None):
 152     """Copy length bytes from fileobj src to fileobj dst.
 153        If length is None, copy the entire content.
 154     """
 155     if length == 0:
 156         return
 157     if length is None:
 158         shutil.copyfileobj(src, dst)
 159         return
 160
 161     BUFSIZE = 16 * 1024
 162     blocks, remainder = divmod(length, BUFSIZE)
 163     for b in xrange(blocks):
 164         buf = src.read(BUFSIZE)
 165         if len(buf) < BUFSIZE:
 166             raise IOError, "end of file reached"
 167         dst.write(buf)
 168
 169     if remainder != 0:
 170         buf = src.read(remainder)
 171         if len(buf) < remainder:
 172             raise IOError, "end of file reached"
 173         dst.write(buf)
 174     return
 175
 176 filemode_table = (
 177     ((S_IFLNK,      "l"),
 178      (S_IFREG,      "-"),
 179      (S_IFBLK,      "b"),
 180      (S_IFDIR,      "d"),
 181      (S_IFCHR,      "c"),
 182      (S_IFIFO,      "p")),
 183
 184     ((TUREAD,       "r"),),
 185     ((TUWRITE,      "w"),),
 186     ((TUEXEC|TSUID, "s"),
 187      (TSUID,        "S"),
 188      (TUEXEC,       "x")),
 189
 190     ((TGREAD,       "r"),),
 191     ((TGWRITE,      "w"),),
 192     ((TGEXEC|TSGID, "s"),
 193      (TSGID,        "S"),
 194      (TGEXEC,       "x")),
 195
 196     ((TOREAD,       "r"),),
 197     ((TOWRITE,      "w"),),
 198     ((TOEXEC|TSVTX, "t"),
 199      (TSVTX,        "T"),
 200      (TOEXEC,       "x"))
 201 )
 202
 203 def filemode(mode):
 204     """Convert a file's mode to a string of the form
 205        -rwxrwxrwx.
 206        Used by TarFile.list()
 207     """
 208     perm = []
 209     for table in filemode_table:
 210         for bit, char in table:
 211             if mode & bit == bit:
 212                 perm.append(char)
 213                 break
 214         else:
 215             perm.append("-")
 216     return "".join(perm)
 217
 218 if os.sep != "/":
 219     normpath = lambda path: os.path.normpath(path).replace(os.sep, "/")
 220 else:
 221     normpath = os.path.normpath
 222
 223 class TarError(Exception):
 224     """Base exception."""
 225     pass
 226 class ExtractError(TarError):
 227     """General exception for extract errors."""
 228     pass
 229 class ReadError(TarError):
 230     """Exception for unreadble tar archives."""
 231     pass
 232 class CompressionError(TarError):
 233     """Exception for unavailable compression methods."""
 234     pass
 235 class StreamError(TarError):
 236     """Exception for unsupported operations on stream-like TarFiles."""
 237     pass
 238
 239 #---------------------------
 240 # internal stream interface
 241 #---------------------------
 242 class _LowLevelFile:
 243     """Low-level file object. Supports reading and writing.
 244        It is used instead of a regular file object for streaming
 245        access.
 246     """
 247
 248     def __init__(self, name, mode):
 249         mode = {
 250             "r": os.O_RDONLY,
 251             "w": os.O_WRONLY | os.O_CREAT | os.O_TRUNC,
 252         }[mode]
 253         if hasattr(os, "O_BINARY"):
 254             mode |= os.O_BINARY
 255         self.fd = os.open(name, mode)
 256
 257     def close(self):
 258         os.close(self.fd)
 259
 260     def read(self, size):
 261         return os.read(self.fd, size)
 262
 263     def write(self, s):
 264         os.write(self.fd, s)
 265
 266 class _Stream:
 267     """Class that serves as an adapter between TarFile and
 268        a stream-like object.  The stream-like object only
 269        needs to have a read() or write() method and is accessed
 270        blockwise.  Use of gzip or bzip2 compression is possible.
 271        A stream-like object could be for example: sys.stdin,
 272        sys.stdout, a socket, a tape device etc.
 273
 274        _Stream is intended to be used only internally.
 275     """
 276
 277     def __init__(self, name, mode, comptype, fileobj, bufsize):
 278         """Construct a _Stream object.
 279         """
 280         self._extfileobj = True
 281         if fileobj is None:
 282             fileobj = _LowLevelFile(name, mode)
 283             self._extfileobj = False
 284
 285         if comptype == '*':
 286             # Enable transparent compression detection for the
 287             # stream interface
 288             fileobj = _StreamProxy(fileobj)
 289             comptype = fileobj.getcomptype()
 290
 291         self.name     = name or ""
 292         self.mode     = mode
 293         self.comptype = comptype
 294         self.fileobj  = fileobj
 295         self.bufsize  = bufsize
 296         self.buf      = ""
 297         self.pos      = 0L
 298         self.closed   = False
 299
 300         if comptype == "gz":
 301             try:
 302                 import zlib
 303             except ImportError:
 304                 raise CompressionError, "zlib module is not available"
 305             self.zlib = zlib
 306             self.crc = zlib.crc32("")
 307             if mode == "r":
 308                 self._init_read_gz()
 309             else:
 310                 self._init_write_gz()
 311
 312         if comptype == "bz2":
 313             try:
 314                 import bz2
 315             except ImportError:
 316                 raise CompressionError, "bz2 module is not available"
 317             if mode == "r":
 318                 self.dbuf = ""
 319                 self.cmp = bz2.BZ2Decompressor()
 320             else:
 321                 self.cmp = bz2.BZ2Compressor()
 322
 323     def __del__(self):
 324         if hasattr(self, "closed") and not self.closed:
 325             self.close()
 326
 327     def _init_write_gz(self):
 328         """Initialize for writing with gzip compression.
 329         """
 330         self.cmp = self.zlib.compressobj(9, self.zlib.DEFLATED,
 331                                             -self.zlib.MAX_WBITS,
 332                                             self.zlib.DEF_MEM_LEVEL,
 333                                             0)
 334         timestamp = struct.pack("<L", long(time.time()))
 335         self.__write("\037\213\010\010%s\002\377" % timestamp)
 336         if self.name.endswith(".gz"):
 337             self.name = self.name[:-3]
 338         self.__write(self.name + NUL)
 339
 340     def write(self, s):
 341         """Write string s to the stream.
 342         """
 343         if self.comptype == "gz":
 344             self.crc = self.zlib.crc32(s, self.crc)
 345         self.pos += len(s)
 346         if self.comptype != "tar":
 347             s = self.cmp.compress(s)
 348         self.__write(s)
 349
 350     def __write(self, s):
 351         """Write string s to the stream if a whole new block
 352            is ready to be written.
 353         """
 354         self.buf += s
 355         while len(self.buf) > self.bufsize:
 356             self.fileobj.write(self.buf[:self.bufsize])
 357             self.buf = self.buf[self.bufsize:]
 358
 359     def close(self):
 360         """Close the _Stream object. No operation should be
 361            done on it afterwards.
 362         """
 363         if self.closed:
 364             return
 365
 366         if self.mode == "w" and self.comptype != "tar":
 367             self.buf += self.cmp.flush()
 368
 369         if self.mode == "w" and self.buf:
 370             blocks, remainder = divmod(len(self.buf), self.bufsize)
 371             if remainder > 0:
 372                 self.buf += NUL * (self.bufsize - remainder)
 373             self.fileobj.write(self.buf)
 374             self.buf = ""
 375             if self.comptype == "gz":
 376                 self.fileobj.write(struct.pack("<l", self.crc))
 377                 self.fileobj.write(struct.pack("<L", self.pos & 0xffffFFFFL))
 378
 379         if not self._extfileobj:
 380             self.fileobj.close()
 381
 382         self.closed = True
 383
 384     def _init_read_gz(self):
 385         """Initialize for reading a gzip compressed fileobj.
 386         """
 387         self.cmp = self.zlib.decompressobj(-self.zlib.MAX_WBITS)
 388         self.dbuf = ""
 389
 390         # taken from gzip.GzipFile with some alterations
 391         if self.__read(2) != "\037\213":
 392             raise ReadError, "not a gzip file"
 393         if self.__read(1) != "\010":
 394             raise CompressionError, "unsupported compression method"
 395
 396         flag = ord(self.__read(1))
 397         self.__read(6)
 398
 399         if flag & 4:
 400             xlen = ord(self.__read(1)) + 256 * ord(self.__read(1))
 401             self.read(xlen)
 402         if flag & 8:
 403             while True:
 404                 s = self.__read(1)
 405                 if not s or s == NUL:
 406                     break
 407         if flag & 16:
 408             while True:
 409                 s = self.__read(1)
 410                 if not s or s == NUL:
 411                     break
 412         if flag & 2:
 413             self.__read(2)
 414
 415     def tell(self):
 416         """Return the stream's file pointer position.
 417         """
 418         return self.pos
 419
 420     def seek(self, pos=0):
 421         """Set the stream's file pointer to pos. Negative seeking
 422            is forbidden.
 423         """
 424         if pos - self.pos >= 0:
 425             blocks, remainder = divmod(pos - self.pos, self.bufsize)
 426             for i in xrange(blocks):
 427                 self.read(self.bufsize)
 428             self.read(remainder)
 429         else:
 430             raise StreamError, "seeking backwards is not allowed"
 431         return self.pos
 432
 433     def read(self, size=None):
 434         """Return the next size number of bytes from the stream.
 435            If size is not defined, return all bytes of the stream
 436            up to EOF.
 437         """
 438         if size is None:
 439             t = []
 440             while True:
 441                 buf = self._read(self.bufsize)
 442                 if not buf:
 443                     break
 444                 t.append(buf)
 445             buf = "".join(t)
 446         else:
 447             buf = self._read(size)
 448         self.pos += len(buf)
 449         return buf
 450
 451     def _read(self, size):
 452         """Return size bytes from the stream.
 453         """
 454         if self.comptype == "tar":
 455             return self.__read(size)
 456
 457         c = len(self.dbuf)
 458         t = [self.dbuf]
 459         while c < size:
 460             buf = self.__read(self.bufsize)
 461             if not buf:
 462                 break
 463             buf = self.cmp.decompress(buf)
 464             t.append(buf)
 465             c += len(buf)
 466         t = "".join(t)
 467         self.dbuf = t[size:]
 468         return t[:size]
 469
 470     def __read(self, size):
 471         """Return size bytes from stream. If internal buffer is empty,
 472            read another block from the stream.
 473         """
 474         c = len(self.buf)
 475         t = [self.buf]
 476         while c < size:
 477             buf = self.fileobj.read(self.bufsize)
 478             if not buf:
 479                 break
 480             t.append(buf)
 481             c += len(buf)
 482         t = "".join(t)
 483         self.buf = t[size:]
 484         return t[:size]
 485 # class _Stream
 486
 487 class _StreamProxy(object):
 488     """Small proxy class that enables transparent compression
 489        detection for the Stream interface (mode 'r|*').
 490     """
 491
 492     def __init__(self, fileobj):
 493         self.fileobj = fileobj
 494         self.buf = self.fileobj.read(BLOCKSIZE)
 495
 496     def read(self, size):
 497         self.read = self.fileobj.read
 498         return self.buf
 499
 500     def getcomptype(self):
 501         if self.buf.startswith("\037\213\010"):
 502             return "gz"
 503         if self.buf.startswith("BZh91"):
 504             return "bz2"
 505         return "tar"
 506
 507     def close(self):
 508         self.fileobj.close()
 509 # class StreamProxy
 510
 511 #------------------------
 512 # Extraction file object
 513 #------------------------
 514 class ExFileObject(object):
 515     """File-like object for reading an archive member.
 516        Is returned by TarFile.extractfile(). Support for
 517        sparse files included.
 518     """
 519
 520     def __init__(self, tarfile, tarinfo):
 521         self.fileobj = tarfile.fileobj
 522         self.name    = tarinfo.name
 523         self.mode    = "r"
 524         self.closed  = False
 525         self.offset  = tarinfo.offset_data
 526         self.size    = tarinfo.size
 527         self.pos     = 0L
 528         self.linebuffer = ""
 529         if tarinfo.issparse():
 530             self.sparse = tarinfo.sparse
 531             self.read = self._readsparse
 532         else:
 533             self.read = self._readnormal
 534
 535     def __read(self, size):
 536         """Overloadable read method.
 537         """
 538         return self.fileobj.read(size)
 539
 540     def readline(self, size=-1):
 541         """Read a line with approx. size. If size is negative,
 542            read a whole line. readline() and read() must not
 543            be mixed up (!).
 544         """
 545         if size < 0:
 546             size = sys.maxint
 547
 548         nl = self.linebuffer.find("\n")
 549         if nl >= 0:
 550             nl = min(nl, size)
 551         else:
 552             size -= len(self.linebuffer)
 553             while (nl < 0 and size > 0):
 554                 buf = self.read(min(size, 100))
 555                 if not buf:
 556                     break
 557                 self.linebuffer += buf
 558                 size -= len(buf)
 559                 nl = self.linebuffer.find("\n")
 560             if nl == -1:
 561                 s = self.linebuffer
 562                 self.linebuffer = ""
 563                 return s
 564         buf = self.linebuffer[:nl]
 565         self.linebuffer = self.linebuffer[nl + 1:]
 566         while buf[-1:] == "\r":
 567             buf = buf[:-1]
 568         return buf + "\n"
 569
 570     def readlines(self):
 571         """Return a list with all (following) lines.
 572         """
 573         result = []
 574         while True:
 575             line = self.readline()
 576             if not line: break
 577             result.append(line)
 578         return result
 579
 580     def _readnormal(self, size=None):
 581         """Read operation for regular files.
 582         """
 583         if self.closed:
 584             raise ValueError, "file is closed"
 585         self.fileobj.seek(self.offset + self.pos)
 586         bytesleft = self.size - self.pos
 587         if size is None:
 588             bytestoread = bytesleft
 589         else:
 590             bytestoread = min(size, bytesleft)
 591         self.pos += bytestoread
 592         return self.__read(bytestoread)
 593
 594     def _readsparse(self, size=None):
 595         """Read operation for sparse files.
 596         """
 597         if self.closed:
 598             raise ValueError, "file is closed"
 599
 600         if size is None:
 601             size = self.size - self.pos
 602
 603         data = []
 604         while size > 0:
 605             buf = self._readsparsesection(size)
 606             if not buf:
 607                 break
 608             size -= len(buf)
 609             data.append(buf)
 610         return "".join(data)
 611
 612     def _readsparsesection(self, size):
 613         """Read a single section of a sparse file.
 614         """
 615         section = self.sparse.find(self.pos)
 616
 617         if section is None:
 618             return ""
 619
 620         toread = min(size, section.offset + section.size - self.pos)
 621         if isinstance(section, _data):
 622             realpos = section.realpos + self.pos - section.offset
 623             self.pos += toread
 624             self.fileobj.seek(self.offset + realpos)
 625             return self.__read(toread)
 626         else:
 627             self.pos += toread
 628             return NUL * toread
 629
 630     def tell(self):
 631         """Return the current file position.
 632         """
 633         return self.pos
 634
 635     def seek(self, pos, whence=0):
 636         """Seek to a position in the file.
 637         """
 638         self.linebuffer = ""
 639         if whence == 0:
 640             self.pos = min(max(pos, 0), self.size)
 641         if whence == 1:
 642             if pos < 0:
 643                 self.pos = max(self.pos + pos, 0)
 644             else:
 645                 self.pos = min(self.pos + pos, self.size)
 646         if whence == 2:
 647             self.pos = max(min(self.size + pos, self.size), 0)
 648
 649     def close(self):
 650         """Close the file object.
 651         """
 652         self.closed = True
 653
 654     def __iter__(self):
 655         """Get an iterator over the file object.
 656         """
 657         if self.closed:
 658             raise ValueError("I/O operation on closed file")
 659         return self
 660
 661     def next(self):
 662         """Get the next item from the file iterator.
 663         """
 664         result = self.readline()
 665         if not result:
 666             raise StopIteration
 667         return result
 668
 669 #class ExFileObject
 670
 671 #------------------
 672 # Exported Classes
 673 #------------------
 674 class TarInfo(object):
 675     """Informational class which holds the details about an
 676        archive member given by a tar header block.
 677        TarInfo objects are returned by TarFile.getmember(),
 678        TarFile.getmembers() and TarFile.gettarinfo() and are
 679        usually created internally.
 680     """
 681
 682     def __init__(self, name=""):
 683         """Construct a TarInfo object. name is the optional name
 684            of the member.
 685         """
 686
 687         self.name     = name       # member name (dirnames must end with '/')
 688         self.mode     = 0666       # file permissions
 689         self.uid      = 0          # user id
 690         self.gid      = 0          # group id
 691         self.size     = 0          # file size
 692         self.mtime    = 0          # modification time
 693         self.chksum   = 0          # header checksum
 694         self.type     = REGTYPE    # member type
 695         self.linkname = ""         # link name
 696         self.uname    = "user"     # user name
 697         self.gname    = "group"    # group name
 698         self.devmajor = 0          #-
 699         self.devminor = 0          #-for use with CHRTYPE and BLKTYPE
 700         self.prefix   = ""         # prefix to filename or holding information
 701                                    # about sparse files
 702
 703         self.offset   = 0          # the tar header starts here
 704         self.offset_data = 0       # the file's data starts here
 705
 706     def __repr__(self):
 707         return "<%s %r at %#x>" % (self.__class__.__name__,self.name,id(self))
 708
 709     @classmethod
 710     def frombuf(cls, buf):
 711         """Construct a TarInfo object from a 512 byte string buffer.
 712         """
 713         tarinfo = cls()
 714         tarinfo.name   = nts(buf[0:100])
 715         tarinfo.mode   = int(buf[100:108], 8)
 716         tarinfo.uid    = int(buf[108:116],8)
 717         tarinfo.gid    = int(buf[116:124],8)
 718
 719         # There are two possible codings for the size field we
 720         # have to discriminate, see comment in tobuf() below.
 721         if buf[124] != chr(0200):
 722             tarinfo.size = long(buf[124:136], 8)
 723         else:
 724             tarinfo.size = 0L
 725             for i in range(11):
 726                 tarinfo.size <<= 8
 727                 tarinfo.size += ord(buf[125 + i])
 728
 729         tarinfo.mtime  = long(buf[136:148], 8)
 730         tarinfo.chksum = int(buf[148:156], 8)
 731         tarinfo.type   = buf[156:157]
 732         tarinfo.linkname = nts(buf[157:257])
 733         tarinfo.uname  = nts(buf[265:297])
 734         tarinfo.gname  = nts(buf[297:329])
 735         try:
 736             tarinfo.devmajor = int(buf[329:337], 8)
 737             tarinfo.devminor = int(buf[337:345], 8)
 738         except ValueError:
 739             tarinfo.devmajor = tarinfo.devmajor = 0
 740         tarinfo.prefix = buf[345:500]
 741
 742         # Some old tar programs represent a directory as a regular
 743         # file with a trailing slash.
 744         if tarinfo.isreg() and tarinfo.name.endswith("/"):
 745             tarinfo.type = DIRTYPE
 746
 747         # The prefix field is used for filenames > 100 in
 748         # the POSIX standard.
 749         # name = prefix + '/' + name
 750         if tarinfo.type != GNUTYPE_SPARSE:
 751             tarinfo.name = normpath(os.path.join(nts(tarinfo.prefix), tarinfo.name))
 752
 753         # Directory names should have a '/' at the end.
 754         if tarinfo.isdir():
 755             tarinfo.name += "/"
 756         return tarinfo
 757
 758     def tobuf(self):
 759         """Return a tar header block as a 512 byte string.
 760         """
 761         # Prefer the size to be encoded as 11 octal ascii digits
 762         # which is the most portable. If the size exceeds this
 763         # limit (>= 8 GB), encode it as an 88-bit value which is
 764         # a GNU tar feature.
 765         if self.size <= MAXSIZE_MEMBER:
 766             size = "%011o" % self.size
 767         else:
 768             s = self.size
 769             size = ""
 770             for i in range(11):
 771                 size = chr(s & 0377) + size
 772                 s >>= 8
 773             size = chr(0200) + size
 774
 775         # The following code was contributed by Detlef Lannert.
 776         parts = []
 777         for value, fieldsize in (
 778                 (self.name, 100),
 779                 ("%07o" % (self.mode & 07777), 8),
 780                 ("%07o" % self.uid, 8),
 781                 ("%07o" % self.gid, 8),
 782                 (size, 12),
 783                 ("%011o" % self.mtime, 12),
 784                 ("        ", 8),
 785                 (self.type, 1),
 786                 (self.linkname, 100),
 787                 (MAGIC, 6),
 788                 (VERSION, 2),
 789                 (self.uname, 32),
 790                 (self.gname, 32),
 791                 ("%07o" % self.devmajor, 8),
 792                 ("%07o" % self.devminor, 8),
 793                 (self.prefix, 155)
 794             ):
 795             l = len(value)
 796             parts.append(value[:fieldsize] + (fieldsize - l) * NUL)
 797
 798         buf = "".join(parts)
 799         chksum = calc_chksum(buf)
 800         buf = buf[:148] + "%06o\0" % chksum + buf[155:]
 801         buf += (BLOCKSIZE - len(buf)) * NUL
 802         self.buf = buf
 803         return buf
 804
 805     def isreg(self):
 806         return self.type in REGULAR_TYPES
 807     def isfile(self):
 808         return self.isreg()
 809     def isdir(self):
 810         return self.type == DIRTYPE
 811     def issym(self):
 812         return self.type == SYMTYPE
 813     def islnk(self):
 814         return self.type == LNKTYPE
 815     def ischr(self):
 816         return self.type == CHRTYPE
 817     def isblk(self):
 818         return self.type == BLKTYPE
 819     def isfifo(self):
 820         return self.type == FIFOTYPE
 821     def issparse(self):
 822         return self.type == GNUTYPE_SPARSE
 823     def isdev(self):
 824         return self.type in (CHRTYPE, BLKTYPE, FIFOTYPE)
 825 # class TarInfo
 826
 827 class TarFile(object):
 828     """The TarFile Class provides an interface to tar archives.
 829     """
 830
 831     debug = 0                   # May be set from 0 (no msgs) to 3 (all msgs)
 832
 833     dereference = False         # If true, add content of linked file to the
 834                                 # tar file, else the link.
 835
 836     ignore_zeros = False        # If true, skips empty or invalid blocks and
 837                                 # continues processing.
 838
 839     errorlevel = 0              # If 0, fatal errors only appear in debug
 840                                 # messages (if debug >= 0). If > 0, errors
 841                                 # are passed to the caller as exceptions.
 842
 843     posix = False               # If True, generates POSIX.1-1990-compliant
 844                                 # archives (no GNU extensions!)
 845
 846     fileobject = ExFileObject
 847
 848     def __init__(self, name=None, mode="r", fileobj=None):
 849         """Open an (uncompressed) tar archive `name'. `mode' is either 'r' to
 850            read from an existing archive, 'a' to append data to an existing
 851            file or 'w' to create a new file overwriting an existing one. `mode'
 852            defaults to 'r'.
 853            If `fileobj' is given, it is used for reading or writing data. If it
 854            can be determined, `mode' is overridden by `fileobj's mode.
 855            `fileobj' is not closed, when TarFile is closed.
 856         """
 857         self.name = name
 858
 859         if len(mode) > 1 or mode not in "raw":
 860             raise ValueError, "mode must be 'r', 'a' or 'w'"
 861         self._mode = mode
 862         self.mode = {"r": "rb", "a": "r+b", "w": "wb"}[mode]
 863
 864         if not fileobj:
 865             fileobj = file(self.name, self.mode)
 866             self._extfileobj = False
 867         else:
 868             if self.name is None and hasattr(fileobj, "name"):
 869                 self.name = fileobj.name
 870             if hasattr(fileobj, "mode"):
 871                 self.mode = fileobj.mode
 872             self._extfileobj = True
 873         self.fileobj = fileobj
 874
 875         # Init datastructures
 876         self.closed      = False
 877         self.members     = []       # list of members as TarInfo objects
 878         self._loaded     = False    # flag if all members have been read
 879         self.offset      = 0L       # current position in the archive file
 880         self.inodes      = {}       # dictionary caching the inodes of
 881                                     # archive members already added
 882
 883         if self._mode == "r":
 884             self.firstmember = None
 885             self.firstmember = self.next()
 886
 887         if self._mode == "a":
 888             # Move to the end of the archive,
 889             # before the first empty block.
 890             self.firstmember = None
 891             while True:
 892                 try:
 893                     tarinfo = self.next()
 894                 except ReadError:
 895                     self.fileobj.seek(0)
 896                     break
 897                 if tarinfo is None:
 898                     self.fileobj.seek(- BLOCKSIZE, 1)
 899                     break
 900
 901         if self._mode in "aw":
 902             self._loaded = True
 903
 904     #--------------------------------------------------------------------------
 905     # Below are the classmethods which act as alternate constructors to the
 906     # TarFile class. The open() method is the only one that is needed for
 907     # public use; it is the "super"-constructor and is able to select an
 908     # adequate "sub"-constructor for a particular compression using the mapping
 909     # from OPEN_METH.
 910     #
 911     # This concept allows one to subclass TarFile without losing the comfort of
 912     # the super-constructor. A sub-constructor is registered and made available
 913     # by adding it to the mapping in OPEN_METH.
 914
 915     @classmethod
 916     def open(cls, name=None, mode="r", fileobj=None, bufsize=20*512):
 917         """Open a tar archive for reading, writing or appending. Return
 918            an appropriate TarFile class.
 919
 920            mode:
 921            'r' or 'r:*' open for reading with transparent compression
 922            'r:'         open for reading exclusively uncompressed
 923            'r:gz'       open for reading with gzip compression
 924            'r:bz2'      open for reading with bzip2 compression
 925            'a' or 'a:'  open for appending
 926            'w' or 'w:'  open for writing without compression
 927            'w:gz'       open for writing with gzip compression
 928            'w:bz2'      open for writing with bzip2 compression
 929
 930            'r|*'        open a stream of tar blocks with transparent compression
 931            'r|'         open an uncompressed stream of tar blocks for reading
 932            'r|gz'       open a gzip compressed stream of tar blocks
 933            'r|bz2'      open a bzip2 compressed stream of tar blocks
 934            'w|'         open an uncompressed stream for writing
 935            'w|gz'       open a gzip compressed stream for writing
 936            'w|bz2'      open a bzip2 compressed stream for writing
 937         """
 938
 939         if not name and not fileobj:
 940             raise ValueError, "nothing to open"
 941
 942         if mode in ("r", "r:*"):
 943             # Find out which *open() is appropriate for opening the file.
 944             for comptype in cls.OPEN_METH:
 945                 func = getattr(cls, cls.OPEN_METH[comptype])
 946                 try:
 947                     return func(name, "r", fileobj)
 948                 except (ReadError, CompressionError):
 949                     continue
 950             raise ReadError, "file could not be opened successfully"
 951
 952         elif ":" in mode:
 953             filemode, comptype = mode.split(":", 1)
 954             filemode = filemode or "r"
 955             comptype = comptype or "tar"
 956
 957             # Select the *open() function according to
 958             # given compression.
 959             if comptype in cls.OPEN_METH:
 960                 func = getattr(cls, cls.OPEN_METH[comptype])
 961             else:
 962                 raise CompressionError, "unknown compression type %r" % comptype
 963             return func(name, filemode, fileobj)
 964
 965         elif "|" in mode:
 966             filemode, comptype = mode.split("|", 1)
 967             filemode = filemode or "r"
 968             comptype = comptype or "tar"
 969
 970             if filemode not in "rw":
 971                 raise ValueError, "mode must be 'r' or 'w'"
 972
 973             t = cls(name, filemode,
 974                     _Stream(name, filemode, comptype, fileobj, bufsize))
 975             t._extfileobj = False
 976             return t
 977
 978         elif mode in "aw":
 979             return cls.taropen(name, mode, fileobj)
 980
 981         raise ValueError, "undiscernible mode"
 982
 983     @classmethod
 984     def taropen(cls, name, mode="r", fileobj=None):
 985         """Open uncompressed tar archive name for reading or writing.
 986         """
 987         if len(mode) > 1 or mode not in "raw":
 988             raise ValueError, "mode must be 'r', 'a' or 'w'"
 989         return cls(name, mode, fileobj)
 990
 991     @classmethod
 992     def gzopen(cls, name, mode="r", fileobj=None, compresslevel=9):
 993         """Open gzip compressed tar archive name for reading or writing.
 994            Appending is not allowed.
 995         """
 996         if len(mode) > 1 or mode not in "rw":
 997             raise ValueError, "mode must be 'r' or 'w'"
 998
 999         try:
1000             import gzip
1001             gzip.GzipFile
1002         except (ImportError, AttributeError):
1003             raise CompressionError, "gzip module is not available"
1004
1005         pre, ext = os.path.splitext(name)
1006         pre = os.path.basename(pre)
1007         if ext == ".tgz":
1008             ext = ".tar"
1009         if ext == ".gz":
1010             ext = ""
1011         tarname = pre + ext
1012
1013         if fileobj is None:
1014             fileobj = file(name, mode + "b")
1015
1016         if mode != "r":
1017             name = tarname
1018
1019         try:
1020             t = cls.taropen(tarname, mode,
1021                 gzip.GzipFile(name, mode, compresslevel, fileobj)
1022             )
1023         except IOError:
1024             raise ReadError, "not a gzip file"
1025         t._extfileobj = False
1026         return t
1027
1028     @classmethod
1029     def bz2open(cls, name, mode="r", fileobj=None, compresslevel=9):
1030         """Open bzip2 compressed tar archive name for reading or writing.
1031            Appending is not allowed.
1032         """
1033         if len(mode) > 1 or mode not in "rw":
1034             raise ValueError, "mode must be 'r' or 'w'."
1035
1036         try:
1037             import bz2
1038         except ImportError:
1039             raise CompressionError, "bz2 module is not available"
1040
1041         pre, ext = os.path.splitext(name)
1042         pre = os.path.basename(pre)
1043         if ext == ".tbz2":
1044             ext = ".tar"
1045         if ext == ".bz2":
1046             ext = ""
1047         tarname = pre + ext
1048
1049         if fileobj is not None:
1050             raise ValueError, "no support for external file objects"
1051
1052         try:
1053             t = cls.taropen(tarname, mode, bz2.BZ2File(name, mode, compresslevel=compresslevel))
1054         except IOError:
1055             raise ReadError, "not a bzip2 file"
1056         t._extfileobj = False
1057         return t
1058
1059     # All *open() methods are registered here.
1060     OPEN_METH = {
1061         "tar": "taropen",   # uncompressed tar
1062         "gz":  "gzopen",    # gzip compressed tar
1063         "bz2": "bz2open"    # bzip2 compressed tar
1064     }
1065
1066     #--------------------------------------------------------------------------
1067     # The public methods which TarFile provides:
1068
1069     def close(self):
1070         """Close the TarFile. In write-mode, two finishing zero blocks are
1071            appended to the archive.
1072         """
1073         if self.closed:
1074             return
1075
1076         if self._mode in "aw":
1077             self.fileobj.write(NUL * (BLOCKSIZE * 2))
1078             self.offset += (BLOCKSIZE * 2)
1079             # fill up the end with zero-blocks
1080             # (like option -b20 for tar does)
1081             blocks, remainder = divmod(self.offset, RECORDSIZE)
1082             if remainder > 0:
1083                 self.fileobj.write(NUL * (RECORDSIZE - remainder))
1084
1085         if not self._extfileobj:
1086             self.fileobj.close()
1087         self.closed = True
1088
1089     def getmember(self, name):
1090         """Return a TarInfo object for member `name'. If `name' can not be
1091            found in the archive, KeyError is raised. If a member occurs more
1092            than once in the archive, its last occurence is assumed to be the
1093            most up-to-date version.
1094         """
1095         tarinfo = self._getmember(name)
1096         if tarinfo is None:
1097             raise KeyError, "filename %r not found" % name
1098         return tarinfo
1099
1100     def getmembers(self):
1101         """Return the members of the archive as a list of TarInfo objects. The
1102            list has the same order as the members in the archive.
1103         """
1104         self._check()
1105         if not self._loaded:    # if we want to obtain a list of
1106             self._load()        # all members, we first have to
1107                                 # scan the whole archive.
1108         return self.members
1109
1110     def getnames(self):
1111         """Return the members of the archive as a list of their names. It has
1112            the same order as the list returned by getmembers().
1113         """
1114         return [tarinfo.name for tarinfo in self.getmembers()]
1115
1116     def gettarinfo(self, name=None, arcname=None, fileobj=None):
1117         """Create a TarInfo object for either the file `name' or the file
1118            object `fileobj' (using os.fstat on its file descriptor). You can
1119            modify some of the TarInfo's attributes before you add it using
1120            addfile(). If given, `arcname' specifies an alternative name for the
1121            file in the archive.
1122         """
1123         self._check("aw")
1124
1125         # When fileobj is given, replace name by
1126         # fileobj's real name.
1127         if fileobj is not None:
1128             name = fileobj.name
1129
1130         # Building the name of the member in the archive.
1131         # Backward slashes are converted to forward slashes,
1132         # Absolute paths are turned to relative paths.
1133         if arcname is None:
1134             arcname = name
1135         arcname = normpath(arcname)
1136         drv, arcname = os.path.splitdrive(arcname)
1137         while arcname[0:1] == "/":
1138             arcname = arcname[1:]
1139
1140         # Now, fill the TarInfo object with
1141         # information specific for the file.
1142         tarinfo = TarInfo()
1143
1144         # Use os.stat or os.lstat, depending on platform
1145         # and if symlinks shall be resolved.
1146         if fileobj is None:
1147             if hasattr(os, "lstat") and not self.dereference:
1148                 statres = os.lstat(name)
1149             else:
1150                 statres = os.stat(name)
1151         else:
1152             statres = os.fstat(fileobj.fileno())
1153         linkname = ""
1154
1155         stmd = statres.st_mode
1156         if stat.S_ISREG(stmd):
1157             inode = (statres.st_ino, statres.st_dev)
1158             if not self.dereference and \
1159                     statres.st_nlink > 1 and inode in self.inodes:
1160                 # Is it a hardlink to an already
1161                 # archived file?
1162                 type = LNKTYPE
1163                 linkname = self.inodes[inode]
1164             else:
1165                 # The inode is added only if its valid.
1166                 # For win32 it is always 0.
1167                 type = REGTYPE
1168                 if inode[0]:
1169                     self.inodes[inode] = arcname
1170         elif stat.S_ISDIR(stmd):
1171             type = DIRTYPE
1172             if arcname[-1:] != "/":
1173                 arcname += "/"
1174         elif stat.S_ISFIFO(stmd):
1175             type = FIFOTYPE
1176         elif stat.S_ISLNK(stmd):
1177             type = SYMTYPE
1178             linkname = os.readlink(name)
1179         elif stat.S_ISCHR(stmd):
1180             type = CHRTYPE
1181         elif stat.S_ISBLK(stmd):
1182             type = BLKTYPE
1183         else:
1184             return None
1185
1186         # Fill the TarInfo object with all
1187         # information we can get.
1188         tarinfo.name = arcname
1189         tarinfo.mode = stmd
1190         tarinfo.uid = statres.st_uid
1191         tarinfo.gid = statres.st_gid
1192         if stat.S_ISREG(stmd):
1193             tarinfo.size = statres.st_size
1194         else:
1195             tarinfo.size = 0L
1196         tarinfo.mtime = statres.st_mtime
1197         tarinfo.type = type
1198         tarinfo.linkname = linkname
1199         if pwd:
1200             try:
1201                 tarinfo.uname = pwd.getpwuid(tarinfo.uid)[0]
1202             except KeyError:
1203                 pass
1204         if grp:
1205             try:
1206                 tarinfo.gname = grp.getgrgid(tarinfo.gid)[0]
1207             except KeyError:
1208                 pass
1209
1210         if type in (CHRTYPE, BLKTYPE):
1211             if hasattr(os, "major") and hasattr(os, "minor"):
1212                 tarinfo.devmajor = os.major(statres.st_rdev)
1213                 tarinfo.devminor = os.minor(statres.st_rdev)
1214         return tarinfo
1215
1216     def list(self, verbose=True):
1217         """Print a table of contents to sys.stdout. If `verbose' is False, only
1218            the names of the members are printed. If it is True, an `ls -l'-like
1219            output is produced.
1220         """
1221         self._check()
1222
1223         for tarinfo in self:
1224             if verbose:
1225                 print filemode(tarinfo.mode),
1226                 print "%s/%s" % (tarinfo.uname or tarinfo.uid,
1227                                  tarinfo.gname or tarinfo.gid),
1228                 if tarinfo.ischr() or tarinfo.isblk():
1229                     print "%10s" % ("%d,%d" \
1230                                     % (tarinfo.devmajor, tarinfo.devminor)),
1231                 else:
1232                     print "%10d" % tarinfo.size,
1233                 print "%d-%02d-%02d %02d:%02d:%02d" \
1234                       % time.localtime(tarinfo.mtime)[:6],
1235
1236             print tarinfo.name,
1237
1238             if verbose:
1239                 if tarinfo.issym():
1240                     print "->", tarinfo.linkname,
1241                 if tarinfo.islnk():
1242                     print "link to", tarinfo.linkname,
1243             print
1244
1245     def add(self, name, arcname=None, recursive=True):
1246         """Add the file `name' to the archive. `name' may be any type of file
1247            (directory, fifo, symbolic link, etc.). If given, `arcname'
1248            specifies an alternative name for the file in the archive.
1249            Directories are added recursively by default. This can be avoided by
1250            setting `recursive' to False.
1251         """
1252         self._check("aw")
1253
1254         if arcname is None:
1255             arcname = name
1256
1257         # Skip if somebody tries to archive the archive...
1258         if self.name is not None \
1259             and os.path.abspath(name) == os.path.abspath(self.name):
1260             self._dbg(2, "tarfile: Skipped %r" % name)
1261             return
1262
1263         # Special case: The user wants to add the current
1264         # working directory.
1265         if name == ".":
1266             if recursive:
1267                 if arcname == ".":
1268                     arcname = ""
1269                 for f in os.listdir("."):
1270                     self.add(f, os.path.join(arcname, f))
1271             return
1272
1273         self._dbg(1, name)
1274
1275         # Create a TarInfo object from the file.
1276         tarinfo = self.gettarinfo(name, arcname)
1277
1278         if tarinfo is None:
1279             self._dbg(1, "tarfile: Unsupported type %r" % name)
1280             return
1281
1282         # Append the tar header and data to the archive.
1283         if tarinfo.isreg():
1284             f = file(name, "rb")
1285             self.addfile(tarinfo, f)
1286             f.close()
1287
1288         elif tarinfo.isdir():
1289             self.addfile(tarinfo)
1290             if recursive:
1291                 for f in os.listdir(name):
1292                     self.add(os.path.join(name, f), os.path.join(arcname, f))
1293
1294         else:
1295             self.addfile(tarinfo)
1296
1297     def addfile(self, tarinfo, fileobj=None):
1298         """Add the TarInfo object `tarinfo' to the archive. If `fileobj' is
1299            given, tarinfo.size bytes are read from it and added to the archive.
1300            You can create TarInfo objects using gettarinfo().
1301            On Windows platforms, `fileobj' should always be opened with mode
1302            'rb' to avoid irritation about the file size.
1303         """
1304         self._check("aw")
1305
1306         tarinfo.name = normpath(tarinfo.name)
1307         if tarinfo.isdir():
1308             # directories should end with '/'
1309             tarinfo.name += "/"
1310
1311         if tarinfo.linkname:
1312             tarinfo.linkname = normpath(tarinfo.linkname)
1313
1314         if tarinfo.size > MAXSIZE_MEMBER:
1315             if self.posix:
1316                 raise ValueError, "file is too large (>= 8 GB)"
1317             else:
1318                 self._dbg(2, "tarfile: Created GNU tar largefile header")
1319
1320
1321         if len(tarinfo.linkname) > LENGTH_LINK:
1322             if self.posix:
1323                 raise ValueError, "linkname is too long (>%d)" \
1324                                   % (LENGTH_LINK)
1325             else:
1326                 self._create_gnulong(tarinfo.linkname, GNUTYPE_LONGLINK)
1327                 tarinfo.linkname = tarinfo.linkname[:LENGTH_LINK -1]
1328                 self._dbg(2, "tarfile: Created GNU tar extension LONGLINK")
1329
1330         if len(tarinfo.name) > LENGTH_NAME:
1331             if self.posix:
1332                 prefix = tarinfo.name[:LENGTH_PREFIX + 1]
1333                 while prefix and prefix[-1] != "/":
1334                     prefix = prefix[:-1]
1335
1336                 name = tarinfo.name[len(prefix):]
1337                 prefix = prefix[:-1]
1338
1339                 if not prefix or len(name) > LENGTH_NAME:
1340                     raise ValueError, "name is too long (>%d)" \
1341                                       % (LENGTH_NAME)
1342
1343                 tarinfo.name   = name
1344                 tarinfo.prefix = prefix
1345             else:
1346                 self._create_gnulong(tarinfo.name, GNUTYPE_LONGNAME)
1347                 tarinfo.name = tarinfo.name[:LENGTH_NAME - 1]
1348                 self._dbg(2, "tarfile: Created GNU tar extension LONGNAME")
1349
1350         self.fileobj.write(tarinfo.tobuf())
1351         self.offset += BLOCKSIZE
1352
1353         # If there's data to follow, append it.
1354         if fileobj is not None:
1355             copyfileobj(fileobj, self.fileobj, tarinfo.size)
1356             blocks, remainder = divmod(tarinfo.size, BLOCKSIZE)
1357             if remainder > 0:
1358                 self.fileobj.write(NUL * (BLOCKSIZE - remainder))
1359                 blocks += 1
1360             self.offset += blocks * BLOCKSIZE
1361
1362         self.members.append(tarinfo)
1363
1364     def extractall(self, path=".", members=None):
1365         """Extract all members from the archive to the current working
1366            directory and set owner, modification time and permissions on
1367            directories afterwards. `path' specifies a different directory
1368            to extract to. `members' is optional and must be a subset of the
1369            list returned by getmembers().
1370         """
1371         directories = []
1372
1373         if members is None:
1374             members = self
1375
1376         for tarinfo in members:
1377             if tarinfo.isdir():
1378                 # Extract directory with a safe mode, so that
1379                 # all files below can be extracted as well.
1380                 try:
1381                     os.makedirs(os.path.join(path, tarinfo.name), 0777)
1382                 except EnvironmentError:
1383                     pass
1384                 directories.append(tarinfo)
1385             else:
1386                 self.extract(tarinfo, path)
1387
1388         # Reverse sort directories.
1389         directories.sort(lambda a, b: cmp(a.name, b.name))
1390         directories.reverse()
1391
1392         # Set correct owner, mtime and filemode on directories.
1393         for tarinfo in directories:
1394             path = os.path.join(path, tarinfo.name)
1395             try:
1396                 self.chown(tarinfo, path)
1397                 self.utime(tarinfo, path)
1398                 self.chmod(tarinfo, path)
1399             except ExtractError, e:
1400                 if self.errorlevel > 1:
1401                     raise
1402                 else:
1403                     self._dbg(1, "tarfile: %s" % e)
1404
1405     def extract(self, member, path=""):
1406         """Extract a member from the archive to the current working directory,
1407            using its full name. Its file information is extracted as accurately
1408            as possible. `member' may be a filename or a TarInfo object. You can
1409            specify a different directory using `path'.
1410         """
1411         self._check("r")
1412
1413         if isinstance(member, TarInfo):
1414             tarinfo = member
1415         else:
1416             tarinfo = self.getmember(member)
1417
1418         # Prepare the link target for makelink().
1419         if tarinfo.islnk():
1420             tarinfo._link_target = os.path.join(path, tarinfo.linkname)
1421
1422         try:
1423             self._extract_member(tarinfo, os.path.join(path, tarinfo.name))
1424         except EnvironmentError, e:
1425             if self.errorlevel > 0:
1426                 raise
1427             else:
1428                 if e.filename is None:
1429                     self._dbg(1, "tarfile: %s" % e.strerror)
1430                 else:
1431                     self._dbg(1, "tarfile: %s %r" % (e.strerror, e.filename))
1432         except ExtractError, e:
1433             if self.errorlevel > 1:
1434                 raise
1435             else:
1436                 self._dbg(1, "tarfile: %s" % e)
1437
1438     def extractfile(self, member):
1439         """Extract a member from the archive as a file object. `member' may be
1440            a filename or a TarInfo object. If `member' is a regular file, a
1441            file-like object is returned. If `member' is a link, a file-like
1442            object is constructed from the link's target. If `member' is none of
1443            the above, None is returned.
1444            The file-like object is read-only and provides the following
1445            methods: read(), readline(), readlines(), seek() and tell()
1446         """
1447         self._check("r")
1448
1449         if isinstance(member, TarInfo):
1450             tarinfo = member
1451         else:
1452             tarinfo = self.getmember(member)
1453
1454         if tarinfo.isreg():
1455             return self.fileobject(self, tarinfo)
1456
1457         elif tarinfo.type not in SUPPORTED_TYPES:
1458             # If a member's type is unknown, it is treated as a
1459             # regular file.
1460             return self.fileobject(self, tarinfo)
1461
1462         elif tarinfo.islnk() or tarinfo.issym():
1463             if isinstance(self.fileobj, _Stream):
1464                 # A small but ugly workaround for the case that someone tries
1465                 # to extract a (sym)link as a file-object from a non-seekable
1466                 # stream of tar blocks.
1467                 raise StreamError, "cannot extract (sym)link as file object"
1468             else:
1469                 # A (sym)link's file object is its target's file object.
1470                 return self.extractfile(self._getmember(tarinfo.linkname,
1471                                                         tarinfo))
1472         else:
1473             # If there's no data associated with the member (directory, chrdev,
1474             # blkdev, etc.), return None instead of a file object.
1475             return None
1476
1477     def _extract_member(self, tarinfo, targetpath):
1478         """Extract the TarInfo object tarinfo to a physical
1479            file called targetpath.
1480         """
1481         # Fetch the TarInfo object for the given name
1482         # and build the destination pathname, replacing
1483         # forward slashes to platform specific separators.
1484         if targetpath[-1:] == "/":
1485             targetpath = targetpath[:-1]
1486         targetpath = os.path.normpath(targetpath)
1487
1488         # Create all upper directories.
1489         upperdirs = os.path.dirname(targetpath)
1490         if upperdirs and not os.path.exists(upperdirs):
1491             ti = TarInfo()
1492             ti.name  = upperdirs
1493             ti.type  = DIRTYPE
1494             ti.mode  = 0777
1495             ti.mtime = tarinfo.mtime
1496             ti.uid   = tarinfo.uid
1497             ti.gid   = tarinfo.gid
1498             ti.uname = tarinfo.uname
1499             ti.gname = tarinfo.gname
1500             try:
1501                 self._extract_member(ti, ti.name)
1502             except:
1503                 pass
1504
1505         if tarinfo.islnk() or tarinfo.issym():
1506             self._dbg(1, "%s -> %s" % (tarinfo.name, tarinfo.linkname))
1507         else:
1508             self._dbg(1, tarinfo.name)
1509
1510         if tarinfo.isreg():
1511             self.makefile(tarinfo, targetpath)
1512         elif tarinfo.isdir():
1513             self.makedir(tarinfo, targetpath)
1514         elif tarinfo.isfifo():
1515             self.makefifo(tarinfo, targetpath)
1516         elif tarinfo.ischr() or tarinfo.isblk():
1517             self.makedev(tarinfo, targetpath)
1518         elif tarinfo.islnk() or tarinfo.issym():
1519             self.makelink(tarinfo, targetpath)
1520         elif tarinfo.type not in SUPPORTED_TYPES:
1521             self.makeunknown(tarinfo, targetpath)
1522         else:
1523             self.makefile(tarinfo, targetpath)
1524
1525         self.chown(tarinfo, targetpath)
1526         if not tarinfo.issym():
1527             self.chmod(tarinfo, targetpath)
1528             self.utime(tarinfo, targetpath)
1529
1530     #--------------------------------------------------------------------------
1531     # Below are the different file methods. They are called via
1532     # _extract_member() when extract() is called. They can be replaced in a
1533     # subclass to implement other functionality.
1534
1535     def makedir(self, tarinfo, targetpath):
1536         """Make a directory called targetpath.
1537         """
1538         try:
1539             os.mkdir(targetpath)
1540         except EnvironmentError, e:
1541             if e.errno != errno.EEXIST:
1542                 raise
1543
1544     def makefile(self, tarinfo, targetpath):
1545         """Make a file called targetpath.
1546         """
1547         source = self.extractfile(tarinfo)
1548         target = file(targetpath, "wb")
1549         copyfileobj(source, target)
1550         source.close()
1551         target.close()
1552
1553     def makeunknown(self, tarinfo, targetpath):
1554         """Make a file from a TarInfo object with an unknown type
1555            at targetpath.
1556         """
1557         self.makefile(tarinfo, targetpath)
1558         self._dbg(1, "tarfile: Unknown file type %r, " \
1559                      "extracted as regular file." % tarinfo.type)
1560
1561     def makefifo(self, tarinfo, targetpath):
1562         """Make a fifo called targetpath.
1563         """
1564         if hasattr(os, "mkfifo"):
1565             os.mkfifo(targetpath)
1566         else:
1567             raise ExtractError, "fifo not supported by system"
1568
1569     def makedev(self, tarinfo, targetpath):
1570         """Make a character or block device called targetpath.
1571         """
1572         if not hasattr(os, "mknod") or not hasattr(os, "makedev"):
1573             raise ExtractError, "special devices not supported by system"
1574
1575         mode = tarinfo.mode
1576         if tarinfo.isblk():
1577             mode |= stat.S_IFBLK
1578         else:
1579             mode |= stat.S_IFCHR
1580
1581         os.mknod(targetpath, mode,
1582                  os.makedev(tarinfo.devmajor, tarinfo.devminor))
1583
1584     def makelink(self, tarinfo, targetpath):
1585         """Make a (symbolic) link called targetpath. If it cannot be created
1586           (platform limitation), we try to make a copy of the referenced file
1587           instead of a link.
1588         """
1589         linkpath = tarinfo.linkname
1590         try:
1591             if tarinfo.issym():
1592                 os.symlink(linkpath, targetpath)
1593             else:
1594                 # See extract().
1595                 os.link(tarinfo._link_target, targetpath)
1596         except AttributeError:
1597             if tarinfo.issym():
1598                 linkpath = os.path.join(os.path.dirname(tarinfo.name),
1599                                         linkpath)
1600                 linkpath = normpath(linkpath)
1601
1602             try:
1603                 self._extract_member(self.getmember(linkpath), targetpath)
1604             except (EnvironmentError, KeyError), e:
1605                 linkpath = os.path.normpath(linkpath)
1606                 try:
1607                     shutil.copy2(linkpath, targetpath)
1608                 except EnvironmentError, e:
1609                     raise IOError, "link could not be created"
1610
1611     def chown(self, tarinfo, targetpath):
1612         """Set owner of targetpath according to tarinfo.
1613         """
1614         if pwd and hasattr(os, "geteuid") and os.geteuid() == 0:
1615             # We have to be root to do so.
1616             try:
1617                 g = grp.getgrnam(tarinfo.gname)[2]
1618             except KeyError:
1619                 try:
1620                     g = grp.getgrgid(tarinfo.gid)[2]
1621                 except KeyError:
1622                     g = os.getgid()
1623             try:
1624                 u = pwd.getpwnam(tarinfo.uname)[2]
1625             except KeyError:
1626                 try:
1627                     u = pwd.getpwuid(tarinfo.uid)[2]
1628                 except KeyError:
1629                     u = os.getuid()
1630             try:
1631                 if tarinfo.issym() and hasattr(os, "lchown"):
1632                     os.lchown(targetpath, u, g)
1633                 else:
1634                     if sys.platform != "os2emx":
1635                         os.chown(targetpath, u, g)
1636             except EnvironmentError, e:
1637                 raise ExtractError, "could not change owner"
1638
1639     def chmod(self, tarinfo, targetpath):
1640         """Set file permissions of targetpath according to tarinfo.
1641         """
1642         if hasattr(os, 'chmod'):
1643             try:
1644                 os.chmod(targetpath, tarinfo.mode)
1645             except EnvironmentError, e:
1646                 raise ExtractError, "could not change mode"
1647
1648     def utime(self, tarinfo, targetpath):
1649         """Set modification time of targetpath according to tarinfo.
1650         """
1651         if not hasattr(os, 'utime'):
1652             return
1653         if sys.platform == "win32" and tarinfo.isdir():
1654             # According to msdn.microsoft.com, it is an error (EACCES)
1655             # to use utime() on directories.
1656             return
1657         try:
1658             os.utime(targetpath, (tarinfo.mtime, tarinfo.mtime))
1659         except EnvironmentError, e:
1660             raise ExtractError, "could not change modification time"
1661
1662     #--------------------------------------------------------------------------
1663
1664     def next(self):
1665         """Return the next member of the archive as a TarInfo object, when
1666            TarFile is opened for reading. Return None if there is no more
1667            available.
1668         """
1669         self._check("ra")
1670         if self.firstmember is not None:
1671             m = self.firstmember
1672             self.firstmember = None
1673             return m
1674
1675         # Read the next block.
1676         self.fileobj.seek(self.offset)
1677         while True:
1678             buf = self.fileobj.read(BLOCKSIZE)
1679             if not buf:
1680                 return None
1681             try:
1682                 tarinfo = TarInfo.frombuf(buf)
1683             except ValueError:
1684                 if self.ignore_zeros:
1685                     if buf.count(NUL) == BLOCKSIZE:
1686                         adj = "empty"
1687                     else:
1688                         adj = "invalid"
1689                     self._dbg(2, "0x%X: %s block" % (self.offset, adj))
1690                     self.offset += BLOCKSIZE
1691                     continue
1692                 else:
1693                     # Block is empty or unreadable.
1694                     if self.offset == 0:
1695                         # If the first block is invalid. That does not
1696                         # look like a tar archive we can handle.
1697                         raise ReadError,"empty, unreadable or compressed file"
1698                     return None
1699             break
1700
1701         # We shouldn't rely on this checksum, because some tar programs
1702         # calculate it differently and it is merely validating the
1703         # header block. We could just as well skip this part, which would
1704         # have a slight effect on performance...
1705         if tarinfo.chksum != calc_chksum(buf):
1706             self._dbg(1, "tarfile: Bad Checksum %r" % tarinfo.name)
1707
1708         # Set the TarInfo object's offset to the current position of the
1709         # TarFile and set self.offset to the position where the data blocks
1710         # should begin.
1711         tarinfo.offset = self.offset
1712         self.offset += BLOCKSIZE
1713
1714         # Check if the TarInfo object has a typeflag for which a callback
1715         # method is registered in the TYPE_METH. If so, then call it.
1716         if tarinfo.type in self.TYPE_METH:
1717             return self.TYPE_METH[tarinfo.type](self, tarinfo)
1718
1719         tarinfo.offset_data = self.offset
1720         if tarinfo.isreg() or tarinfo.type not in SUPPORTED_TYPES:
1721             # Skip the following data blocks.
1722             self.offset += self._block(tarinfo.size)
1723
1724         self.members.append(tarinfo)
1725         return tarinfo
1726
1727     #--------------------------------------------------------------------------
1728     # Below are some methods which are called for special typeflags in the
1729     # next() method, e.g. for unwrapping GNU longname/longlink blocks. They
1730     # are registered in TYPE_METH below. You can register your own methods
1731     # with this mapping.
1732     # A registered method is called with a TarInfo object as only argument.
1733     #
1734     # During its execution the method MUST perform the following tasks:
1735     # 1. set tarinfo.offset_data to the position where the data blocks begin,
1736     #    if there is data to follow.
1737     # 2. set self.offset to the position where the next member's header will
1738     #    begin.
1739     # 3. append the tarinfo object to self.members, if it is supposed to appear
1740     #    as a member of the TarFile object.
1741     # 4. return tarinfo or another valid TarInfo object.
1742
1743     def proc_gnulong(self, tarinfo):
1744         """Evaluate the blocks that hold a GNU longname
1745            or longlink member.
1746         """
1747         buf = ""
1748         count = tarinfo.size
1749         while count > 0:
1750             block = self.fileobj.read(BLOCKSIZE)
1751             buf += block
1752             self.offset += BLOCKSIZE
1753             count -= BLOCKSIZE
1754
1755         # Fetch the next header
1756         next = self.next()
1757
1758         next.offset = tarinfo.offset
1759         if tarinfo.type == GNUTYPE_LONGNAME:
1760             next.name = nts(buf)
1761         elif tarinfo.type == GNUTYPE_LONGLINK:
1762             next.linkname = nts(buf)
1763
1764         return next
1765
1766     def proc_sparse(self, tarinfo):
1767         """Analyze a GNU sparse header plus extra headers.
1768         """
1769         buf = tarinfo.tobuf()
1770         sp = _ringbuffer()
1771         pos = 386
1772         lastpos = 0L
1773         realpos = 0L
1774         # There are 4 possible sparse structs in the
1775         # first header.
1776         for i in xrange(4):
1777             try:
1778                 offset = int(buf[pos:pos + 12], 8)
1779                 numbytes = int(buf[pos + 12:pos + 24], 8)
1780             except ValueError:
1781                 break
1782             if offset > lastpos:
1783                 sp.append(_hole(lastpos, offset - lastpos))
1784             sp.append(_data(offset, numbytes, realpos))
1785             realpos += numbytes
1786             lastpos = offset + numbytes
1787             pos += 24
1788
1789         isextended = ord(buf[482])
1790         origsize = int(buf[483:495], 8)
1791
1792         # If the isextended flag is given,
1793         # there are extra headers to process.
1794         while isextended == 1:
1795             buf = self.fileobj.read(BLOCKSIZE)
1796             self.offset += BLOCKSIZE
1797             pos = 0
1798             for i in xrange(21):
1799                 try:
1800                     offset = int(buf[pos:pos + 12], 8)
1801                     numbytes = int(buf[pos + 12:pos + 24], 8)
1802                 except ValueError:
1803                     break
1804                 if offset > lastpos:
1805                     sp.append(_hole(lastpos, offset - lastpos))
1806                 sp.append(_data(offset, numbytes, realpos))
1807                 realpos += numbytes
1808                 lastpos = offset + numbytes
1809                 pos += 24
1810             isextended = ord(buf[504])
1811
1812         if lastpos < origsize:
1813             sp.append(_hole(lastpos, origsize - lastpos))
1814
1815         tarinfo.sparse = sp
1816
1817         tarinfo.offset_data = self.offset
1818         self.offset += self._block(tarinfo.size)
1819         tarinfo.size = origsize
1820
1821         self.members.append(tarinfo)
1822         return tarinfo
1823
1824     # The type mapping for the next() method. The keys are single character
1825     # strings, the typeflag. The values are methods which are called when
1826     # next() encounters such a typeflag.
1827     TYPE_METH = {
1828         GNUTYPE_LONGNAME: proc_gnulong,
1829         GNUTYPE_LONGLINK: proc_gnulong,
1830         GNUTYPE_SPARSE:   proc_sparse
1831     }
1832
1833     #--------------------------------------------------------------------------
1834     # Little helper methods:
1835
1836     def _block(self, count):
1837         """Round up a byte count by BLOCKSIZE and return it,
1838            e.g. _block(834) => 1024.
1839         """
1840         blocks, remainder = divmod(count, BLOCKSIZE)
1841         if remainder:
1842             blocks += 1
1843         return blocks * BLOCKSIZE
1844
1845     def _getmember(self, name, tarinfo=None):
1846         """Find an archive member by name from bottom to top.
1847            If tarinfo is given, it is used as the starting point.
1848         """
1849         # Ensure that all members have been loaded.
1850         members = self.getmembers()
1851
1852         if tarinfo is None:
1853             end = len(members)
1854         else:
1855             end = members.index(tarinfo)
1856
1857         for i in xrange(end - 1, -1, -1):
1858             if name == members[i].name:
1859                 return members[i]
1860
1861     def _load(self):
1862         """Read through the entire archive file and look for readable
1863            members.
1864         """
1865         while True:
1866             tarinfo = self.next()
1867             if tarinfo is None:
1868                 break
1869         self._loaded = True
1870
1871     def _check(self, mode=None):
1872         """Check if TarFile is still open, and if the operation's mode
1873            corresponds to TarFile's mode.
1874         """
1875         if self.closed:
1876             raise IOError, "%s is closed" % self.__class__.__name__
1877         if mode is not None and self._mode not in mode:
1878             raise IOError, "bad operation for mode %r" % self._mode
1879
1880     def __iter__(self):
1881         """Provide an iterator object.
1882         """
1883         if self._loaded:
1884             return iter(self.members)
1885         else:
1886             return TarIter(self)
1887
1888     def _create_gnulong(self, name, type):
1889         """Write a GNU longname/longlink member to the TarFile.
1890            It consists of an extended tar header, with the length
1891            of the longname as size, followed by data blocks,
1892            which contain the longname as a null terminated string.
1893         """
1894         name += NUL
1895
1896         tarinfo = TarInfo()
1897         tarinfo.name = "././@LongLink"
1898         tarinfo.type = type
1899         tarinfo.mode = 0
1900         tarinfo.size = len(name)
1901
1902         # write extended header
1903         self.fileobj.write(tarinfo.tobuf())
1904         self.offset += BLOCKSIZE
1905         # write name blocks
1906         self.fileobj.write(name)
1907         blocks, remainder = divmod(tarinfo.size, BLOCKSIZE)
1908         if remainder > 0:
1909             self.fileobj.write(NUL * (BLOCKSIZE - remainder))
1910             blocks += 1
1911         self.offset += blocks * BLOCKSIZE
1912
1913     def _dbg(self, level, msg):
1914         """Write debugging output to sys.stderr.
1915         """
1916         if level <= self.debug:
1917             print >> sys.stderr, msg
1918 # class TarFile
1919
1920 class TarIter:
1921     """Iterator Class.
1922
1923        for tarinfo in TarFile(...):
1924            suite...
1925     """
1926
1927     def __init__(self, tarfile):
1928         """Construct a TarIter object.
1929         """
1930         self.tarfile = tarfile
1931         self.index = 0
1932     def __iter__(self):
1933         """Return iterator object.
1934         """
1935         return self
1936     def next(self):
1937         """Return the next item using TarFile's next() method.
1938            When all members have been read, set TarFile as _loaded.
1939         """
1940         # Fix for SF #1100429: Under rare circumstances it can
1941         # happen that getmembers() is called during iteration,
1942         # which will cause TarIter to stop prematurely.
1943         if not self.tarfile._loaded:
1944             tarinfo = self.tarfile.next()
1945             if not tarinfo:
1946                 self.tarfile._loaded = True
1947                 raise StopIteration
1948         else:
1949             try:
1950                 tarinfo = self.tarfile.members[self.index]
1951             except IndexError:
1952                 raise StopIteration
1953         self.index += 1
1954         return tarinfo
1955
1956 # Helper classes for sparse file support
1957 class _section:
1958     """Base class for _data and _hole.
1959     """
1960     def __init__(self, offset, size):
1961         self.offset = offset
1962         self.size = size
1963     def __contains__(self, offset):
1964         return self.offset <= offset < self.offset + self.size
1965
1966 class _data(_section):
1967     """Represent a data section in a sparse file.
1968     """
1969     def __init__(self, offset, size, realpos):
1970         _section.__init__(self, offset, size)
1971         self.realpos = realpos
1972
1973 class _hole(_section):
1974     """Represent a hole section in a sparse file.
1975     """
1976     pass
1977
1978 class _ringbuffer(list):
1979     """Ringbuffer class which increases performance
1980        over a regular list.
1981     """
1982     def __init__(self):
1983         self.idx = 0
1984     def find(self, offset):
1985         idx = self.idx
1986         while True:
1987             item = self[idx]
1988             if offset in item:
1989                 break
1990             idx += 1
1991             if idx == len(self):
1992                 idx = 0
1993             if idx == self.idx:
1994                 # End of File
1995                 return None
1996         self.idx = idx
1997         return item
1998
1999 #---------------------------------------------
2000 # zipfile compatible TarFile class
2001 #---------------------------------------------
2002 TAR_PLAIN = 0           # zipfile.ZIP_STORED
2003 TAR_GZIPPED = 8         # zipfile.ZIP_DEFLATED
2004 class TarFileCompat:
2005     """TarFile class compatible with standard module zipfile's
2006        ZipFile class.
2007     """
2008     def __init__(self, file, mode="r", compression=TAR_PLAIN):
2009         if compression == TAR_PLAIN:
2010             self.tarfile = TarFile.taropen(file, mode)
2011         elif compression == TAR_GZIPPED:
2012             self.tarfile = TarFile.gzopen(file, mode)
2013         else:
2014             raise ValueError, "unknown compression constant"
2015         if mode[0:1] == "r":
2016             members = self.tarfile.getmembers()
2017             for m in members:
2018                 m.filename = m.name
2019                 m.file_size = m.size
2020                 m.date_time = time.gmtime(m.mtime)[:6]
2021     def namelist(self):
2022         return map(lambda m: m.name, self.infolist())
2023     def infolist(self):
2024         return filter(lambda m: m.type in REGULAR_TYPES,
2025                       self.tarfile.getmembers())
2026     def printdir(self):
2027         self.tarfile.list()
2028     def testzip(self):
2029         return
2030     def getinfo(self, name):
2031         return self.tarfile.getmember(name)
2032     def read(self, name):
2033         return self.tarfile.extractfile(self.tarfile.getmember(name)).read()
2034     def write(self, filename, arcname=None, compress_type=None):
2035         self.tarfile.add(filename, arcname)
2036     def writestr(self, zinfo, bytes):
2037         try:
2038             from cStringIO import StringIO
2039         except ImportError:
2040             from StringIO import StringIO
2041         import calendar
2042         zinfo.name = zinfo.filename
2043         zinfo.size = zinfo.file_size
2044         zinfo.mtime = calendar.timegm(zinfo.date_time)
2045         self.tarfile.addfile(zinfo, StringIO(bytes))
2046     def close(self):
2047         self.tarfile.close()
2048 #class TarFileCompat
2049
2050 #--------------------
2051 # exported functions
2052 #--------------------
2053 def is_tarfile(name):
2054     """Return True if name points to a tar archive that we
2055        are able to handle, else return False.
2056     """
2057     try:
2058         t = open(name)
2059         t.close()
2060         return True
2061     except TarError:
2062         return False
2063
2064 open = TarFile.open