1 """Functions that read and write gzipped files.
3 The user of the file doesn't have to worry about the compression,
4 but random access is not allowed."""
6 # based on Andrew Kuchling's minigzip.py distributed with the zlib module
8 import struct
, sys
, time
, os
13 __all__
= ["GzipFile","open"]
15 FTEXT
, FHCRC
, FEXTRA
, FNAME
, FCOMMENT
= 1, 2, 4, 8, 16
19 def write32u(output
, value
):
20 # The L format writes the bit pattern correctly whether signed
22 output
.write(struct
.pack("<L", value
))
25 return struct
.unpack("<I", input.read(4))[0]
27 def open(filename
, mode
="rb", compresslevel
=9):
28 """Shorthand for GzipFile(filename, mode, compresslevel).
30 The filename argument is required; mode defaults to 'rb'
31 and compresslevel defaults to 9.
34 return GzipFile(filename
, mode
, compresslevel
)
36 class GzipFile(io
.BufferedIOBase
):
37 """The GzipFile class simulates most of the methods of a file object with
38 the exception of the readinto() and truncate() methods.
43 max_read_chunk
= 10 * 1024 * 1024 # 10Mb
45 def __init__(self
, filename
=None, mode
=None,
46 compresslevel
=9, fileobj
=None, mtime
=None):
47 """Constructor for the GzipFile class.
49 At least one of fileobj and filename must be given a
52 The new class instance is based on fileobj, which can be a regular
53 file, a StringIO object, or any other object which simulates a file.
54 It defaults to None, in which case filename is opened to provide
57 When fileobj is not None, the filename argument is only used to be
58 included in the gzip file header, which may includes the original
59 filename of the uncompressed file. It defaults to the filename of
60 fileobj, if discernible; otherwise, it defaults to the empty string,
61 and in this case the original filename is not included in the header.
63 The mode argument can be any of 'r', 'rb', 'a', 'ab', 'w', or 'wb',
64 depending on whether the file will be read or written. The default
65 is the mode of fileobj if discernible; otherwise, the default is 'rb'.
66 Be aware that only the 'rb', 'ab', and 'wb' values should be used
67 for cross-platform portability.
69 The compresslevel argument is an integer from 0 to 9 controlling the
70 level of compression; 1 is fastest and produces the least compression,
71 and 9 is slowest and produces the most compression. 0 is no compression
72 at all. The default is 9.
74 The mtime argument is an optional numeric timestamp to be written
75 to the stream when compressing. All gzip compressed streams
76 are required to contain a timestamp. If omitted or None, the
77 current time is used. This module ignores the timestamp when
78 decompressing; however, some programs, such as gunzip, make use
79 of it. The format of the timestamp is the same as that of the
80 return value of time.time() and of the st_mtime member of the
81 object returned by os.stat().
85 # Make sure we don't inadvertently enable universal newlines on the
86 # underlying file object - in read mode, this causes data corruption.
88 mode
= mode
.replace('U', '')
89 # guarantee the file is opened in binary mode on platforms
90 # that care about that sort of thing
91 if mode
and 'b' not in mode
:
94 fileobj
= self
.myfileobj
= __builtin__
.open(filename
, mode
or 'rb')
96 # Issue #13781: os.fdopen() creates a fileobj with a bogus name
97 # attribute. Avoid saving this in the gzip header's filename field.
98 if hasattr(fileobj
, 'name') and fileobj
.name
!= '<fdopen>':
99 filename
= fileobj
.name
103 if hasattr(fileobj
, 'mode'): mode
= fileobj
.mode
108 # Set flag indicating start of a new member
109 self
._new
_member
= True
110 # Buffer data read from gzip file. extrastart is offset in
111 # stream where buffer starts. extrasize is number of
112 # bytes remaining in buffer from current stream position.
117 # Starts small, scales exponentially
118 self
.min_readsize
= 100
120 elif mode
[0:1] == 'w' or mode
[0:1] == 'a':
122 self
._init
_write
(filename
)
123 self
.compress
= zlib
.compressobj(compresslevel
,
129 raise IOError, "Mode " + mode
+ " not supported"
131 self
.fileobj
= fileobj
135 if self
.mode
== WRITE
:
136 self
._write
_gzip
_header
()
141 warnings
.warn("use the name attribute", DeprecationWarning, 2)
142 if self
.mode
== WRITE
and self
.name
[-3:] != ".gz":
143 return self
.name
+ ".gz"
147 s
= repr(self
.fileobj
)
148 return '<gzip ' + s
[1:-1] + ' ' + hex(id(self
)) + '>'
150 def _check_closed(self
):
151 """Raises a ValueError if the underlying file object has been closed.
155 raise ValueError('I/O operation on closed file.')
157 def _init_write(self
, filename
):
159 self
.crc
= zlib
.crc32("") & 0xffffffffL
164 def _write_gzip_header(self
):
165 self
.fileobj
.write('\037\213') # magic header
166 self
.fileobj
.write('\010') # compression method
167 fname
= os
.path
.basename(self
.name
)
168 if fname
.endswith(".gz"):
173 self
.fileobj
.write(chr(flags
))
177 write32u(self
.fileobj
, long(mtime
))
178 self
.fileobj
.write('\002')
179 self
.fileobj
.write('\377')
181 self
.fileobj
.write(fname
+ '\000')
183 def _init_read(self
):
184 self
.crc
= zlib
.crc32("") & 0xffffffffL
187 def _read_gzip_header(self
):
188 magic
= self
.fileobj
.read(2)
189 if magic
!= '\037\213':
190 raise IOError, 'Not a gzipped file'
191 method
= ord( self
.fileobj
.read(1) )
193 raise IOError, 'Unknown compression method'
194 flag
= ord( self
.fileobj
.read(1) )
195 self
.mtime
= read32(self
.fileobj
)
196 # extraflag = self.fileobj.read(1)
197 # os = self.fileobj.read(1)
201 # Read & discard the extra field, if present
202 xlen
= ord(self
.fileobj
.read(1))
203 xlen
= xlen
+ 256*ord(self
.fileobj
.read(1))
204 self
.fileobj
.read(xlen
)
206 # Read and discard a null-terminated string containing the filename
208 s
= self
.fileobj
.read(1)
209 if not s
or s
=='\000':
212 # Read and discard a null-terminated string containing a comment
214 s
= self
.fileobj
.read(1)
215 if not s
or s
=='\000':
218 self
.fileobj
.read(2) # Read & discard the 16-bit header CRC
220 def write(self
,data
):
222 if self
.mode
!= WRITE
:
224 raise IOError(errno
.EBADF
, "write() on read-only GzipFile object")
226 if self
.fileobj
is None:
227 raise ValueError, "write() on closed GzipFile object"
229 # Convert data type if called by io.BufferedWriter.
230 if isinstance(data
, memoryview
):
231 data
= data
.tobytes()
234 self
.size
= self
.size
+ len(data
)
235 self
.crc
= zlib
.crc32(data
, self
.crc
) & 0xffffffffL
236 self
.fileobj
.write( self
.compress
.compress(data
) )
237 self
.offset
+= len(data
)
241 def read(self
, size
=-1):
243 if self
.mode
!= READ
:
245 raise IOError(errno
.EBADF
, "read() on write-only GzipFile object")
247 if self
.extrasize
<= 0 and self
.fileobj
is None:
251 if size
< 0: # get the whole thing
255 readsize
= min(self
.max_read_chunk
, readsize
* 2)
257 size
= self
.extrasize
258 else: # just get some more of it
260 while size
> self
.extrasize
:
262 readsize
= min(self
.max_read_chunk
, readsize
* 2)
264 if size
> self
.extrasize
:
265 size
= self
.extrasize
267 offset
= self
.offset
- self
.extrastart
268 chunk
= self
.extrabuf
[offset
: offset
+ size
]
269 self
.extrasize
= self
.extrasize
- size
274 def _unread(self
, buf
):
275 self
.extrasize
= len(buf
) + self
.extrasize
276 self
.offset
-= len(buf
)
278 def _read(self
, size
=1024):
279 if self
.fileobj
is None:
280 raise EOFError, "Reached EOF"
283 # If the _new_member flag is set, we have to
284 # jump to the next member, if there is one.
286 # First, check if we're at the end of the file;
287 # if so, it's time to stop; no more members to read.
288 pos
= self
.fileobj
.tell() # Save current position
289 self
.fileobj
.seek(0, 2) # Seek to end of file
290 if pos
== self
.fileobj
.tell():
291 raise EOFError, "Reached EOF"
293 self
.fileobj
.seek( pos
) # Return to original position
296 self
._read
_gzip
_header
()
297 self
.decompress
= zlib
.decompressobj(-zlib
.MAX_WBITS
)
298 self
._new
_member
= False
300 # Read a chunk of data from the file
301 buf
= self
.fileobj
.read(size
)
303 # If the EOF has been reached, flush the decompression object
304 # and mark this object as finished.
307 uncompress
= self
.decompress
.flush()
309 self
._add
_read
_data
( uncompress
)
310 raise EOFError, 'Reached EOF'
312 uncompress
= self
.decompress
.decompress(buf
)
313 self
._add
_read
_data
( uncompress
)
315 if self
.decompress
.unused_data
!= "":
316 # Ending case: we've come to the end of a member in the file,
317 # so seek back to the start of the unused data, finish up
318 # this member, and read a new gzip header.
319 # (The number of bytes to seek back is the length of the unused
320 # data, minus 8 because _read_eof() will rewind a further 8 bytes)
321 self
.fileobj
.seek( -len(self
.decompress
.unused_data
)+8, 1)
323 # Check the CRC and file size, and set the flag so we read
324 # a new member on the next call
326 self
._new
_member
= True
328 def _add_read_data(self
, data
):
329 self
.crc
= zlib
.crc32(data
, self
.crc
) & 0xffffffffL
330 offset
= self
.offset
- self
.extrastart
331 self
.extrabuf
= self
.extrabuf
[offset
:] + data
332 self
.extrasize
= self
.extrasize
+ len(data
)
333 self
.extrastart
= self
.offset
334 self
.size
= self
.size
+ len(data
)
337 # We've read to the end of the file, so we have to rewind in order
338 # to reread the 8 bytes containing the CRC and the file size.
339 # We check the that the computed CRC and size of the
340 # uncompressed data matches the stored values. Note that the size
341 # stored is the true file size mod 2**32.
342 self
.fileobj
.seek(-8, 1)
343 crc32
= read32(self
.fileobj
)
344 isize
= read32(self
.fileobj
) # may exceed 2GB
345 if crc32
!= self
.crc
:
346 raise IOError("CRC check failed %s != %s" % (hex(crc32
),
348 elif isize
!= (self
.size
& 0xffffffffL
):
349 raise IOError, "Incorrect length of data produced"
351 # Gzip files can be padded with zeroes and still have archives.
352 # Consume all zero bytes and set the file position to the first
353 # non-zero byte. See http://www.gzip.org/#faq8
356 c
= self
.fileobj
.read(1)
358 self
.fileobj
.seek(-1, 1)
362 return self
.fileobj
is None
365 if self
.fileobj
is None:
367 if self
.mode
== WRITE
:
368 self
.fileobj
.write(self
.compress
.flush())
369 write32u(self
.fileobj
, self
.crc
)
370 # self.size may exceed 2GB, or even 4GB
371 write32u(self
.fileobj
, self
.size
& 0xffffffffL
)
373 elif self
.mode
== READ
:
376 self
.myfileobj
.close()
377 self
.myfileobj
= None
379 def flush(self
,zlib_mode
=zlib
.Z_SYNC_FLUSH
):
381 if self
.mode
== WRITE
:
382 # Ensure the compressor's buffer is flushed
383 self
.fileobj
.write(self
.compress
.flush(zlib_mode
))
387 """Invoke the underlying file object's fileno() method.
389 This will raise AttributeError if the underlying file object
390 doesn't support fileno().
392 return self
.fileobj
.fileno()
395 '''Return the uncompressed stream file position indicator to the
396 beginning of the file'''
397 if self
.mode
!= READ
:
398 raise IOError("Can't rewind in write mode")
400 self
._new
_member
= True
407 return self
.mode
== READ
410 return self
.mode
== WRITE
415 def seek(self
, offset
, whence
=0):
418 offset
= self
.offset
+ offset
420 raise ValueError('Seek from end not supported')
421 if self
.mode
== WRITE
:
422 if offset
< self
.offset
:
423 raise IOError('Negative seek in write mode')
424 count
= offset
- self
.offset
425 for i
in xrange(count
// 1024):
426 self
.write(1024 * '\0')
427 self
.write((count
% 1024) * '\0')
428 elif self
.mode
== READ
:
429 if offset
< self
.offset
:
430 # for negative seek, rewind and do positive seek
432 count
= offset
- self
.offset
433 for i
in xrange(count
// 1024):
435 self
.read(count
% 1024)
439 def readline(self
, size
=-1):
441 # Shortcut common case - newline found in buffer.
442 offset
= self
.offset
- self
.extrastart
443 i
= self
.extrabuf
.find('\n', offset
) + 1
445 self
.extrasize
-= i
- offset
446 self
.offset
+= i
- offset
447 return self
.extrabuf
[offset
: i
]
450 readsize
= self
.min_readsize
455 c
= self
.read(readsize
)
458 # We set i=size to break out of the loop under two
459 # conditions: 1) there's no newline, and the chunk is
460 # larger than size, or 2) there is a newline, but the
461 # resulting line would be longer than 'size'.
462 if (size
<= i
) or (i
== -1 and len(c
) > size
):
465 if i
>= 0 or c
== '':
466 bufs
.append(c
[:i
+ 1]) # Add portion of last chunk
467 self
._unread
(c
[i
+ 1:]) # Push back rest of chunk
470 # Append chunk to list, decrease 'size',
473 readsize
= min(size
, readsize
* 2)
474 if readsize
> self
.min_readsize
:
475 self
.min_readsize
= min(readsize
, self
.min_readsize
* 2, 512)
476 return ''.join(bufs
) # Return resulting line
480 # Act like gzip; with -d, act like gunzip.
481 # The input file is not deleted, however, nor are any other gzip
482 # options or features supported.
484 decompress
= args
and args
[0] == "-d"
492 f
= GzipFile(filename
="", mode
="rb", fileobj
=sys
.stdin
)
495 if arg
[-3:] != ".gz":
496 print "filename doesn't end in .gz:", repr(arg
)
499 g
= __builtin__
.open(arg
[:-3], "wb")
503 g
= GzipFile(filename
="", mode
="wb", fileobj
=sys
.stdout
)
505 f
= __builtin__
.open(arg
, "rb")
506 g
= open(arg
+ ".gz", "wb")
512 if g
is not sys
.stdout
:
514 if f
is not sys
.stdin
:
517 if __name__
== '__main__':