1 """Functions that read and write gzipped files.
3 The user of the file doesn't have to worry about the compression,
4 but random access is not allowed."""
6 # based on Andrew Kuchling's minigzip.py distributed with the zlib module
8 import struct
, sys
, time
, os
13 __all__
= ["GzipFile","open"]
15 FTEXT
, FHCRC
, FEXTRA
, FNAME
, FCOMMENT
= 1, 2, 4, 8, 16
19 def write32u(output
, value
):
20 # The L format writes the bit pattern correctly whether signed
22 output
.write(struct
.pack("<L", value
))
25 return struct
.unpack("<I", input.read(4))[0]
27 def open(filename
, mode
="rb", compresslevel
=9):
28 """Shorthand for GzipFile(filename, mode, compresslevel).
30 The filename argument is required; mode defaults to 'rb'
31 and compresslevel defaults to 9.
34 return GzipFile(filename
, mode
, compresslevel
)
36 class GzipFile(io
.BufferedIOBase
):
37 """The GzipFile class simulates most of the methods of a file object with
38 the exception of the readinto() and truncate() methods.
43 max_read_chunk
= 10 * 1024 * 1024 # 10Mb
45 def __init__(self
, filename
=None, mode
=None,
46 compresslevel
=9, fileobj
=None, mtime
=None):
47 """Constructor for the GzipFile class.
49 At least one of fileobj and filename must be given a
52 The new class instance is based on fileobj, which can be a regular
53 file, a StringIO object, or any other object which simulates a file.
54 It defaults to None, in which case filename is opened to provide
57 When fileobj is not None, the filename argument is only used to be
58 included in the gzip file header, which may includes the original
59 filename of the uncompressed file. It defaults to the filename of
60 fileobj, if discernible; otherwise, it defaults to the empty string,
61 and in this case the original filename is not included in the header.
63 The mode argument can be any of 'r', 'rb', 'a', 'ab', 'w', or 'wb',
64 depending on whether the file will be read or written. The default
65 is the mode of fileobj if discernible; otherwise, the default is 'rb'.
66 Be aware that only the 'rb', 'ab', and 'wb' values should be used
67 for cross-platform portability.
69 The compresslevel argument is an integer from 1 to 9 controlling the
70 level of compression; 1 is fastest and produces the least compression,
71 and 9 is slowest and produces the most compression. The default is 9.
73 The mtime argument is an optional numeric timestamp to be written
74 to the stream when compressing. All gzip compressed streams
75 are required to contain a timestamp. If omitted or None, the
76 current time is used. This module ignores the timestamp when
77 decompressing; however, some programs, such as gunzip, make use
78 of it. The format of the timestamp is the same as that of the
79 return value of time.time() and of the st_mtime member of the
80 object returned by os.stat().
84 # guarantee the file is opened in binary mode on platforms
85 # that care about that sort of thing
86 if mode
and 'b' not in mode
:
89 fileobj
= self
.myfileobj
= __builtin__
.open(filename
, mode
or 'rb')
91 if hasattr(fileobj
, 'name'): filename
= fileobj
.name
94 if hasattr(fileobj
, 'mode'): mode
= fileobj
.mode
99 # Set flag indicating start of a new member
100 self
._new
_member
= True
101 # Buffer data read from gzip file. extrastart is offset in
102 # stream where buffer starts. extrasize is number of
103 # bytes remaining in buffer from current stream position.
108 # Starts small, scales exponentially
109 self
.min_readsize
= 100
111 elif mode
[0:1] == 'w' or mode
[0:1] == 'a':
113 self
._init
_write
(filename
)
114 self
.compress
= zlib
.compressobj(compresslevel
,
120 raise IOError, "Mode " + mode
+ " not supported"
122 self
.fileobj
= fileobj
126 if self
.mode
== WRITE
:
127 self
._write
_gzip
_header
()
132 warnings
.warn("use the name attribute", DeprecationWarning, 2)
133 if self
.mode
== WRITE
and self
.name
[-3:] != ".gz":
134 return self
.name
+ ".gz"
138 s
= repr(self
.fileobj
)
139 return '<gzip ' + s
[1:-1] + ' ' + hex(id(self
)) + '>'
141 def _init_write(self
, filename
):
143 self
.crc
= zlib
.crc32("") & 0xffffffffL
148 def _write_gzip_header(self
):
149 self
.fileobj
.write('\037\213') # magic header
150 self
.fileobj
.write('\010') # compression method
151 fname
= os
.path
.basename(self
.name
)
152 if fname
.endswith(".gz"):
157 self
.fileobj
.write(chr(flags
))
161 write32u(self
.fileobj
, long(mtime
))
162 self
.fileobj
.write('\002')
163 self
.fileobj
.write('\377')
165 self
.fileobj
.write(fname
+ '\000')
167 def _init_read(self
):
168 self
.crc
= zlib
.crc32("") & 0xffffffffL
171 def _read_gzip_header(self
):
172 magic
= self
.fileobj
.read(2)
173 if magic
!= '\037\213':
174 raise IOError, 'Not a gzipped file'
175 method
= ord( self
.fileobj
.read(1) )
177 raise IOError, 'Unknown compression method'
178 flag
= ord( self
.fileobj
.read(1) )
179 self
.mtime
= read32(self
.fileobj
)
180 # extraflag = self.fileobj.read(1)
181 # os = self.fileobj.read(1)
185 # Read & discard the extra field, if present
186 xlen
= ord(self
.fileobj
.read(1))
187 xlen
= xlen
+ 256*ord(self
.fileobj
.read(1))
188 self
.fileobj
.read(xlen
)
190 # Read and discard a null-terminated string containing the filename
192 s
= self
.fileobj
.read(1)
193 if not s
or s
=='\000':
196 # Read and discard a null-terminated string containing a comment
198 s
= self
.fileobj
.read(1)
199 if not s
or s
=='\000':
202 self
.fileobj
.read(2) # Read & discard the 16-bit header CRC
204 def write(self
,data
):
205 if self
.mode
!= WRITE
:
207 raise IOError(errno
.EBADF
, "write() on read-only GzipFile object")
209 if self
.fileobj
is None:
210 raise ValueError, "write() on closed GzipFile object"
212 # Convert data type if called by io.BufferedWriter.
213 if isinstance(data
, memoryview
):
214 data
= data
.tobytes()
217 self
.size
= self
.size
+ len(data
)
218 self
.crc
= zlib
.crc32(data
, self
.crc
) & 0xffffffffL
219 self
.fileobj
.write( self
.compress
.compress(data
) )
220 self
.offset
+= len(data
)
224 def read(self
, size
=-1):
225 if self
.mode
!= READ
:
227 raise IOError(errno
.EBADF
, "read() on write-only GzipFile object")
229 if self
.extrasize
<= 0 and self
.fileobj
is None:
233 if size
< 0: # get the whole thing
237 readsize
= min(self
.max_read_chunk
, readsize
* 2)
239 size
= self
.extrasize
240 else: # just get some more of it
242 while size
> self
.extrasize
:
244 readsize
= min(self
.max_read_chunk
, readsize
* 2)
246 if size
> self
.extrasize
:
247 size
= self
.extrasize
249 offset
= self
.offset
- self
.extrastart
250 chunk
= self
.extrabuf
[offset
: offset
+ size
]
251 self
.extrasize
= self
.extrasize
- size
256 def _unread(self
, buf
):
257 self
.extrasize
= len(buf
) + self
.extrasize
258 self
.offset
-= len(buf
)
260 def _read(self
, size
=1024):
261 if self
.fileobj
is None:
262 raise EOFError, "Reached EOF"
265 # If the _new_member flag is set, we have to
266 # jump to the next member, if there is one.
268 # First, check if we're at the end of the file;
269 # if so, it's time to stop; no more members to read.
270 pos
= self
.fileobj
.tell() # Save current position
271 self
.fileobj
.seek(0, 2) # Seek to end of file
272 if pos
== self
.fileobj
.tell():
273 raise EOFError, "Reached EOF"
275 self
.fileobj
.seek( pos
) # Return to original position
278 self
._read
_gzip
_header
()
279 self
.decompress
= zlib
.decompressobj(-zlib
.MAX_WBITS
)
280 self
._new
_member
= False
282 # Read a chunk of data from the file
283 buf
= self
.fileobj
.read(size
)
285 # If the EOF has been reached, flush the decompression object
286 # and mark this object as finished.
289 uncompress
= self
.decompress
.flush()
291 self
._add
_read
_data
( uncompress
)
292 raise EOFError, 'Reached EOF'
294 uncompress
= self
.decompress
.decompress(buf
)
295 self
._add
_read
_data
( uncompress
)
297 if self
.decompress
.unused_data
!= "":
298 # Ending case: we've come to the end of a member in the file,
299 # so seek back to the start of the unused data, finish up
300 # this member, and read a new gzip header.
301 # (The number of bytes to seek back is the length of the unused
302 # data, minus 8 because _read_eof() will rewind a further 8 bytes)
303 self
.fileobj
.seek( -len(self
.decompress
.unused_data
)+8, 1)
305 # Check the CRC and file size, and set the flag so we read
306 # a new member on the next call
308 self
._new
_member
= True
310 def _add_read_data(self
, data
):
311 self
.crc
= zlib
.crc32(data
, self
.crc
) & 0xffffffffL
312 offset
= self
.offset
- self
.extrastart
313 self
.extrabuf
= self
.extrabuf
[offset
:] + data
314 self
.extrasize
= self
.extrasize
+ len(data
)
315 self
.extrastart
= self
.offset
316 self
.size
= self
.size
+ len(data
)
319 # We've read to the end of the file, so we have to rewind in order
320 # to reread the 8 bytes containing the CRC and the file size.
321 # We check the that the computed CRC and size of the
322 # uncompressed data matches the stored values. Note that the size
323 # stored is the true file size mod 2**32.
324 self
.fileobj
.seek(-8, 1)
325 crc32
= read32(self
.fileobj
)
326 isize
= read32(self
.fileobj
) # may exceed 2GB
327 if crc32
!= self
.crc
:
328 raise IOError("CRC check failed %s != %s" % (hex(crc32
),
330 elif isize
!= (self
.size
& 0xffffffffL
):
331 raise IOError, "Incorrect length of data produced"
335 return self
.fileobj
is None
338 if self
.fileobj
is None:
340 if self
.mode
== WRITE
:
341 self
.fileobj
.write(self
.compress
.flush())
342 write32u(self
.fileobj
, self
.crc
)
343 # self.size may exceed 2GB, or even 4GB
344 write32u(self
.fileobj
, self
.size
& 0xffffffffL
)
346 elif self
.mode
== READ
:
349 self
.myfileobj
.close()
350 self
.myfileobj
= None
352 def flush(self
,zlib_mode
=zlib
.Z_SYNC_FLUSH
):
353 if self
.mode
== WRITE
:
354 # Ensure the compressor's buffer is flushed
355 self
.fileobj
.write(self
.compress
.flush(zlib_mode
))
359 """Invoke the underlying file object's fileno() method.
361 This will raise AttributeError if the underlying file object
362 doesn't support fileno().
364 return self
.fileobj
.fileno()
367 '''Return the uncompressed stream file position indicator to the
368 beginning of the file'''
369 if self
.mode
!= READ
:
370 raise IOError("Can't rewind in write mode")
372 self
._new
_member
= True
379 return self
.mode
== READ
382 return self
.mode
== WRITE
387 def seek(self
, offset
, whence
=0):
390 offset
= self
.offset
+ offset
392 raise ValueError('Seek from end not supported')
393 if self
.mode
== WRITE
:
394 if offset
< self
.offset
:
395 raise IOError('Negative seek in write mode')
396 count
= offset
- self
.offset
397 for i
in range(count
// 1024):
398 self
.write(1024 * '\0')
399 self
.write((count
% 1024) * '\0')
400 elif self
.mode
== READ
:
401 if offset
< self
.offset
:
402 # for negative seek, rewind and do positive seek
404 count
= offset
- self
.offset
405 for i
in range(count
// 1024):
407 self
.read(count
% 1024)
411 def readline(self
, size
=-1):
413 # Shortcut common case - newline found in buffer.
414 offset
= self
.offset
- self
.extrastart
415 i
= self
.extrabuf
.find('\n', offset
) + 1
417 self
.extrasize
-= i
- offset
418 self
.offset
+= i
- offset
419 return self
.extrabuf
[offset
: i
]
422 readsize
= self
.min_readsize
427 c
= self
.read(readsize
)
430 # We set i=size to break out of the loop under two
431 # conditions: 1) there's no newline, and the chunk is
432 # larger than size, or 2) there is a newline, but the
433 # resulting line would be longer than 'size'.
434 if (size
<= i
) or (i
== -1 and len(c
) > size
):
437 if i
>= 0 or c
== '':
438 bufs
.append(c
[:i
+ 1]) # Add portion of last chunk
439 self
._unread
(c
[i
+ 1:]) # Push back rest of chunk
442 # Append chunk to list, decrease 'size',
445 readsize
= min(size
, readsize
* 2)
446 if readsize
> self
.min_readsize
:
447 self
.min_readsize
= min(readsize
, self
.min_readsize
* 2, 512)
448 return ''.join(bufs
) # Return resulting line
452 # Act like gzip; with -d, act like gunzip.
453 # The input file is not deleted, however, nor are any other gzip
454 # options or features supported.
456 decompress
= args
and args
[0] == "-d"
464 f
= GzipFile(filename
="", mode
="rb", fileobj
=sys
.stdin
)
467 if arg
[-3:] != ".gz":
468 print "filename doesn't end in .gz:", repr(arg
)
471 g
= __builtin__
.open(arg
[:-3], "wb")
475 g
= GzipFile(filename
="", mode
="wb", fileobj
=sys
.stdout
)
477 f
= __builtin__
.open(arg
, "rb")
478 g
= open(arg
+ ".gz", "wb")
484 if g
is not sys
.stdout
:
486 if f
is not sys
.stdin
:
489 if __name__
== '__main__':