1 // Copyright 2011 The Go Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style
3 // license that can be found in the LICENSE file.
5 // Tests that involve both reading and writing.
24 func TestOver65kFiles(t
*testing
.T
) {
25 if testing
.Short() && testenv
.Builder() == "" {
26 t
.Skip("skipping in short mode")
28 buf
:= new(bytes
.Buffer
)
30 const nFiles
= (1 << 16) + 42
31 for i
:= 0; i
< nFiles
; i
++ {
32 _
, err
:= w
.CreateHeader(&FileHeader
{
33 Name
: fmt
.Sprintf("%d.dat", i
),
34 Method
: Store
, // avoid Issue 6136 and Issue 6138
37 t
.Fatalf("creating file %d: %v", i
, err
)
40 if err
:= w
.Close(); err
!= nil {
41 t
.Fatalf("Writer.Close: %v", err
)
44 zr
, err
:= NewReader(strings
.NewReader(s
), int64(len(s
)))
46 t
.Fatalf("NewReader: %v", err
)
48 if got
:= len(zr
.File
); got
!= nFiles
{
49 t
.Fatalf("File contains %d files, want %d", got
, nFiles
)
51 for i
:= 0; i
< nFiles
; i
++ {
52 want
:= fmt
.Sprintf("%d.dat", i
)
53 if zr
.File
[i
].Name
!= want
{
54 t
.Fatalf("File(%d) = %q, want %q", i
, zr
.File
[i
].Name
, want
)
59 func TestModTime(t
*testing
.T
) {
60 var testTime
= time
.Date(2009, time
.November
, 10, 23, 45, 58, 0, time
.UTC
)
62 fh
.SetModTime(testTime
)
63 outTime
:= fh
.ModTime()
64 if !outTime
.Equal(testTime
) {
65 t
.Errorf("times don't match: got %s, want %s", outTime
, testTime
)
69 func testHeaderRoundTrip(fh
*FileHeader
, wantUncompressedSize
uint32, wantUncompressedSize64
uint64, t
*testing
.T
) {
71 fh2
, err
:= FileInfoHeader(fi
)
75 if got
, want
:= fh2
.Name
, fh
.Name
; got
!= want
{
76 t
.Errorf("Name: got %s, want %s\n", got
, want
)
78 if got
, want
:= fh2
.UncompressedSize
, wantUncompressedSize
; got
!= want
{
79 t
.Errorf("UncompressedSize: got %d, want %d\n", got
, want
)
81 if got
, want
:= fh2
.UncompressedSize64
, wantUncompressedSize64
; got
!= want
{
82 t
.Errorf("UncompressedSize64: got %d, want %d\n", got
, want
)
84 if got
, want
:= fh2
.ModifiedTime
, fh
.ModifiedTime
; got
!= want
{
85 t
.Errorf("ModifiedTime: got %d, want %d\n", got
, want
)
87 if got
, want
:= fh2
.ModifiedDate
, fh
.ModifiedDate
; got
!= want
{
88 t
.Errorf("ModifiedDate: got %d, want %d\n", got
, want
)
91 if sysfh
, ok
:= fi
.Sys().(*FileHeader
); !ok
&& sysfh
!= fh
{
92 t
.Errorf("Sys didn't return original *FileHeader")
96 func TestFileHeaderRoundTrip(t
*testing
.T
) {
99 UncompressedSize
: 987654321,
103 testHeaderRoundTrip(fh
, fh
.UncompressedSize
, uint64(fh
.UncompressedSize
), t
)
106 func TestFileHeaderRoundTrip64(t
*testing
.T
) {
109 UncompressedSize64
: 9876543210,
113 testHeaderRoundTrip(fh
, uint32max
, fh
.UncompressedSize64
, t
)
116 type repeatedByte
struct {
122 // rleBuffer is a run-length-encoded byte buffer.
123 // It's an io.Writer (like a bytes.Buffer) and also an io.ReaderAt,
124 // allowing random-access reads.
125 type rleBuffer
struct {
129 func (r
*rleBuffer
) Size() int64 {
133 last
:= &r
.buf
[len(r
.buf
)-1]
134 return last
.off
+ last
.n
137 func (r
*rleBuffer
) Write(p
[]byte) (n
int, err error
) {
140 rp
= &r
.buf
[len(r
.buf
)-1]
141 // Fast path, if p is entirely the same byte repeated.
142 if lastByte
:= rp
.b
; len(p
) > 0 && p
[0] == lastByte
{
144 for _
, b
:= range p
{
151 rp
.n
+= int64(len(p
))
157 for _
, b
:= range p
{
158 if rp
== nil || rp
.b
!= b
{
159 r
.buf
= append(r
.buf
, repeatedByte
{r
.Size(), b
, 1})
160 rp
= &r
.buf
[len(r
.buf
)-1]
168 func (r
*rleBuffer
) ReadAt(p
[]byte, off
int64) (n
int, err error
) {
172 skipParts
:= sort
.Search(len(r
.buf
), func(i
int) bool {
174 return part
.off
+part
.n
> off
176 parts
:= r
.buf
[skipParts
:]
178 skipBytes
:= off
- parts
[0].off
181 for i
:= skipBytes
; i
< part
.n
; i
++ {
193 err
= io
.ErrUnexpectedEOF
198 // Just testing the rleBuffer used in the Zip64 test above. Not used by the zip code.
199 func TestRLEBuffer(t
*testing
.T
) {
202 writes
:= []string{"abcdeee", "eeeeeee", "eeeefghaaiii"}
203 for _
, w
:= range writes
{
205 all
= append(all
, w
...)
207 if len(b
.buf
) != 10 {
208 t
.Fatalf("len(b.buf) = %d; want 10", len(b
.buf
))
211 for i
:= 0; i
< len(all
); i
++ {
212 for j
:= 0; j
< len(all
)-i
; j
++ {
213 buf
:= make([]byte, j
)
214 n
, err
:= b
.ReadAt(buf
, int64(i
))
215 if err
!= nil || n
!= len(buf
) {
216 t
.Errorf("ReadAt(%d, %d) = %d, %v; want %d, nil", i
, j
, n
, err
, len(buf
))
218 if !bytes
.Equal(buf
, all
[i
:i
+j
]) {
219 t
.Errorf("ReadAt(%d, %d) = %q; want %q", i
, j
, buf
, all
[i
:i
+j
])
225 // fakeHash32 is a dummy Hash32 that always returns 0.
226 type fakeHash32
struct {
230 func (fakeHash32
) Write(p
[]byte) (int, error
) { return len(p
), nil }
231 func (fakeHash32
) Sum32() uint32 { return 0 }
233 func TestZip64(t
*testing
.T
) {
235 t
.Skip("slow test; skipping")
238 const size
= 1 << 32 // before the "END\n" part
239 buf
:= testZip64(t
, size
)
240 testZip64DirectoryRecordLength(buf
, t
)
243 func TestZip64EdgeCase(t
*testing
.T
) {
245 t
.Skip("slow test; skipping")
248 // Test a zip file with uncompressed size 0xFFFFFFFF.
249 // That's the magic marker for a 64-bit file, so even though
250 // it fits in a 32-bit field we must use the 64-bit field.
251 // Go 1.5 and earlier got this wrong,
252 // writing an invalid zip file.
253 const size
= 1<<32 - 1 - int64(len("END\n")) // before the "END\n" part
254 buf
:= testZip64(t
, size
)
255 testZip64DirectoryRecordLength(buf
, t
)
258 // Tests that we generate a zip64 file if the directory at offset
259 // 0xFFFFFFFF, but not before.
260 func TestZip64DirectoryOffset(t
*testing
.T
) {
261 if testing
.Short() && race
.Enabled
{
262 t
.Skip("skipping in short mode")
265 const filename
= "huge.txt"
266 gen
:= func(wantOff
uint64) func(*Writer
) {
267 return func(w
*Writer
) {
268 w
.testHookCloseSizeOffset
= func(size
, off
uint64) {
270 t
.Errorf("central directory offset = %d (%x); want %d", off
, off
, wantOff
)
273 f
, err
:= w
.CreateHeader(&FileHeader
{
280 f
.(*fileWriter
).crc32
= fakeHash32
{}
281 size
:= wantOff
- fileHeaderLen
- uint64(len(filename
)) - dataDescriptorLen
282 if _
, err
:= io
.CopyN(f
, zeros
{}, int64(size
)); err
!= nil {
285 if err
:= w
.Close(); err
!= nil {
290 t
.Run("uint32max-2_NoZip64", func(t
*testing
.T
) {
292 if generatesZip64(t
, gen(0xfffffffe)) {
293 t
.Error("unexpected zip64")
296 t
.Run("uint32max-1_Zip64", func(t
*testing
.T
) {
298 if !generatesZip64(t
, gen(0xffffffff)) {
299 t
.Error("expected zip64")
304 // At 16k records, we need to generate a zip64 file.
305 func TestZip64ManyRecords(t
*testing
.T
) {
306 if testing
.Short() && race
.Enabled
{
307 t
.Skip("skipping in short mode")
310 gen
:= func(numRec
int) func(*Writer
) {
311 return func(w
*Writer
) {
312 for i
:= 0; i
< numRec
; i
++ {
313 _
, err
:= w
.CreateHeader(&FileHeader
{
321 if err
:= w
.Close(); err
!= nil {
326 // 16k-1 records shouldn't make a zip64:
327 t
.Run("uint16max-1_NoZip64", func(t
*testing
.T
) {
329 if generatesZip64(t
, gen(0xfffe)) {
330 t
.Error("unexpected zip64")
333 // 16k records should make a zip64:
334 t
.Run("uint16max_Zip64", func(t
*testing
.T
) {
336 if !generatesZip64(t
, gen(0xffff)) {
337 t
.Error("expected zip64")
342 // suffixSaver is an io.Writer & io.ReaderAt that remembers the last 0
343 // to 'keep' bytes of data written to it. Call Suffix to get the
345 type suffixSaver
struct {
352 func (ss
*suffixSaver
) Size() int64 { return ss
.size
}
354 var errDiscardedBytes
= errors
.New("ReadAt of discarded bytes")
356 func (ss
*suffixSaver
) ReadAt(p
[]byte, off
int64) (n
int, err error
) {
357 back
:= ss
.size
- off
358 if back
> int64(ss
.keep
) {
359 return 0, errDiscardedBytes
362 n
= copy(p
, suf
[len(suf
)-int(back
):])
369 func (ss
*suffixSaver
) Suffix() []byte {
370 if len(ss
.buf
) < ss
.keep
{
373 buf
:= make([]byte, ss
.keep
)
374 n
:= copy(buf
, ss
.buf
[ss
.start
:])
375 copy(buf
[n
:], ss
.buf
[:])
379 func (ss
*suffixSaver
) Write(p
[]byte) (n
int, err error
) {
381 ss
.size
+= int64(len(p
))
382 if len(ss
.buf
) < ss
.keep
{
383 space
:= ss
.keep
- len(ss
.buf
)
388 ss
.buf
= append(ss
.buf
, p
[:add
]...)
392 n
:= copy(ss
.buf
[ss
.start
:], p
)
395 if ss
.start
== ss
.keep
{
402 // generatesZip64 reports whether f wrote a zip64 file.
403 // f is also responsible for closing w.
404 func generatesZip64(t
*testing
.T
, f
func(w
*Writer
)) bool {
405 ss
:= &suffixSaver
{keep
: 10 << 20}
408 return suffixIsZip64(t
, ss
)
411 type sizedReaderAt
interface {
416 func suffixIsZip64(t
*testing
.T
, zip sizedReaderAt
) bool {
417 d
:= make([]byte, 1024)
418 if _
, err
:= zip
.ReadAt(d
, zip
.Size()-int64(len(d
))); err
!= nil {
419 t
.Fatalf("ReadAt: %v", err
)
422 sigOff
:= findSignatureInBlock(d
)
424 t
.Errorf("failed to find signature in block")
428 dirOff
, err
:= findDirectory64End(zip
, zip
.Size()-int64(len(d
))+int64(sigOff
))
430 t
.Fatalf("findDirectory64End: %v", err
)
436 d
= make([]byte, directory64EndLen
)
437 if _
, err
:= zip
.ReadAt(d
, dirOff
); err
!= nil {
438 t
.Fatalf("ReadAt(off=%d): %v", dirOff
, err
)
442 if sig
:= b
.uint32(); sig
!= directory64EndSignature
{
447 if size
!= directory64EndLen
-12 {
448 t
.Errorf("expected length of %d, got %d", directory64EndLen
-12, size
)
453 // Zip64 is required if the total size of the records is uint32max.
454 func TestZip64LargeDirectory(t
*testing
.T
) {
456 t
.Skip("skipping in short mode")
459 // gen returns a func that writes a zip with a wantLen bytes
460 // of central directory.
461 gen
:= func(wantLen
int64) func(*Writer
) {
462 return func(w
*Writer
) {
463 w
.testHookCloseSizeOffset
= func(size
, off
uint64) {
464 if size
!= uint64(wantLen
) {
465 t
.Errorf("Close central directory size = %d; want %d", size
, wantLen
)
469 uint16string
:= strings
.Repeat(".", uint16max
)
472 commentLen
:= int(uint16max
) - directoryHeaderLen
- 1
473 thisRecLen
:= directoryHeaderLen
+ int(uint16max
) + commentLen
474 if int64(thisRecLen
) > remain
{
475 remove
:= thisRecLen
- int(remain
)
479 remain
-= int64(thisRecLen
)
480 f
, err
:= w
.CreateHeader(&FileHeader
{
482 Comment
: uint16string
[:commentLen
],
485 t
.Fatalf("CreateHeader: %v", err
)
487 f
.(*fileWriter
).crc32
= fakeHash32
{}
489 if err
:= w
.Close(); err
!= nil {
490 t
.Fatalf("Close: %v", err
)
494 t
.Run("uint32max-1_NoZip64", func(t
*testing
.T
) {
496 if generatesZip64(t
, gen(uint32max
-1)) {
497 t
.Error("unexpected zip64")
500 t
.Run("uint32max_HasZip64", func(t
*testing
.T
) {
502 if !generatesZip64(t
, gen(uint32max
)) {
503 t
.Error("expected zip64")
508 func testZip64(t testing
.TB
, size
int64) *rleBuffer
{
509 const chunkSize
= 1024
510 chunks
:= int(size
/ chunkSize
)
511 // write size bytes plus "END\n" to a zip file
512 buf
:= new(rleBuffer
)
514 f
, err
:= w
.CreateHeader(&FileHeader
{
521 f
.(*fileWriter
).crc32
= fakeHash32
{}
522 chunk
:= make([]byte, chunkSize
)
523 for i
:= range chunk
{
526 for i
:= 0; i
< chunks
; i
++ {
527 _
, err
:= f
.Write(chunk
)
529 t
.Fatal("write chunk:", err
)
532 if frag
:= int(size
% chunkSize
); frag
> 0 {
533 _
, err
:= f
.Write(chunk
[:frag
])
535 t
.Fatal("write chunk:", err
)
538 end
:= []byte("END\n")
539 _
, err
= f
.Write(end
)
541 t
.Fatal("write end:", err
)
543 if err
:= w
.Close(); err
!= nil {
547 // read back zip file and check that we get to the end of it
548 r
, err
:= NewReader(buf
, int64(buf
.Size()))
550 t
.Fatal("reader:", err
)
555 t
.Fatal("opening:", err
)
557 rc
.(*checksumReader
).hash
= fakeHash32
{}
558 for i
:= 0; i
< chunks
; i
++ {
559 _
, err
:= io
.ReadFull(rc
, chunk
)
561 t
.Fatal("read:", err
)
564 if frag
:= int(size
% chunkSize
); frag
> 0 {
565 _
, err
:= io
.ReadFull(rc
, chunk
[:frag
])
567 t
.Fatal("read:", err
)
570 gotEnd
, err
:= ioutil
.ReadAll(rc
)
572 t
.Fatal("read end:", err
)
574 if !bytes
.Equal(gotEnd
, end
) {
575 t
.Errorf("End of zip64 archive %q, want %q", gotEnd
, end
)
579 t
.Fatal("closing:", err
)
581 if size
+int64(len("END\n")) >= 1<<32-1 {
582 if got
, want
:= f0
.UncompressedSize
, uint32(uint32max
); got
!= want
{
583 t
.Errorf("UncompressedSize %#x, want %#x", got
, want
)
587 if got
, want
:= f0
.UncompressedSize64
, uint64(size
)+uint64(len(end
)); got
!= want
{
588 t
.Errorf("UncompressedSize64 %#x, want %#x", got
, want
)
595 func testZip64DirectoryRecordLength(buf
*rleBuffer
, t
*testing
.T
) {
596 if !suffixIsZip64(t
, buf
) {
597 t
.Fatal("not a zip64")
601 func testValidHeader(h
*FileHeader
, t
*testing
.T
) {
605 f
, err
:= z
.CreateHeader(h
)
607 t
.Fatalf("error creating header: %v", err
)
609 if _
, err
:= f
.Write([]byte("hi")); err
!= nil {
610 t
.Fatalf("error writing content: %v", err
)
612 if err
:= z
.Close(); err
!= nil {
613 t
.Fatalf("error closing zip writer: %v", err
)
617 zf
, err
:= NewReader(bytes
.NewReader(b
), int64(len(b
)))
619 t
.Fatalf("got %v, expected nil", err
)
621 zh
:= zf
.File
[0].FileHeader
622 if zh
.Name
!= h
.Name || zh
.Method
!= h
.Method || zh
.UncompressedSize64
!= uint64(len("hi")) {
623 t
.Fatalf("got %q/%d/%d expected %q/%d/%d", zh
.Name
, zh
.Method
, zh
.UncompressedSize64
, h
.Name
, h
.Method
, len("hi"))
628 func TestHeaderInvalidTagAndSize(t
*testing
.T
) {
629 const timeFormat
= "20060102T150405.000.txt"
632 filename
:= ts
.Format(timeFormat
)
637 Extra
: []byte(ts
.Format(time
.RFC3339Nano
)), // missing tag and len, but Extra is best-effort parsing
641 testValidHeader(&h
, t
)
644 func TestHeaderTooShort(t
*testing
.T
) {
648 Extra
: []byte{zip64ExtraId
}, // missing size and second half of tag, but Extra is best-effort parsing
650 testValidHeader(&h
, t
)
653 func TestHeaderIgnoredSize(t
*testing
.T
) {
657 Extra
: []byte{zip64ExtraId
& 0xFF, zip64ExtraId
>> 8, 24, 0, 1, 2, 3, 4, 5, 6, 7, 8, 1, 2, 3, 4, 5, 6, 7, 8, 1, 2, 3, 4, 5, 6, 7, 8, 1, 2, 3, 4, 5, 6, 7, 8}, // bad size but shouldn't be consulted
659 testValidHeader(&h
, t
)
662 // Issue 4393. It is valid to have an extra data header
663 // which contains no body.
664 func TestZeroLengthHeader(t
*testing
.T
) {
666 Name
: "extadata.txt",
669 85, 84, 5, 0, 3, 154, 144, 195, 77, // tag 21589 size 5
670 85, 120, 0, 0, // tag 30805 size 0
673 testValidHeader(&h
, t
)
676 // Just benchmarking how fast the Zip64 test above is. Not related to
677 // our zip performance, since the test above disabled CRC32 and flate.
678 func BenchmarkZip64Test(b
*testing
.B
) {
679 for i
:= 0; i
< b
.N
; i
++ {
684 func BenchmarkZip64TestSizes(b
*testing
.B
) {
685 for _
, size
:= range []int64{1 << 12, 1 << 20, 1 << 26} {
686 b
.Run(fmt
.Sprint(size
), func(b
*testing
.B
) {
687 b
.RunParallel(func(pb
*testing
.PB
) {
696 func TestSuffixSaver(t
*testing
.T
) {
698 ss
:= &suffixSaver
{keep
: keep
}
699 ss
.Write([]byte("abc"))
700 if got
:= string(ss
.Suffix()); got
!= "abc" {
701 t
.Errorf("got = %q; want abc", got
)
703 ss
.Write([]byte("defghijklmno"))
704 if got
:= string(ss
.Suffix()); got
!= "fghijklmno" {
705 t
.Errorf("got = %q; want fghijklmno", got
)
707 if got
, want
:= ss
.Size(), int64(len("abc")+len("defghijklmno")); got
!= want
{
708 t
.Errorf("Size = %d; want %d", got
, want
)
710 buf
:= make([]byte, ss
.Size())
711 for off
:= int64(0); off
< ss
.Size(); off
++ {
712 for size
:= 1; size
<= int(ss
.Size()-off
); size
++ {
713 readBuf
:= buf
[:size
]
714 n
, err
:= ss
.ReadAt(readBuf
, off
)
715 if off
< ss
.Size()-keep
{
716 if err
!= errDiscardedBytes
{
717 t
.Errorf("off %d, size %d = %v, %v (%q); want errDiscardedBytes", off
, size
, n
, err
, readBuf
[:n
])
721 want
:= "abcdefghijklmno"[off
: off
+int64(size
)]
722 got
:= string(readBuf
[:n
])
723 if err
!= nil || got
!= want
{
724 t
.Errorf("off %d, size %d = %v, %v (%q); want %q", off
, size
, n
, err
, got
, want
)
733 func (zeros
) Read(p
[]byte) (int, error
) {