libgo: update to go1.9
[official-gcc.git] / libgo / go / archive / zip / zip_test.go
blob18c2171ba6c15ba13937336c286e6857aaace10c
1 // Copyright 2011 The Go Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style
3 // license that can be found in the LICENSE file.
5 // Tests that involve both reading and writing.
7 package zip
9 import (
10 "bytes"
11 "errors"
12 "fmt"
13 "hash"
14 "internal/race"
15 "internal/testenv"
16 "io"
17 "io/ioutil"
18 "sort"
19 "strings"
20 "testing"
21 "time"
24 func TestOver65kFiles(t *testing.T) {
25 if testing.Short() && testenv.Builder() == "" {
26 t.Skip("skipping in short mode")
28 buf := new(bytes.Buffer)
29 w := NewWriter(buf)
30 const nFiles = (1 << 16) + 42
31 for i := 0; i < nFiles; i++ {
32 _, err := w.CreateHeader(&FileHeader{
33 Name: fmt.Sprintf("%d.dat", i),
34 Method: Store, // avoid Issue 6136 and Issue 6138
36 if err != nil {
37 t.Fatalf("creating file %d: %v", i, err)
40 if err := w.Close(); err != nil {
41 t.Fatalf("Writer.Close: %v", err)
43 s := buf.String()
44 zr, err := NewReader(strings.NewReader(s), int64(len(s)))
45 if err != nil {
46 t.Fatalf("NewReader: %v", err)
48 if got := len(zr.File); got != nFiles {
49 t.Fatalf("File contains %d files, want %d", got, nFiles)
51 for i := 0; i < nFiles; i++ {
52 want := fmt.Sprintf("%d.dat", i)
53 if zr.File[i].Name != want {
54 t.Fatalf("File(%d) = %q, want %q", i, zr.File[i].Name, want)
59 func TestModTime(t *testing.T) {
60 var testTime = time.Date(2009, time.November, 10, 23, 45, 58, 0, time.UTC)
61 fh := new(FileHeader)
62 fh.SetModTime(testTime)
63 outTime := fh.ModTime()
64 if !outTime.Equal(testTime) {
65 t.Errorf("times don't match: got %s, want %s", outTime, testTime)
69 func testHeaderRoundTrip(fh *FileHeader, wantUncompressedSize uint32, wantUncompressedSize64 uint64, t *testing.T) {
70 fi := fh.FileInfo()
71 fh2, err := FileInfoHeader(fi)
72 if err != nil {
73 t.Fatal(err)
75 if got, want := fh2.Name, fh.Name; got != want {
76 t.Errorf("Name: got %s, want %s\n", got, want)
78 if got, want := fh2.UncompressedSize, wantUncompressedSize; got != want {
79 t.Errorf("UncompressedSize: got %d, want %d\n", got, want)
81 if got, want := fh2.UncompressedSize64, wantUncompressedSize64; got != want {
82 t.Errorf("UncompressedSize64: got %d, want %d\n", got, want)
84 if got, want := fh2.ModifiedTime, fh.ModifiedTime; got != want {
85 t.Errorf("ModifiedTime: got %d, want %d\n", got, want)
87 if got, want := fh2.ModifiedDate, fh.ModifiedDate; got != want {
88 t.Errorf("ModifiedDate: got %d, want %d\n", got, want)
91 if sysfh, ok := fi.Sys().(*FileHeader); !ok && sysfh != fh {
92 t.Errorf("Sys didn't return original *FileHeader")
96 func TestFileHeaderRoundTrip(t *testing.T) {
97 fh := &FileHeader{
98 Name: "foo.txt",
99 UncompressedSize: 987654321,
100 ModifiedTime: 1234,
101 ModifiedDate: 5678,
103 testHeaderRoundTrip(fh, fh.UncompressedSize, uint64(fh.UncompressedSize), t)
106 func TestFileHeaderRoundTrip64(t *testing.T) {
107 fh := &FileHeader{
108 Name: "foo.txt",
109 UncompressedSize64: 9876543210,
110 ModifiedTime: 1234,
111 ModifiedDate: 5678,
113 testHeaderRoundTrip(fh, uint32max, fh.UncompressedSize64, t)
116 type repeatedByte struct {
117 off int64
118 b byte
119 n int64
122 // rleBuffer is a run-length-encoded byte buffer.
123 // It's an io.Writer (like a bytes.Buffer) and also an io.ReaderAt,
124 // allowing random-access reads.
125 type rleBuffer struct {
126 buf []repeatedByte
129 func (r *rleBuffer) Size() int64 {
130 if len(r.buf) == 0 {
131 return 0
133 last := &r.buf[len(r.buf)-1]
134 return last.off + last.n
137 func (r *rleBuffer) Write(p []byte) (n int, err error) {
138 var rp *repeatedByte
139 if len(r.buf) > 0 {
140 rp = &r.buf[len(r.buf)-1]
141 // Fast path, if p is entirely the same byte repeated.
142 if lastByte := rp.b; len(p) > 0 && p[0] == lastByte {
143 all := true
144 for _, b := range p {
145 if b != lastByte {
146 all = false
147 break
150 if all {
151 rp.n += int64(len(p))
152 return len(p), nil
157 for _, b := range p {
158 if rp == nil || rp.b != b {
159 r.buf = append(r.buf, repeatedByte{r.Size(), b, 1})
160 rp = &r.buf[len(r.buf)-1]
161 } else {
162 rp.n++
165 return len(p), nil
168 func (r *rleBuffer) ReadAt(p []byte, off int64) (n int, err error) {
169 if len(p) == 0 {
170 return
172 skipParts := sort.Search(len(r.buf), func(i int) bool {
173 part := &r.buf[i]
174 return part.off+part.n > off
176 parts := r.buf[skipParts:]
177 if len(parts) > 0 {
178 skipBytes := off - parts[0].off
179 for len(parts) > 0 {
180 part := parts[0]
181 for i := skipBytes; i < part.n; i++ {
182 if n == len(p) {
183 return
185 p[n] = part.b
188 parts = parts[1:]
189 skipBytes = 0
192 if n != len(p) {
193 err = io.ErrUnexpectedEOF
195 return
198 // Just testing the rleBuffer used in the Zip64 test above. Not used by the zip code.
199 func TestRLEBuffer(t *testing.T) {
200 b := new(rleBuffer)
201 var all []byte
202 writes := []string{"abcdeee", "eeeeeee", "eeeefghaaiii"}
203 for _, w := range writes {
204 b.Write([]byte(w))
205 all = append(all, w...)
207 if len(b.buf) != 10 {
208 t.Fatalf("len(b.buf) = %d; want 10", len(b.buf))
211 for i := 0; i < len(all); i++ {
212 for j := 0; j < len(all)-i; j++ {
213 buf := make([]byte, j)
214 n, err := b.ReadAt(buf, int64(i))
215 if err != nil || n != len(buf) {
216 t.Errorf("ReadAt(%d, %d) = %d, %v; want %d, nil", i, j, n, err, len(buf))
218 if !bytes.Equal(buf, all[i:i+j]) {
219 t.Errorf("ReadAt(%d, %d) = %q; want %q", i, j, buf, all[i:i+j])
225 // fakeHash32 is a dummy Hash32 that always returns 0.
226 type fakeHash32 struct {
227 hash.Hash32
230 func (fakeHash32) Write(p []byte) (int, error) { return len(p), nil }
231 func (fakeHash32) Sum32() uint32 { return 0 }
233 func TestZip64(t *testing.T) {
234 if testing.Short() {
235 t.Skip("slow test; skipping")
237 t.Parallel()
238 const size = 1 << 32 // before the "END\n" part
239 buf := testZip64(t, size)
240 testZip64DirectoryRecordLength(buf, t)
243 func TestZip64EdgeCase(t *testing.T) {
244 if testing.Short() {
245 t.Skip("slow test; skipping")
247 t.Parallel()
248 // Test a zip file with uncompressed size 0xFFFFFFFF.
249 // That's the magic marker for a 64-bit file, so even though
250 // it fits in a 32-bit field we must use the 64-bit field.
251 // Go 1.5 and earlier got this wrong,
252 // writing an invalid zip file.
253 const size = 1<<32 - 1 - int64(len("END\n")) // before the "END\n" part
254 buf := testZip64(t, size)
255 testZip64DirectoryRecordLength(buf, t)
258 // Tests that we generate a zip64 file if the directory at offset
259 // 0xFFFFFFFF, but not before.
260 func TestZip64DirectoryOffset(t *testing.T) {
261 if testing.Short() && race.Enabled {
262 t.Skip("skipping in short mode")
264 t.Parallel()
265 const filename = "huge.txt"
266 gen := func(wantOff uint64) func(*Writer) {
267 return func(w *Writer) {
268 w.testHookCloseSizeOffset = func(size, off uint64) {
269 if off != wantOff {
270 t.Errorf("central directory offset = %d (%x); want %d", off, off, wantOff)
273 f, err := w.CreateHeader(&FileHeader{
274 Name: filename,
275 Method: Store,
277 if err != nil {
278 t.Fatal(err)
280 f.(*fileWriter).crc32 = fakeHash32{}
281 size := wantOff - fileHeaderLen - uint64(len(filename)) - dataDescriptorLen
282 if _, err := io.CopyN(f, zeros{}, int64(size)); err != nil {
283 t.Fatal(err)
285 if err := w.Close(); err != nil {
286 t.Fatal(err)
290 t.Run("uint32max-2_NoZip64", func(t *testing.T) {
291 t.Parallel()
292 if generatesZip64(t, gen(0xfffffffe)) {
293 t.Error("unexpected zip64")
296 t.Run("uint32max-1_Zip64", func(t *testing.T) {
297 t.Parallel()
298 if !generatesZip64(t, gen(0xffffffff)) {
299 t.Error("expected zip64")
304 // At 16k records, we need to generate a zip64 file.
305 func TestZip64ManyRecords(t *testing.T) {
306 if testing.Short() && race.Enabled {
307 t.Skip("skipping in short mode")
309 t.Parallel()
310 gen := func(numRec int) func(*Writer) {
311 return func(w *Writer) {
312 for i := 0; i < numRec; i++ {
313 _, err := w.CreateHeader(&FileHeader{
314 Name: "a.txt",
315 Method: Store,
317 if err != nil {
318 t.Fatal(err)
321 if err := w.Close(); err != nil {
322 t.Fatal(err)
326 // 16k-1 records shouldn't make a zip64:
327 t.Run("uint16max-1_NoZip64", func(t *testing.T) {
328 t.Parallel()
329 if generatesZip64(t, gen(0xfffe)) {
330 t.Error("unexpected zip64")
333 // 16k records should make a zip64:
334 t.Run("uint16max_Zip64", func(t *testing.T) {
335 t.Parallel()
336 if !generatesZip64(t, gen(0xffff)) {
337 t.Error("expected zip64")
342 // suffixSaver is an io.Writer & io.ReaderAt that remembers the last 0
343 // to 'keep' bytes of data written to it. Call Suffix to get the
344 // suffix bytes.
345 type suffixSaver struct {
346 keep int
347 buf []byte
348 start int
349 size int64
352 func (ss *suffixSaver) Size() int64 { return ss.size }
354 var errDiscardedBytes = errors.New("ReadAt of discarded bytes")
356 func (ss *suffixSaver) ReadAt(p []byte, off int64) (n int, err error) {
357 back := ss.size - off
358 if back > int64(ss.keep) {
359 return 0, errDiscardedBytes
361 suf := ss.Suffix()
362 n = copy(p, suf[len(suf)-int(back):])
363 if n != len(p) {
364 err = io.EOF
366 return
369 func (ss *suffixSaver) Suffix() []byte {
370 if len(ss.buf) < ss.keep {
371 return ss.buf
373 buf := make([]byte, ss.keep)
374 n := copy(buf, ss.buf[ss.start:])
375 copy(buf[n:], ss.buf[:])
376 return buf
379 func (ss *suffixSaver) Write(p []byte) (n int, err error) {
380 n = len(p)
381 ss.size += int64(len(p))
382 if len(ss.buf) < ss.keep {
383 space := ss.keep - len(ss.buf)
384 add := len(p)
385 if add > space {
386 add = space
388 ss.buf = append(ss.buf, p[:add]...)
389 p = p[add:]
391 for len(p) > 0 {
392 n := copy(ss.buf[ss.start:], p)
393 p = p[n:]
394 ss.start += n
395 if ss.start == ss.keep {
396 ss.start = 0
399 return
402 // generatesZip64 reports whether f wrote a zip64 file.
403 // f is also responsible for closing w.
404 func generatesZip64(t *testing.T, f func(w *Writer)) bool {
405 ss := &suffixSaver{keep: 10 << 20}
406 w := NewWriter(ss)
407 f(w)
408 return suffixIsZip64(t, ss)
411 type sizedReaderAt interface {
412 io.ReaderAt
413 Size() int64
416 func suffixIsZip64(t *testing.T, zip sizedReaderAt) bool {
417 d := make([]byte, 1024)
418 if _, err := zip.ReadAt(d, zip.Size()-int64(len(d))); err != nil {
419 t.Fatalf("ReadAt: %v", err)
422 sigOff := findSignatureInBlock(d)
423 if sigOff == -1 {
424 t.Errorf("failed to find signature in block")
425 return false
428 dirOff, err := findDirectory64End(zip, zip.Size()-int64(len(d))+int64(sigOff))
429 if err != nil {
430 t.Fatalf("findDirectory64End: %v", err)
432 if dirOff == -1 {
433 return false
436 d = make([]byte, directory64EndLen)
437 if _, err := zip.ReadAt(d, dirOff); err != nil {
438 t.Fatalf("ReadAt(off=%d): %v", dirOff, err)
441 b := readBuf(d)
442 if sig := b.uint32(); sig != directory64EndSignature {
443 return false
446 size := b.uint64()
447 if size != directory64EndLen-12 {
448 t.Errorf("expected length of %d, got %d", directory64EndLen-12, size)
450 return true
453 // Zip64 is required if the total size of the records is uint32max.
454 func TestZip64LargeDirectory(t *testing.T) {
455 if testing.Short() {
456 t.Skip("skipping in short mode")
458 t.Parallel()
459 // gen returns a func that writes a zip with a wantLen bytes
460 // of central directory.
461 gen := func(wantLen int64) func(*Writer) {
462 return func(w *Writer) {
463 w.testHookCloseSizeOffset = func(size, off uint64) {
464 if size != uint64(wantLen) {
465 t.Errorf("Close central directory size = %d; want %d", size, wantLen)
469 uint16string := strings.Repeat(".", uint16max)
470 remain := wantLen
471 for remain > 0 {
472 commentLen := int(uint16max) - directoryHeaderLen - 1
473 thisRecLen := directoryHeaderLen + int(uint16max) + commentLen
474 if int64(thisRecLen) > remain {
475 remove := thisRecLen - int(remain)
476 commentLen -= remove
477 thisRecLen -= remove
479 remain -= int64(thisRecLen)
480 f, err := w.CreateHeader(&FileHeader{
481 Name: uint16string,
482 Comment: uint16string[:commentLen],
484 if err != nil {
485 t.Fatalf("CreateHeader: %v", err)
487 f.(*fileWriter).crc32 = fakeHash32{}
489 if err := w.Close(); err != nil {
490 t.Fatalf("Close: %v", err)
494 t.Run("uint32max-1_NoZip64", func(t *testing.T) {
495 t.Parallel()
496 if generatesZip64(t, gen(uint32max-1)) {
497 t.Error("unexpected zip64")
500 t.Run("uint32max_HasZip64", func(t *testing.T) {
501 t.Parallel()
502 if !generatesZip64(t, gen(uint32max)) {
503 t.Error("expected zip64")
508 func testZip64(t testing.TB, size int64) *rleBuffer {
509 const chunkSize = 1024
510 chunks := int(size / chunkSize)
511 // write size bytes plus "END\n" to a zip file
512 buf := new(rleBuffer)
513 w := NewWriter(buf)
514 f, err := w.CreateHeader(&FileHeader{
515 Name: "huge.txt",
516 Method: Store,
518 if err != nil {
519 t.Fatal(err)
521 f.(*fileWriter).crc32 = fakeHash32{}
522 chunk := make([]byte, chunkSize)
523 for i := range chunk {
524 chunk[i] = '.'
526 for i := 0; i < chunks; i++ {
527 _, err := f.Write(chunk)
528 if err != nil {
529 t.Fatal("write chunk:", err)
532 if frag := int(size % chunkSize); frag > 0 {
533 _, err := f.Write(chunk[:frag])
534 if err != nil {
535 t.Fatal("write chunk:", err)
538 end := []byte("END\n")
539 _, err = f.Write(end)
540 if err != nil {
541 t.Fatal("write end:", err)
543 if err := w.Close(); err != nil {
544 t.Fatal(err)
547 // read back zip file and check that we get to the end of it
548 r, err := NewReader(buf, int64(buf.Size()))
549 if err != nil {
550 t.Fatal("reader:", err)
552 f0 := r.File[0]
553 rc, err := f0.Open()
554 if err != nil {
555 t.Fatal("opening:", err)
557 rc.(*checksumReader).hash = fakeHash32{}
558 for i := 0; i < chunks; i++ {
559 _, err := io.ReadFull(rc, chunk)
560 if err != nil {
561 t.Fatal("read:", err)
564 if frag := int(size % chunkSize); frag > 0 {
565 _, err := io.ReadFull(rc, chunk[:frag])
566 if err != nil {
567 t.Fatal("read:", err)
570 gotEnd, err := ioutil.ReadAll(rc)
571 if err != nil {
572 t.Fatal("read end:", err)
574 if !bytes.Equal(gotEnd, end) {
575 t.Errorf("End of zip64 archive %q, want %q", gotEnd, end)
577 err = rc.Close()
578 if err != nil {
579 t.Fatal("closing:", err)
581 if size+int64(len("END\n")) >= 1<<32-1 {
582 if got, want := f0.UncompressedSize, uint32(uint32max); got != want {
583 t.Errorf("UncompressedSize %#x, want %#x", got, want)
587 if got, want := f0.UncompressedSize64, uint64(size)+uint64(len(end)); got != want {
588 t.Errorf("UncompressedSize64 %#x, want %#x", got, want)
591 return buf
594 // Issue 9857
595 func testZip64DirectoryRecordLength(buf *rleBuffer, t *testing.T) {
596 if !suffixIsZip64(t, buf) {
597 t.Fatal("not a zip64")
601 func testValidHeader(h *FileHeader, t *testing.T) {
602 var buf bytes.Buffer
603 z := NewWriter(&buf)
605 f, err := z.CreateHeader(h)
606 if err != nil {
607 t.Fatalf("error creating header: %v", err)
609 if _, err := f.Write([]byte("hi")); err != nil {
610 t.Fatalf("error writing content: %v", err)
612 if err := z.Close(); err != nil {
613 t.Fatalf("error closing zip writer: %v", err)
616 b := buf.Bytes()
617 zf, err := NewReader(bytes.NewReader(b), int64(len(b)))
618 if err != nil {
619 t.Fatalf("got %v, expected nil", err)
621 zh := zf.File[0].FileHeader
622 if zh.Name != h.Name || zh.Method != h.Method || zh.UncompressedSize64 != uint64(len("hi")) {
623 t.Fatalf("got %q/%d/%d expected %q/%d/%d", zh.Name, zh.Method, zh.UncompressedSize64, h.Name, h.Method, len("hi"))
627 // Issue 4302.
628 func TestHeaderInvalidTagAndSize(t *testing.T) {
629 const timeFormat = "20060102T150405.000.txt"
631 ts := time.Now()
632 filename := ts.Format(timeFormat)
634 h := FileHeader{
635 Name: filename,
636 Method: Deflate,
637 Extra: []byte(ts.Format(time.RFC3339Nano)), // missing tag and len, but Extra is best-effort parsing
639 h.SetModTime(ts)
641 testValidHeader(&h, t)
644 func TestHeaderTooShort(t *testing.T) {
645 h := FileHeader{
646 Name: "foo.txt",
647 Method: Deflate,
648 Extra: []byte{zip64ExtraId}, // missing size and second half of tag, but Extra is best-effort parsing
650 testValidHeader(&h, t)
653 func TestHeaderIgnoredSize(t *testing.T) {
654 h := FileHeader{
655 Name: "foo.txt",
656 Method: Deflate,
657 Extra: []byte{zip64ExtraId & 0xFF, zip64ExtraId >> 8, 24, 0, 1, 2, 3, 4, 5, 6, 7, 8, 1, 2, 3, 4, 5, 6, 7, 8, 1, 2, 3, 4, 5, 6, 7, 8, 1, 2, 3, 4, 5, 6, 7, 8}, // bad size but shouldn't be consulted
659 testValidHeader(&h, t)
662 // Issue 4393. It is valid to have an extra data header
663 // which contains no body.
664 func TestZeroLengthHeader(t *testing.T) {
665 h := FileHeader{
666 Name: "extadata.txt",
667 Method: Deflate,
668 Extra: []byte{
669 85, 84, 5, 0, 3, 154, 144, 195, 77, // tag 21589 size 5
670 85, 120, 0, 0, // tag 30805 size 0
673 testValidHeader(&h, t)
676 // Just benchmarking how fast the Zip64 test above is. Not related to
677 // our zip performance, since the test above disabled CRC32 and flate.
678 func BenchmarkZip64Test(b *testing.B) {
679 for i := 0; i < b.N; i++ {
680 testZip64(b, 1<<26)
684 func BenchmarkZip64TestSizes(b *testing.B) {
685 for _, size := range []int64{1 << 12, 1 << 20, 1 << 26} {
686 b.Run(fmt.Sprint(size), func(b *testing.B) {
687 b.RunParallel(func(pb *testing.PB) {
688 for pb.Next() {
689 testZip64(b, size)
696 func TestSuffixSaver(t *testing.T) {
697 const keep = 10
698 ss := &suffixSaver{keep: keep}
699 ss.Write([]byte("abc"))
700 if got := string(ss.Suffix()); got != "abc" {
701 t.Errorf("got = %q; want abc", got)
703 ss.Write([]byte("defghijklmno"))
704 if got := string(ss.Suffix()); got != "fghijklmno" {
705 t.Errorf("got = %q; want fghijklmno", got)
707 if got, want := ss.Size(), int64(len("abc")+len("defghijklmno")); got != want {
708 t.Errorf("Size = %d; want %d", got, want)
710 buf := make([]byte, ss.Size())
711 for off := int64(0); off < ss.Size(); off++ {
712 for size := 1; size <= int(ss.Size()-off); size++ {
713 readBuf := buf[:size]
714 n, err := ss.ReadAt(readBuf, off)
715 if off < ss.Size()-keep {
716 if err != errDiscardedBytes {
717 t.Errorf("off %d, size %d = %v, %v (%q); want errDiscardedBytes", off, size, n, err, readBuf[:n])
719 continue
721 want := "abcdefghijklmno"[off : off+int64(size)]
722 got := string(readBuf[:n])
723 if err != nil || got != want {
724 t.Errorf("off %d, size %d = %v, %v (%q); want %q", off, size, n, err, got, want)
731 type zeros struct{}
733 func (zeros) Read(p []byte) (int, error) {
734 for i := range p {
735 p[i] = 0
737 return len(p), nil