2 * Copyright (c) 2006 Rene Scharfe
9 #include "xdiff-interface.h"
14 /* We only care about the "buf" part here. */
15 static struct strbuf zip_dir
;
17 static uintmax_t zip_offset
;
18 static uint64_t zip_dir_entries
;
20 static unsigned int max_creator_version
;
22 #define ZIP_STREAM (1 << 3)
23 #define ZIP_UTF8 (1 << 11)
25 struct zip_local_header
{
26 unsigned char magic
[4];
27 unsigned char version
[2];
28 unsigned char flags
[2];
29 unsigned char compression_method
[2];
30 unsigned char mtime
[2];
31 unsigned char mdate
[2];
32 unsigned char crc32
[4];
33 unsigned char compressed_size
[4];
34 unsigned char size
[4];
35 unsigned char filename_length
[2];
36 unsigned char extra_length
[2];
37 unsigned char _end
[1];
40 struct zip_data_desc
{
41 unsigned char magic
[4];
42 unsigned char crc32
[4];
43 unsigned char compressed_size
[4];
44 unsigned char size
[4];
45 unsigned char _end
[1];
48 struct zip64_data_desc
{
49 unsigned char magic
[4];
50 unsigned char crc32
[4];
51 unsigned char compressed_size
[8];
52 unsigned char size
[8];
53 unsigned char _end
[1];
56 struct zip_dir_trailer
{
57 unsigned char magic
[4];
58 unsigned char disk
[2];
59 unsigned char directory_start_disk
[2];
60 unsigned char entries_on_this_disk
[2];
61 unsigned char entries
[2];
62 unsigned char size
[4];
63 unsigned char offset
[4];
64 unsigned char comment_length
[2];
65 unsigned char _end
[1];
68 struct zip_extra_mtime
{
69 unsigned char magic
[2];
70 unsigned char extra_size
[2];
71 unsigned char flags
[1];
72 unsigned char mtime
[4];
73 unsigned char _end
[1];
77 unsigned char magic
[2];
78 unsigned char extra_size
[2];
79 unsigned char size
[8];
80 unsigned char compressed_size
[8];
81 unsigned char _end
[1];
84 struct zip64_dir_trailer
{
85 unsigned char magic
[4];
86 unsigned char record_size
[8];
87 unsigned char creator_version
[2];
88 unsigned char version
[2];
89 unsigned char disk
[4];
90 unsigned char directory_start_disk
[4];
91 unsigned char entries_on_this_disk
[8];
92 unsigned char entries
[8];
93 unsigned char size
[8];
94 unsigned char offset
[8];
95 unsigned char _end
[1];
98 struct zip64_dir_trailer_locator
{
99 unsigned char magic
[4];
100 unsigned char disk
[4];
101 unsigned char offset
[8];
102 unsigned char number_of_disks
[4];
103 unsigned char _end
[1];
107 * On ARM, padding is added at the end of the struct, so a simple
108 * sizeof(struct ...) reports two bytes more than the payload size
109 * we're interested in.
111 #define ZIP_LOCAL_HEADER_SIZE offsetof(struct zip_local_header, _end)
112 #define ZIP_DATA_DESC_SIZE offsetof(struct zip_data_desc, _end)
113 #define ZIP64_DATA_DESC_SIZE offsetof(struct zip64_data_desc, _end)
114 #define ZIP_DIR_HEADER_SIZE offsetof(struct zip_dir_header, _end)
115 #define ZIP_DIR_TRAILER_SIZE offsetof(struct zip_dir_trailer, _end)
116 #define ZIP_EXTRA_MTIME_SIZE offsetof(struct zip_extra_mtime, _end)
117 #define ZIP_EXTRA_MTIME_PAYLOAD_SIZE \
118 (ZIP_EXTRA_MTIME_SIZE - offsetof(struct zip_extra_mtime, flags))
119 #define ZIP64_EXTRA_SIZE offsetof(struct zip64_extra, _end)
120 #define ZIP64_EXTRA_PAYLOAD_SIZE \
121 (ZIP64_EXTRA_SIZE - offsetof(struct zip64_extra, size))
122 #define ZIP64_DIR_TRAILER_SIZE offsetof(struct zip64_dir_trailer, _end)
123 #define ZIP64_DIR_TRAILER_RECORD_SIZE \
124 (ZIP64_DIR_TRAILER_SIZE - \
125 offsetof(struct zip64_dir_trailer, creator_version))
126 #define ZIP64_DIR_TRAILER_LOCATOR_SIZE \
127 offsetof(struct zip64_dir_trailer_locator, _end)
129 static void copy_le16(unsigned char *dest
, unsigned int n
)
132 dest
[1] = 0xff & (n
>> 010);
135 static void copy_le32(unsigned char *dest
, unsigned int n
)
138 dest
[1] = 0xff & (n
>> 010);
139 dest
[2] = 0xff & (n
>> 020);
140 dest
[3] = 0xff & (n
>> 030);
143 static void copy_le64(unsigned char *dest
, uint64_t n
)
146 dest
[1] = 0xff & (n
>> 010);
147 dest
[2] = 0xff & (n
>> 020);
148 dest
[3] = 0xff & (n
>> 030);
149 dest
[4] = 0xff & (n
>> 040);
150 dest
[5] = 0xff & (n
>> 050);
151 dest
[6] = 0xff & (n
>> 060);
152 dest
[7] = 0xff & (n
>> 070);
155 static uint64_t clamp_max(uint64_t n
, uint64_t max
, int *clamped
)
163 static void copy_le16_clamp(unsigned char *dest
, uint64_t n
, int *clamped
)
165 copy_le16(dest
, clamp_max(n
, 0xffff, clamped
));
168 static void copy_le32_clamp(unsigned char *dest
, uint64_t n
, int *clamped
)
170 copy_le32(dest
, clamp_max(n
, 0xffffffff, clamped
));
173 static int strbuf_add_le(struct strbuf
*sb
, size_t size
, uintmax_t n
)
176 strbuf_addch(sb
, n
& 0xff);
182 static uint32_t clamp32(uintmax_t n
)
184 const uintmax_t max
= 0xffffffff;
185 return (n
< max
) ? n
: max
;
188 static void *zlib_deflate_raw(void *data
, unsigned long size
,
189 int compression_level
,
190 unsigned long *compressed_size
)
193 unsigned long maxsize
;
197 git_deflate_init_raw(&stream
, compression_level
);
198 maxsize
= git_deflate_bound(&stream
, size
);
199 buffer
= xmalloc(maxsize
);
201 stream
.next_in
= data
;
202 stream
.avail_in
= size
;
203 stream
.next_out
= buffer
;
204 stream
.avail_out
= maxsize
;
207 result
= git_deflate(&stream
, Z_FINISH
);
208 } while (result
== Z_OK
);
210 if (result
!= Z_STREAM_END
) {
215 git_deflate_end(&stream
);
216 *compressed_size
= stream
.total_out
;
221 static void write_zip_data_desc(unsigned long size
,
222 unsigned long compressed_size
,
225 if (size
>= 0xffffffff || compressed_size
>= 0xffffffff) {
226 struct zip64_data_desc trailer
;
227 copy_le32(trailer
.magic
, 0x08074b50);
228 copy_le32(trailer
.crc32
, crc
);
229 copy_le64(trailer
.compressed_size
, compressed_size
);
230 copy_le64(trailer
.size
, size
);
231 write_or_die(1, &trailer
, ZIP64_DATA_DESC_SIZE
);
232 zip_offset
+= ZIP64_DATA_DESC_SIZE
;
234 struct zip_data_desc trailer
;
235 copy_le32(trailer
.magic
, 0x08074b50);
236 copy_le32(trailer
.crc32
, crc
);
237 copy_le32(trailer
.compressed_size
, compressed_size
);
238 copy_le32(trailer
.size
, size
);
239 write_or_die(1, &trailer
, ZIP_DATA_DESC_SIZE
);
240 zip_offset
+= ZIP_DATA_DESC_SIZE
;
244 static void set_zip_header_data_desc(struct zip_local_header
*header
,
246 unsigned long compressed_size
,
249 copy_le32(header
->crc32
, crc
);
250 copy_le32(header
->compressed_size
, compressed_size
);
251 copy_le32(header
->size
, size
);
254 static int has_only_ascii(const char *s
)
265 static int entry_is_binary(const char *path
, const void *buffer
, size_t size
)
267 struct userdiff_driver
*driver
= userdiff_find_by_path(path
);
269 driver
= userdiff_find_by_name("default");
270 if (driver
->binary
!= -1)
271 return driver
->binary
;
272 return buffer_is_binary(buffer
, size
);
275 #define STREAM_BUFFER_SIZE (1024 * 16)
277 static int write_zip_entry(struct archiver_args
*args
,
278 const unsigned char *sha1
,
279 const char *path
, size_t pathlen
,
282 struct zip_local_header header
;
283 uintmax_t offset
= zip_offset
;
284 struct zip_extra_mtime extra
;
285 struct zip64_extra extra64
;
286 size_t header_extra_size
= ZIP_EXTRA_MTIME_SIZE
;
287 int need_zip64_extra
= 0;
289 unsigned long compressed_size
;
293 void *deflated
= NULL
;
295 struct git_istream
*stream
= NULL
;
296 unsigned long flags
= 0;
299 const char *path_without_prefix
= path
+ args
->baselen
;
300 unsigned int creator_version
= 0;
301 size_t zip_dir_extra_size
= ZIP_EXTRA_MTIME_SIZE
;
302 size_t zip64_dir_extra_payload_size
= 0;
304 crc
= crc32(0, NULL
, 0);
306 if (!has_only_ascii(path
)) {
310 warning("Path is not valid UTF-8: %s", path
);
313 if (pathlen
> 0xffff) {
314 return error("path too long (%d chars, SHA1: %s): %s",
315 (int)pathlen
, sha1_to_hex(sha1
), path
);
318 if (S_ISDIR(mode
) || S_ISGITLINK(mode
)) {
325 } else if (S_ISREG(mode
) || S_ISLNK(mode
)) {
326 enum object_type type
= sha1_object_info(sha1
, &size
);
329 attr2
= S_ISLNK(mode
) ? ((mode
| 0777) << 16) :
330 (mode
& 0111) ? ((mode
) << 16) : 0;
331 if (S_ISLNK(mode
) || (mode
& 0111))
332 creator_version
= 0x0317;
333 if (S_ISREG(mode
) && args
->compression_level
!= 0 && size
> 0)
336 if (S_ISREG(mode
) && type
== OBJ_BLOB
&& !args
->convert
&&
337 size
> big_file_threshold
) {
338 stream
= open_istream(sha1
, &type
, &size
, NULL
);
340 return error("cannot stream blob %s",
345 buffer
= sha1_file_to_archive(args
, path
, sha1
, mode
,
348 return error("cannot read %s",
350 crc
= crc32(crc
, buffer
, size
);
351 is_binary
= entry_is_binary(path_without_prefix
,
355 compressed_size
= (method
== 0) ? size
: 0;
357 return error("unsupported file mode: 0%o (SHA1: %s)", mode
,
361 if (creator_version
> max_creator_version
)
362 max_creator_version
= creator_version
;
364 if (buffer
&& method
== 8) {
365 out
= deflated
= zlib_deflate_raw(buffer
, size
,
366 args
->compression_level
,
368 if (!out
|| compressed_size
>= size
) {
371 compressed_size
= size
;
375 copy_le16(extra
.magic
, 0x5455);
376 copy_le16(extra
.extra_size
, ZIP_EXTRA_MTIME_PAYLOAD_SIZE
);
377 extra
.flags
[0] = 1; /* just mtime */
378 copy_le32(extra
.mtime
, args
->time
);
380 if (size
> 0xffffffff || compressed_size
> 0xffffffff)
381 need_zip64_extra
= 1;
382 if (stream
&& size
> 0x7fffffff)
383 need_zip64_extra
= 1;
385 copy_le32(header
.magic
, 0x04034b50);
386 copy_le16(header
.version
, 10);
387 copy_le16(header
.flags
, flags
);
388 copy_le16(header
.compression_method
, method
);
389 copy_le16(header
.mtime
, zip_time
);
390 copy_le16(header
.mdate
, zip_date
);
391 if (need_zip64_extra
) {
392 set_zip_header_data_desc(&header
, 0xffffffff, 0xffffffff, crc
);
393 header_extra_size
+= ZIP64_EXTRA_SIZE
;
395 set_zip_header_data_desc(&header
, size
, compressed_size
, crc
);
397 copy_le16(header
.filename_length
, pathlen
);
398 copy_le16(header
.extra_length
, header_extra_size
);
399 write_or_die(1, &header
, ZIP_LOCAL_HEADER_SIZE
);
400 zip_offset
+= ZIP_LOCAL_HEADER_SIZE
;
401 write_or_die(1, path
, pathlen
);
402 zip_offset
+= pathlen
;
403 write_or_die(1, &extra
, ZIP_EXTRA_MTIME_SIZE
);
404 zip_offset
+= ZIP_EXTRA_MTIME_SIZE
;
405 if (need_zip64_extra
) {
406 copy_le16(extra64
.magic
, 0x0001);
407 copy_le16(extra64
.extra_size
, ZIP64_EXTRA_PAYLOAD_SIZE
);
408 copy_le64(extra64
.size
, size
);
409 copy_le64(extra64
.compressed_size
, compressed_size
);
410 write_or_die(1, &extra64
, ZIP64_EXTRA_SIZE
);
411 zip_offset
+= ZIP64_EXTRA_SIZE
;
414 if (stream
&& method
== 0) {
415 unsigned char buf
[STREAM_BUFFER_SIZE
];
419 readlen
= read_istream(stream
, buf
, sizeof(buf
));
422 crc
= crc32(crc
, buf
, readlen
);
424 is_binary
= entry_is_binary(path_without_prefix
,
426 write_or_die(1, buf
, readlen
);
428 close_istream(stream
);
432 compressed_size
= size
;
433 zip_offset
+= compressed_size
;
435 write_zip_data_desc(size
, compressed_size
, crc
);
436 } else if (stream
&& method
== 8) {
437 unsigned char buf
[STREAM_BUFFER_SIZE
];
442 unsigned char compressed
[STREAM_BUFFER_SIZE
* 2];
444 git_deflate_init_raw(&zstream
, args
->compression_level
);
447 zstream
.next_out
= compressed
;
448 zstream
.avail_out
= sizeof(compressed
);
451 readlen
= read_istream(stream
, buf
, sizeof(buf
));
454 crc
= crc32(crc
, buf
, readlen
);
456 is_binary
= entry_is_binary(path_without_prefix
,
459 zstream
.next_in
= buf
;
460 zstream
.avail_in
= readlen
;
461 result
= git_deflate(&zstream
, 0);
463 die("deflate error (%d)", result
);
464 out_len
= zstream
.next_out
- compressed
;
467 write_or_die(1, compressed
, out_len
);
468 compressed_size
+= out_len
;
469 zstream
.next_out
= compressed
;
470 zstream
.avail_out
= sizeof(compressed
);
474 close_istream(stream
);
478 zstream
.next_in
= buf
;
479 zstream
.avail_in
= 0;
480 result
= git_deflate(&zstream
, Z_FINISH
);
481 if (result
!= Z_STREAM_END
)
482 die("deflate error (%d)", result
);
484 git_deflate_end(&zstream
);
485 out_len
= zstream
.next_out
- compressed
;
486 write_or_die(1, compressed
, out_len
);
487 compressed_size
+= out_len
;
488 zip_offset
+= compressed_size
;
490 write_zip_data_desc(size
, compressed_size
, crc
);
491 } else if (compressed_size
> 0) {
492 write_or_die(1, out
, compressed_size
);
493 zip_offset
+= compressed_size
;
499 if (compressed_size
> 0xffffffff || size
> 0xffffffff ||
500 offset
> 0xffffffff) {
501 if (compressed_size
>= 0xffffffff)
502 zip64_dir_extra_payload_size
+= 8;
503 if (size
>= 0xffffffff)
504 zip64_dir_extra_payload_size
+= 8;
505 if (offset
>= 0xffffffff)
506 zip64_dir_extra_payload_size
+= 8;
507 zip_dir_extra_size
+= 2 + 2 + zip64_dir_extra_payload_size
;
510 strbuf_add_le(&zip_dir
, 4, 0x02014b50); /* magic */
511 strbuf_add_le(&zip_dir
, 2, creator_version
);
512 strbuf_add_le(&zip_dir
, 2, 10); /* version */
513 strbuf_add_le(&zip_dir
, 2, flags
);
514 strbuf_add_le(&zip_dir
, 2, method
);
515 strbuf_add_le(&zip_dir
, 2, zip_time
);
516 strbuf_add_le(&zip_dir
, 2, zip_date
);
517 strbuf_add_le(&zip_dir
, 4, crc
);
518 strbuf_add_le(&zip_dir
, 4, clamp32(compressed_size
));
519 strbuf_add_le(&zip_dir
, 4, clamp32(size
));
520 strbuf_add_le(&zip_dir
, 2, pathlen
);
521 strbuf_add_le(&zip_dir
, 2, zip_dir_extra_size
);
522 strbuf_add_le(&zip_dir
, 2, 0); /* comment length */
523 strbuf_add_le(&zip_dir
, 2, 0); /* disk */
524 strbuf_add_le(&zip_dir
, 2, !is_binary
);
525 strbuf_add_le(&zip_dir
, 4, attr2
);
526 strbuf_add_le(&zip_dir
, 4, clamp32(offset
));
527 strbuf_add(&zip_dir
, path
, pathlen
);
528 strbuf_add(&zip_dir
, &extra
, ZIP_EXTRA_MTIME_SIZE
);
529 if (zip64_dir_extra_payload_size
) {
530 strbuf_add_le(&zip_dir
, 2, 0x0001); /* magic */
531 strbuf_add_le(&zip_dir
, 2, zip64_dir_extra_payload_size
);
532 if (size
>= 0xffffffff)
533 strbuf_add_le(&zip_dir
, 8, size
);
534 if (compressed_size
>= 0xffffffff)
535 strbuf_add_le(&zip_dir
, 8, compressed_size
);
536 if (offset
>= 0xffffffff)
537 strbuf_add_le(&zip_dir
, 8, offset
);
544 static void write_zip64_trailer(void)
546 struct zip64_dir_trailer trailer64
;
547 struct zip64_dir_trailer_locator locator64
;
549 copy_le32(trailer64
.magic
, 0x06064b50);
550 copy_le64(trailer64
.record_size
, ZIP64_DIR_TRAILER_RECORD_SIZE
);
551 copy_le16(trailer64
.creator_version
, max_creator_version
);
552 copy_le16(trailer64
.version
, 45);
553 copy_le32(trailer64
.disk
, 0);
554 copy_le32(trailer64
.directory_start_disk
, 0);
555 copy_le64(trailer64
.entries_on_this_disk
, zip_dir_entries
);
556 copy_le64(trailer64
.entries
, zip_dir_entries
);
557 copy_le64(trailer64
.size
, zip_dir
.len
);
558 copy_le64(trailer64
.offset
, zip_offset
);
560 copy_le32(locator64
.magic
, 0x07064b50);
561 copy_le32(locator64
.disk
, 0);
562 copy_le64(locator64
.offset
, zip_offset
+ zip_dir
.len
);
563 copy_le32(locator64
.number_of_disks
, 1);
565 write_or_die(1, &trailer64
, ZIP64_DIR_TRAILER_SIZE
);
566 write_or_die(1, &locator64
, ZIP64_DIR_TRAILER_LOCATOR_SIZE
);
569 static void write_zip_trailer(const unsigned char *sha1
)
571 struct zip_dir_trailer trailer
;
574 copy_le32(trailer
.magic
, 0x06054b50);
575 copy_le16(trailer
.disk
, 0);
576 copy_le16(trailer
.directory_start_disk
, 0);
577 copy_le16_clamp(trailer
.entries_on_this_disk
, zip_dir_entries
,
579 copy_le16_clamp(trailer
.entries
, zip_dir_entries
, &clamped
);
580 copy_le32(trailer
.size
, zip_dir
.len
);
581 copy_le32_clamp(trailer
.offset
, zip_offset
, &clamped
);
582 copy_le16(trailer
.comment_length
, sha1
? GIT_SHA1_HEXSZ
: 0);
584 write_or_die(1, zip_dir
.buf
, zip_dir
.len
);
586 write_zip64_trailer();
587 write_or_die(1, &trailer
, ZIP_DIR_TRAILER_SIZE
);
589 write_or_die(1, sha1_to_hex(sha1
), GIT_SHA1_HEXSZ
);
592 static void dos_time(time_t *time
, int *dos_date
, int *dos_time
)
594 struct tm
*t
= localtime(time
);
596 *dos_date
= t
->tm_mday
+ (t
->tm_mon
+ 1) * 32 +
597 (t
->tm_year
+ 1900 - 1980) * 512;
598 *dos_time
= t
->tm_sec
/ 2 + t
->tm_min
* 32 + t
->tm_hour
* 2048;
601 static int archive_zip_config(const char *var
, const char *value
, void *data
)
603 return userdiff_config(var
, value
);
606 static int write_zip_archive(const struct archiver
*ar
,
607 struct archiver_args
*args
)
611 git_config(archive_zip_config
, NULL
);
613 dos_time(&args
->time
, &zip_date
, &zip_time
);
615 strbuf_init(&zip_dir
, 0);
617 err
= write_archive_entries(args
, write_zip_entry
);
619 write_zip_trailer(args
->commit_sha1
);
621 strbuf_release(&zip_dir
);
626 static struct archiver zip_archiver
= {
629 ARCHIVER_WANT_COMPRESSION_LEVELS
|ARCHIVER_REMOTE
632 void init_zip_archiver(void)
634 register_archiver(&zip_archiver
);