2 * Copyright (c) 2006 Rene Scharfe
10 #include "xdiff-interface.h"
15 static unsigned char *zip_dir
;
16 static unsigned int zip_dir_size
;
18 static unsigned int zip_offset
;
19 static unsigned int zip_dir_offset
;
20 static uint64_t zip_dir_entries
;
22 static unsigned int max_creator_version
;
24 #define ZIP_DIRECTORY_MIN_SIZE (1024 * 1024)
25 #define ZIP_STREAM (1 << 3)
26 #define ZIP_UTF8 (1 << 11)
28 struct zip_local_header
{
29 unsigned char magic
[4];
30 unsigned char version
[2];
31 unsigned char flags
[2];
32 unsigned char compression_method
[2];
33 unsigned char mtime
[2];
34 unsigned char mdate
[2];
35 unsigned char crc32
[4];
36 unsigned char compressed_size
[4];
37 unsigned char size
[4];
38 unsigned char filename_length
[2];
39 unsigned char extra_length
[2];
40 unsigned char _end
[1];
43 struct zip_data_desc
{
44 unsigned char magic
[4];
45 unsigned char crc32
[4];
46 unsigned char compressed_size
[4];
47 unsigned char size
[4];
48 unsigned char _end
[1];
51 struct zip_dir_header
{
52 unsigned char magic
[4];
53 unsigned char creator_version
[2];
54 unsigned char version
[2];
55 unsigned char flags
[2];
56 unsigned char compression_method
[2];
57 unsigned char mtime
[2];
58 unsigned char mdate
[2];
59 unsigned char crc32
[4];
60 unsigned char compressed_size
[4];
61 unsigned char size
[4];
62 unsigned char filename_length
[2];
63 unsigned char extra_length
[2];
64 unsigned char comment_length
[2];
65 unsigned char disk
[2];
66 unsigned char attr1
[2];
67 unsigned char attr2
[4];
68 unsigned char offset
[4];
69 unsigned char _end
[1];
72 struct zip_dir_trailer
{
73 unsigned char magic
[4];
74 unsigned char disk
[2];
75 unsigned char directory_start_disk
[2];
76 unsigned char entries_on_this_disk
[2];
77 unsigned char entries
[2];
78 unsigned char size
[4];
79 unsigned char offset
[4];
80 unsigned char comment_length
[2];
81 unsigned char _end
[1];
84 struct zip_extra_mtime
{
85 unsigned char magic
[2];
86 unsigned char extra_size
[2];
87 unsigned char flags
[1];
88 unsigned char mtime
[4];
89 unsigned char _end
[1];
92 struct zip64_dir_trailer
{
93 unsigned char magic
[4];
94 unsigned char record_size
[8];
95 unsigned char creator_version
[2];
96 unsigned char version
[2];
97 unsigned char disk
[4];
98 unsigned char directory_start_disk
[4];
99 unsigned char entries_on_this_disk
[8];
100 unsigned char entries
[8];
101 unsigned char size
[8];
102 unsigned char offset
[8];
103 unsigned char _end
[1];
106 struct zip64_dir_trailer_locator
{
107 unsigned char magic
[4];
108 unsigned char disk
[4];
109 unsigned char offset
[8];
110 unsigned char number_of_disks
[4];
111 unsigned char _end
[1];
115 * On ARM, padding is added at the end of the struct, so a simple
116 * sizeof(struct ...) reports two bytes more than the payload size
117 * we're interested in.
119 #define ZIP_LOCAL_HEADER_SIZE offsetof(struct zip_local_header, _end)
120 #define ZIP_DATA_DESC_SIZE offsetof(struct zip_data_desc, _end)
121 #define ZIP_DIR_HEADER_SIZE offsetof(struct zip_dir_header, _end)
122 #define ZIP_DIR_TRAILER_SIZE offsetof(struct zip_dir_trailer, _end)
123 #define ZIP_EXTRA_MTIME_SIZE offsetof(struct zip_extra_mtime, _end)
124 #define ZIP_EXTRA_MTIME_PAYLOAD_SIZE \
125 (ZIP_EXTRA_MTIME_SIZE - offsetof(struct zip_extra_mtime, flags))
126 #define ZIP64_DIR_TRAILER_SIZE offsetof(struct zip64_dir_trailer, _end)
127 #define ZIP64_DIR_TRAILER_RECORD_SIZE \
128 (ZIP64_DIR_TRAILER_SIZE - \
129 offsetof(struct zip64_dir_trailer, creator_version))
130 #define ZIP64_DIR_TRAILER_LOCATOR_SIZE \
131 offsetof(struct zip64_dir_trailer_locator, _end)
133 static void copy_le16(unsigned char *dest
, unsigned int n
)
136 dest
[1] = 0xff & (n
>> 010);
139 static void copy_le32(unsigned char *dest
, unsigned int n
)
142 dest
[1] = 0xff & (n
>> 010);
143 dest
[2] = 0xff & (n
>> 020);
144 dest
[3] = 0xff & (n
>> 030);
147 static void copy_le64(unsigned char *dest
, uint64_t n
)
150 dest
[1] = 0xff & (n
>> 010);
151 dest
[2] = 0xff & (n
>> 020);
152 dest
[3] = 0xff & (n
>> 030);
153 dest
[4] = 0xff & (n
>> 040);
154 dest
[5] = 0xff & (n
>> 050);
155 dest
[6] = 0xff & (n
>> 060);
156 dest
[7] = 0xff & (n
>> 070);
159 static uint64_t clamp_max(uint64_t n
, uint64_t max
, int *clamped
)
167 static void copy_le16_clamp(unsigned char *dest
, uint64_t n
, int *clamped
)
169 copy_le16(dest
, clamp_max(n
, 0xffff, clamped
));
172 static void *zlib_deflate_raw(void *data
, unsigned long size
,
173 int compression_level
,
174 unsigned long *compressed_size
)
177 unsigned long maxsize
;
181 git_deflate_init_raw(&stream
, compression_level
);
182 maxsize
= git_deflate_bound(&stream
, size
);
183 buffer
= xmalloc(maxsize
);
185 stream
.next_in
= data
;
186 stream
.avail_in
= size
;
187 stream
.next_out
= buffer
;
188 stream
.avail_out
= maxsize
;
191 result
= git_deflate(&stream
, Z_FINISH
);
192 } while (result
== Z_OK
);
194 if (result
!= Z_STREAM_END
) {
199 git_deflate_end(&stream
);
200 *compressed_size
= stream
.total_out
;
205 static void write_zip_data_desc(unsigned long size
,
206 unsigned long compressed_size
,
209 struct zip_data_desc trailer
;
211 copy_le32(trailer
.magic
, 0x08074b50);
212 copy_le32(trailer
.crc32
, crc
);
213 copy_le32(trailer
.compressed_size
, compressed_size
);
214 copy_le32(trailer
.size
, size
);
215 write_or_die(1, &trailer
, ZIP_DATA_DESC_SIZE
);
218 static void set_zip_dir_data_desc(struct zip_dir_header
*header
,
220 unsigned long compressed_size
,
223 copy_le32(header
->crc32
, crc
);
224 copy_le32(header
->compressed_size
, compressed_size
);
225 copy_le32(header
->size
, size
);
228 static void set_zip_header_data_desc(struct zip_local_header
*header
,
230 unsigned long compressed_size
,
233 copy_le32(header
->crc32
, crc
);
234 copy_le32(header
->compressed_size
, compressed_size
);
235 copy_le32(header
->size
, size
);
238 static int has_only_ascii(const char *s
)
249 static int entry_is_binary(const char *path
, const void *buffer
, size_t size
)
251 struct userdiff_driver
*driver
= userdiff_find_by_path(path
);
253 driver
= userdiff_find_by_name("default");
254 if (driver
->binary
!= -1)
255 return driver
->binary
;
256 return buffer_is_binary(buffer
, size
);
259 #define STREAM_BUFFER_SIZE (1024 * 16)
261 static int write_zip_entry(struct archiver_args
*args
,
262 const unsigned char *sha1
,
263 const char *path
, size_t pathlen
,
266 struct zip_local_header header
;
267 struct zip_dir_header dirent
;
268 struct zip_extra_mtime extra
;
270 unsigned long compressed_size
;
272 unsigned long direntsize
;
275 void *deflated
= NULL
;
277 struct git_istream
*stream
= NULL
;
278 unsigned long flags
= 0;
281 const char *path_without_prefix
= path
+ args
->baselen
;
282 unsigned int creator_version
= 0;
284 crc
= crc32(0, NULL
, 0);
286 if (!has_only_ascii(path
)) {
290 warning("Path is not valid UTF-8: %s", path
);
293 if (pathlen
> 0xffff) {
294 return error("path too long (%d chars, SHA1: %s): %s",
295 (int)pathlen
, sha1_to_hex(sha1
), path
);
298 if (S_ISDIR(mode
) || S_ISGITLINK(mode
)) {
305 } else if (S_ISREG(mode
) || S_ISLNK(mode
)) {
306 enum object_type type
= sha1_object_info(sha1
, &size
);
309 attr2
= S_ISLNK(mode
) ? ((mode
| 0777) << 16) :
310 (mode
& 0111) ? ((mode
) << 16) : 0;
311 if (S_ISLNK(mode
) || (mode
& 0111))
312 creator_version
= 0x0317;
313 if (S_ISREG(mode
) && args
->compression_level
!= 0 && size
> 0)
316 if (S_ISREG(mode
) && type
== OBJ_BLOB
&& !args
->convert
&&
317 size
> big_file_threshold
) {
318 stream
= open_istream(sha1
, &type
, &size
, NULL
);
320 return error("cannot stream blob %s",
325 buffer
= sha1_file_to_archive(args
, path
, sha1
, mode
,
328 return error("cannot read %s",
330 crc
= crc32(crc
, buffer
, size
);
331 is_binary
= entry_is_binary(path_without_prefix
,
335 compressed_size
= (method
== 0) ? size
: 0;
337 return error("unsupported file mode: 0%o (SHA1: %s)", mode
,
341 if (creator_version
> max_creator_version
)
342 max_creator_version
= creator_version
;
344 if (buffer
&& method
== 8) {
345 out
= deflated
= zlib_deflate_raw(buffer
, size
,
346 args
->compression_level
,
348 if (!out
|| compressed_size
>= size
) {
351 compressed_size
= size
;
355 copy_le16(extra
.magic
, 0x5455);
356 copy_le16(extra
.extra_size
, ZIP_EXTRA_MTIME_PAYLOAD_SIZE
);
357 extra
.flags
[0] = 1; /* just mtime */
358 copy_le32(extra
.mtime
, args
->time
);
360 /* make sure we have enough free space in the dictionary */
361 direntsize
= ZIP_DIR_HEADER_SIZE
+ pathlen
+ ZIP_EXTRA_MTIME_SIZE
;
362 while (zip_dir_size
< zip_dir_offset
+ direntsize
) {
363 zip_dir_size
+= ZIP_DIRECTORY_MIN_SIZE
;
364 zip_dir
= xrealloc(zip_dir
, zip_dir_size
);
367 copy_le32(dirent
.magic
, 0x02014b50);
368 copy_le16(dirent
.creator_version
, creator_version
);
369 copy_le16(dirent
.version
, 10);
370 copy_le16(dirent
.flags
, flags
);
371 copy_le16(dirent
.compression_method
, method
);
372 copy_le16(dirent
.mtime
, zip_time
);
373 copy_le16(dirent
.mdate
, zip_date
);
374 set_zip_dir_data_desc(&dirent
, size
, compressed_size
, crc
);
375 copy_le16(dirent
.filename_length
, pathlen
);
376 copy_le16(dirent
.extra_length
, ZIP_EXTRA_MTIME_SIZE
);
377 copy_le16(dirent
.comment_length
, 0);
378 copy_le16(dirent
.disk
, 0);
379 copy_le32(dirent
.attr2
, attr2
);
380 copy_le32(dirent
.offset
, zip_offset
);
382 copy_le32(header
.magic
, 0x04034b50);
383 copy_le16(header
.version
, 10);
384 copy_le16(header
.flags
, flags
);
385 copy_le16(header
.compression_method
, method
);
386 copy_le16(header
.mtime
, zip_time
);
387 copy_le16(header
.mdate
, zip_date
);
388 set_zip_header_data_desc(&header
, size
, compressed_size
, crc
);
389 copy_le16(header
.filename_length
, pathlen
);
390 copy_le16(header
.extra_length
, ZIP_EXTRA_MTIME_SIZE
);
391 write_or_die(1, &header
, ZIP_LOCAL_HEADER_SIZE
);
392 zip_offset
+= ZIP_LOCAL_HEADER_SIZE
;
393 write_or_die(1, path
, pathlen
);
394 zip_offset
+= pathlen
;
395 write_or_die(1, &extra
, ZIP_EXTRA_MTIME_SIZE
);
396 zip_offset
+= ZIP_EXTRA_MTIME_SIZE
;
397 if (stream
&& method
== 0) {
398 unsigned char buf
[STREAM_BUFFER_SIZE
];
402 readlen
= read_istream(stream
, buf
, sizeof(buf
));
405 crc
= crc32(crc
, buf
, readlen
);
407 is_binary
= entry_is_binary(path_without_prefix
,
409 write_or_die(1, buf
, readlen
);
411 close_istream(stream
);
415 compressed_size
= size
;
416 zip_offset
+= compressed_size
;
418 write_zip_data_desc(size
, compressed_size
, crc
);
419 zip_offset
+= ZIP_DATA_DESC_SIZE
;
421 set_zip_dir_data_desc(&dirent
, size
, compressed_size
, crc
);
422 } else if (stream
&& method
== 8) {
423 unsigned char buf
[STREAM_BUFFER_SIZE
];
428 unsigned char compressed
[STREAM_BUFFER_SIZE
* 2];
430 git_deflate_init_raw(&zstream
, args
->compression_level
);
433 zstream
.next_out
= compressed
;
434 zstream
.avail_out
= sizeof(compressed
);
437 readlen
= read_istream(stream
, buf
, sizeof(buf
));
440 crc
= crc32(crc
, buf
, readlen
);
442 is_binary
= entry_is_binary(path_without_prefix
,
445 zstream
.next_in
= buf
;
446 zstream
.avail_in
= readlen
;
447 result
= git_deflate(&zstream
, 0);
449 die("deflate error (%d)", result
);
450 out_len
= zstream
.next_out
- compressed
;
453 write_or_die(1, compressed
, out_len
);
454 compressed_size
+= out_len
;
455 zstream
.next_out
= compressed
;
456 zstream
.avail_out
= sizeof(compressed
);
460 close_istream(stream
);
464 zstream
.next_in
= buf
;
465 zstream
.avail_in
= 0;
466 result
= git_deflate(&zstream
, Z_FINISH
);
467 if (result
!= Z_STREAM_END
)
468 die("deflate error (%d)", result
);
470 git_deflate_end(&zstream
);
471 out_len
= zstream
.next_out
- compressed
;
472 write_or_die(1, compressed
, out_len
);
473 compressed_size
+= out_len
;
474 zip_offset
+= compressed_size
;
476 write_zip_data_desc(size
, compressed_size
, crc
);
477 zip_offset
+= ZIP_DATA_DESC_SIZE
;
479 set_zip_dir_data_desc(&dirent
, size
, compressed_size
, crc
);
480 } else if (compressed_size
> 0) {
481 write_or_die(1, out
, compressed_size
);
482 zip_offset
+= compressed_size
;
488 copy_le16(dirent
.attr1
, !is_binary
);
490 memcpy(zip_dir
+ zip_dir_offset
, &dirent
, ZIP_DIR_HEADER_SIZE
);
491 zip_dir_offset
+= ZIP_DIR_HEADER_SIZE
;
492 memcpy(zip_dir
+ zip_dir_offset
, path
, pathlen
);
493 zip_dir_offset
+= pathlen
;
494 memcpy(zip_dir
+ zip_dir_offset
, &extra
, ZIP_EXTRA_MTIME_SIZE
);
495 zip_dir_offset
+= ZIP_EXTRA_MTIME_SIZE
;
501 static void write_zip64_trailer(void)
503 struct zip64_dir_trailer trailer64
;
504 struct zip64_dir_trailer_locator locator64
;
506 copy_le32(trailer64
.magic
, 0x06064b50);
507 copy_le64(trailer64
.record_size
, ZIP64_DIR_TRAILER_RECORD_SIZE
);
508 copy_le16(trailer64
.creator_version
, max_creator_version
);
509 copy_le16(trailer64
.version
, 45);
510 copy_le32(trailer64
.disk
, 0);
511 copy_le32(trailer64
.directory_start_disk
, 0);
512 copy_le64(trailer64
.entries_on_this_disk
, zip_dir_entries
);
513 copy_le64(trailer64
.entries
, zip_dir_entries
);
514 copy_le64(trailer64
.size
, zip_dir_offset
);
515 copy_le64(trailer64
.offset
, zip_offset
);
517 copy_le32(locator64
.magic
, 0x07064b50);
518 copy_le32(locator64
.disk
, 0);
519 copy_le64(locator64
.offset
, zip_offset
+ zip_dir_offset
);
520 copy_le32(locator64
.number_of_disks
, 1);
522 write_or_die(1, &trailer64
, ZIP64_DIR_TRAILER_SIZE
);
523 write_or_die(1, &locator64
, ZIP64_DIR_TRAILER_LOCATOR_SIZE
);
526 static void write_zip_trailer(const unsigned char *sha1
)
528 struct zip_dir_trailer trailer
;
531 copy_le32(trailer
.magic
, 0x06054b50);
532 copy_le16(trailer
.disk
, 0);
533 copy_le16(trailer
.directory_start_disk
, 0);
534 copy_le16_clamp(trailer
.entries_on_this_disk
, zip_dir_entries
,
536 copy_le16_clamp(trailer
.entries
, zip_dir_entries
, &clamped
);
537 copy_le32(trailer
.size
, zip_dir_offset
);
538 copy_le32(trailer
.offset
, zip_offset
);
539 copy_le16(trailer
.comment_length
, sha1
? GIT_SHA1_HEXSZ
: 0);
541 write_or_die(1, zip_dir
, zip_dir_offset
);
543 write_zip64_trailer();
544 write_or_die(1, &trailer
, ZIP_DIR_TRAILER_SIZE
);
546 write_or_die(1, sha1_to_hex(sha1
), GIT_SHA1_HEXSZ
);
549 static void dos_time(time_t *time
, int *dos_date
, int *dos_time
)
551 struct tm
*t
= localtime(time
);
553 *dos_date
= t
->tm_mday
+ (t
->tm_mon
+ 1) * 32 +
554 (t
->tm_year
+ 1900 - 1980) * 512;
555 *dos_time
= t
->tm_sec
/ 2 + t
->tm_min
* 32 + t
->tm_hour
* 2048;
558 static int archive_zip_config(const char *var
, const char *value
, void *data
)
560 return userdiff_config(var
, value
);
563 static int write_zip_archive(const struct archiver
*ar
,
564 struct archiver_args
*args
)
568 git_config(archive_zip_config
, NULL
);
570 dos_time(&args
->time
, &zip_date
, &zip_time
);
572 zip_dir
= xmalloc(ZIP_DIRECTORY_MIN_SIZE
);
573 zip_dir_size
= ZIP_DIRECTORY_MIN_SIZE
;
575 err
= write_archive_entries(args
, write_zip_entry
);
577 write_zip_trailer(args
->commit_sha1
);
584 static struct archiver zip_archiver
= {
587 ARCHIVER_WANT_COMPRESSION_LEVELS
|ARCHIVER_REMOTE
590 void init_zip_archiver(void)
592 register_archiver(&zip_archiver
);