2 * GIT - The information manager from hell
4 * Copyright (C) Linus Torvalds, 2005
6 * This handles basic git sha1 object files - packing, unpacking,
18 #if defined(__linux__) && (defined(__i386__) || defined(__PPC__))
19 #define O_NOATIME 01000000
25 const unsigned char null_sha1
[20] = { 0, };
27 static unsigned int sha1_file_open_flag
= O_NOATIME
;
29 static unsigned hexval(char c
)
31 if (c
>= '0' && c
<= '9')
33 if (c
>= 'a' && c
<= 'f')
35 if (c
>= 'A' && c
<= 'F')
40 int get_sha1_hex(const char *hex
, unsigned char *sha1
)
43 for (i
= 0; i
< 20; i
++) {
44 unsigned int val
= (hexval(hex
[0]) << 4) | hexval(hex
[1]);
53 int adjust_shared_perm(const char *path
)
58 if (!shared_repository
)
60 if (lstat(path
, &st
) < 0)
71 if (chmod(path
, mode
) < 0)
76 int safe_create_leading_directories(char *path
)
85 pos
= strchr(pos
, '/');
89 if (!stat(path
, &st
)) {
91 if (!S_ISDIR(st
.st_mode
)) {
96 else if (mkdir(path
, 0777)) {
100 else if (adjust_shared_perm(path
)) {
109 char * sha1_to_hex(const unsigned char *sha1
)
112 static char hexbuffer
[4][50];
113 static const char hex
[] = "0123456789abcdef";
114 char *buffer
= hexbuffer
[3 & ++bufno
], *buf
= buffer
;
117 for (i
= 0; i
< 20; i
++) {
118 unsigned int val
= *sha1
++;
119 *buf
++ = hex
[val
>> 4];
120 *buf
++ = hex
[val
& 0xf];
127 static void fill_sha1_path(char *pathbuf
, const unsigned char *sha1
)
130 for (i
= 0; i
< 20; i
++) {
131 static char hex
[] = "0123456789abcdef";
132 unsigned int val
= sha1
[i
];
133 char *pos
= pathbuf
+ i
*2 + (i
> 0);
134 *pos
++ = hex
[val
>> 4];
135 *pos
= hex
[val
& 0xf];
140 * NOTE! This returns a statically allocated buffer, so you have to be
141 * careful about using it. Do a "strdup()" if you need to save the
144 * Also note that this returns the location for creating. Reading
145 * SHA1 file can happen from any alternate directory listed in the
146 * DB_ENVIRONMENT environment variable if it is not found in
147 * the primary object database.
149 char *sha1_file_name(const unsigned char *sha1
)
151 static char *name
, *base
;
154 const char *sha1_file_directory
= get_object_directory();
155 int len
= strlen(sha1_file_directory
);
156 base
= xmalloc(len
+ 60);
157 memcpy(base
, sha1_file_directory
, len
);
158 memset(base
+len
, 0, 60);
161 name
= base
+ len
+ 1;
163 fill_sha1_path(name
, sha1
);
167 char *sha1_pack_name(const unsigned char *sha1
)
169 static const char hex
[] = "0123456789abcdef";
170 static char *name
, *base
, *buf
;
174 const char *sha1_file_directory
= get_object_directory();
175 int len
= strlen(sha1_file_directory
);
176 base
= xmalloc(len
+ 60);
177 sprintf(base
, "%s/pack/pack-1234567890123456789012345678901234567890.pack", sha1_file_directory
);
178 name
= base
+ len
+ 11;
183 for (i
= 0; i
< 20; i
++) {
184 unsigned int val
= *sha1
++;
185 *buf
++ = hex
[val
>> 4];
186 *buf
++ = hex
[val
& 0xf];
192 char *sha1_pack_index_name(const unsigned char *sha1
)
194 static const char hex
[] = "0123456789abcdef";
195 static char *name
, *base
, *buf
;
199 const char *sha1_file_directory
= get_object_directory();
200 int len
= strlen(sha1_file_directory
);
201 base
= xmalloc(len
+ 60);
202 sprintf(base
, "%s/pack/pack-1234567890123456789012345678901234567890.idx", sha1_file_directory
);
203 name
= base
+ len
+ 11;
208 for (i
= 0; i
< 20; i
++) {
209 unsigned int val
= *sha1
++;
210 *buf
++ = hex
[val
>> 4];
211 *buf
++ = hex
[val
& 0xf];
217 struct alternate_object_database
*alt_odb_list
;
218 static struct alternate_object_database
**alt_odb_tail
;
220 static void read_info_alternates(const char * alternates
, int depth
);
223 * Prepare alternate object database registry.
225 * The variable alt_odb_list points at the list of struct
226 * alternate_object_database. The elements on this list come from
227 * non-empty elements from colon separated ALTERNATE_DB_ENVIRONMENT
228 * environment variable, and $GIT_OBJECT_DIRECTORY/info/alternates,
229 * whose contents is similar to that environment variable but can be
230 * LF separated. Its base points at a statically allocated buffer that
231 * contains "/the/directory/corresponding/to/.git/objects/...", while
232 * its name points just after the slash at the end of ".git/objects/"
233 * in the example above, and has enough space to hold 40-byte hex
234 * SHA1, an extra slash for the first level indirection, and the
237 static int link_alt_odb_entry(const char * entry
, int len
, const char * relative_base
, int depth
)
240 const char *objdir
= get_object_directory();
241 struct alternate_object_database
*ent
;
242 struct alternate_object_database
*alt
;
243 /* 43 = 40-byte + 2 '/' + terminating NUL */
245 int entlen
= pfxlen
+ 43;
248 if (*entry
!= '/' && relative_base
) {
249 /* Relative alt-odb */
251 base_len
= strlen(relative_base
) + 1;
255 ent
= xmalloc(sizeof(*ent
) + entlen
);
257 if (*entry
!= '/' && relative_base
) {
258 memcpy(ent
->base
, relative_base
, base_len
- 1);
259 ent
->base
[base_len
- 1] = '/';
260 memcpy(ent
->base
+ base_len
, entry
, len
);
263 memcpy(ent
->base
, entry
, pfxlen
);
265 ent
->name
= ent
->base
+ pfxlen
+ 1;
266 ent
->base
[pfxlen
+ 3] = '/';
267 ent
->base
[pfxlen
] = ent
->base
[entlen
-1] = 0;
269 /* Detect cases where alternate disappeared */
270 if (stat(ent
->base
, &st
) || !S_ISDIR(st
.st_mode
)) {
271 error("object directory %s does not exist; "
272 "check .git/objects/info/alternates.",
278 /* Prevent the common mistake of listing the same
279 * thing twice, or object directory itself.
281 for (alt
= alt_odb_list
; alt
; alt
= alt
->next
) {
282 if (!memcmp(ent
->base
, alt
->base
, pfxlen
)) {
287 if (!memcmp(ent
->base
, objdir
, pfxlen
)) {
292 /* add the alternate entry */
294 alt_odb_tail
= &(ent
->next
);
297 /* recursively add alternates */
298 read_info_alternates(ent
->base
, depth
+ 1);
300 ent
->base
[pfxlen
] = '/';
305 static void link_alt_odb_entries(const char *alt
, const char *ep
, int sep
,
306 const char *relative_base
, int depth
)
308 const char *cp
, *last
;
311 error("%s: ignoring alternate object stores, nesting too deep.",
319 if (cp
< ep
&& *cp
== '#') {
320 while (cp
< ep
&& *cp
!= sep
)
325 while (cp
< ep
&& *cp
!= sep
)
328 if ((*last
!= '/') && depth
) {
329 error("%s: ignoring relative alternate object store %s",
330 relative_base
, last
);
332 link_alt_odb_entry(last
, cp
- last
,
333 relative_base
, depth
);
336 while (cp
< ep
&& *cp
== sep
)
342 static void read_info_alternates(const char * relative_base
, int depth
)
349 sprintf(path
, "%s/info/alternates", relative_base
);
350 fd
= open(path
, O_RDONLY
);
353 if (fstat(fd
, &st
) || (st
.st_size
== 0)) {
357 map
= mmap(NULL
, st
.st_size
, PROT_READ
, MAP_PRIVATE
, fd
, 0);
359 if (map
== MAP_FAILED
)
362 link_alt_odb_entries(map
, map
+ st
.st_size
, '\n', relative_base
, depth
);
364 munmap(map
, st
.st_size
);
367 void prepare_alt_odb(void)
371 alt
= getenv(ALTERNATE_DB_ENVIRONMENT
);
376 alt_odb_tail
= &alt_odb_list
;
377 link_alt_odb_entries(alt
, alt
+ strlen(alt
), ':', NULL
, 0);
379 read_info_alternates(get_object_directory(), 0);
382 static char *find_sha1_file(const unsigned char *sha1
, struct stat
*st
)
384 char *name
= sha1_file_name(sha1
);
385 struct alternate_object_database
*alt
;
390 for (alt
= alt_odb_list
; alt
; alt
= alt
->next
) {
392 fill_sha1_path(name
, sha1
);
393 if (!stat(alt
->base
, st
))
399 #define PACK_MAX_SZ (1<<26)
400 static int pack_used_ctr
;
401 static unsigned long pack_mapped
;
402 struct packed_git
*packed_git
;
404 static int check_packed_git_idx(const char *path
, unsigned long *idx_size_
,
409 unsigned long idx_size
;
411 int fd
= open(path
, O_RDONLY
);
415 if (fstat(fd
, &st
)) {
419 idx_size
= st
.st_size
;
420 idx_map
= mmap(NULL
, idx_size
, PROT_READ
, MAP_PRIVATE
, fd
, 0);
422 if (idx_map
== MAP_FAILED
)
427 *idx_size_
= idx_size
;
429 /* check index map */
430 if (idx_size
< 4*256 + 20 + 20)
431 return error("index file too small");
433 for (i
= 0; i
< 256; i
++) {
434 unsigned int n
= ntohl(index
[i
]);
436 return error("non-monotonic index");
442 * - 256 index entries 4 bytes each
443 * - 24-byte entries * nr (20-byte sha1 + 4-byte offset)
444 * - 20-byte SHA1 of the packfile
445 * - 20-byte SHA1 file checksum
447 if (idx_size
!= 4*256 + nr
* 24 + 20 + 20)
448 return error("wrong index file size");
453 static int unuse_one_packed_git(void)
455 struct packed_git
*p
, *lru
= NULL
;
457 for (p
= packed_git
; p
; p
= p
->next
) {
458 if (p
->pack_use_cnt
|| !p
->pack_base
)
460 if (!lru
|| p
->pack_last_used
< lru
->pack_last_used
)
465 munmap(lru
->pack_base
, lru
->pack_size
);
466 lru
->pack_base
= NULL
;
470 void unuse_packed_git(struct packed_git
*p
)
475 int use_packed_git(struct packed_git
*p
)
479 // We created the struct before we had the pack
480 stat(p
->pack_name
, &st
);
481 if (!S_ISREG(st
.st_mode
))
482 die("packfile %s not a regular file", p
->pack_name
);
483 p
->pack_size
= st
.st_size
;
490 pack_mapped
+= p
->pack_size
;
491 while (PACK_MAX_SZ
< pack_mapped
&& unuse_one_packed_git())
493 fd
= open(p
->pack_name
, O_RDONLY
);
495 die("packfile %s cannot be opened", p
->pack_name
);
496 if (fstat(fd
, &st
)) {
498 die("packfile %s cannot be opened", p
->pack_name
);
500 if (st
.st_size
!= p
->pack_size
)
501 die("packfile %s size mismatch.", p
->pack_name
);
502 map
= mmap(NULL
, p
->pack_size
, PROT_READ
, MAP_PRIVATE
, fd
, 0);
504 if (map
== MAP_FAILED
)
505 die("packfile %s cannot be mapped.", p
->pack_name
);
508 /* Check if the pack file matches with the index file.
511 if (memcmp((char*)(p
->index_base
) + p
->index_size
- 40,
512 p
->pack_base
+ p
->pack_size
- 20, 20)) {
514 die("packfile %s does not match index.", p
->pack_name
);
517 p
->pack_last_used
= pack_used_ctr
++;
522 struct packed_git
*add_packed_git(char *path
, int path_len
, int local
)
525 struct packed_git
*p
;
526 unsigned long idx_size
;
528 unsigned char sha1
[20];
530 if (check_packed_git_idx(path
, &idx_size
, &idx_map
))
533 /* do we have a corresponding .pack file? */
534 strcpy(path
+ path_len
- 4, ".pack");
535 if (stat(path
, &st
) || !S_ISREG(st
.st_mode
)) {
536 munmap(idx_map
, idx_size
);
539 /* ok, it looks sane as far as we can check without
540 * actually mapping the pack file.
542 p
= xmalloc(sizeof(*p
) + path_len
+ 2);
543 strcpy(p
->pack_name
, path
);
544 p
->index_size
= idx_size
;
545 p
->pack_size
= st
.st_size
;
546 p
->index_base
= idx_map
;
549 p
->pack_last_used
= 0;
551 p
->pack_local
= local
;
552 if ((path_len
> 44) && !get_sha1_hex(path
+ path_len
- 44, sha1
))
553 memcpy(p
->sha1
, sha1
, 20);
557 struct packed_git
*parse_pack_index(unsigned char *sha1
)
559 char *path
= sha1_pack_index_name(sha1
);
560 return parse_pack_index_file(sha1
, path
);
563 struct packed_git
*parse_pack_index_file(const unsigned char *sha1
, char *idx_path
)
565 struct packed_git
*p
;
566 unsigned long idx_size
;
570 if (check_packed_git_idx(idx_path
, &idx_size
, &idx_map
))
573 path
= sha1_pack_name(sha1
);
575 p
= xmalloc(sizeof(*p
) + strlen(path
) + 2);
576 strcpy(p
->pack_name
, path
);
577 p
->index_size
= idx_size
;
579 p
->index_base
= idx_map
;
582 p
->pack_last_used
= 0;
584 memcpy(p
->sha1
, sha1
, 20);
588 void install_packed_git(struct packed_git
*pack
)
590 pack
->next
= packed_git
;
594 static void prepare_packed_git_one(char *objdir
, int local
)
601 sprintf(path
, "%s/pack", objdir
);
606 error("unable to open object pack directory: %s: %s",
607 path
, strerror(errno
));
611 while ((de
= readdir(dir
)) != NULL
) {
612 int namelen
= strlen(de
->d_name
);
613 struct packed_git
*p
;
615 if (strcmp(de
->d_name
+ namelen
- 4, ".idx"))
618 /* we have .idx. Is it a file we can map? */
619 strcpy(path
+ len
, de
->d_name
);
620 p
= add_packed_git(path
, len
+ namelen
, local
);
623 p
->next
= packed_git
;
629 static int prepare_packed_git_run_once
= 0;
630 void prepare_packed_git(void)
632 struct alternate_object_database
*alt
;
634 if (prepare_packed_git_run_once
)
636 prepare_packed_git_one(get_object_directory(), 1);
638 for (alt
= alt_odb_list
; alt
; alt
= alt
->next
) {
640 prepare_packed_git_one(alt
->base
, 0);
643 prepare_packed_git_run_once
= 1;
646 static void reprepare_packed_git(void)
648 prepare_packed_git_run_once
= 0;
649 prepare_packed_git();
652 int check_sha1_signature(const unsigned char *sha1
, void *map
, unsigned long size
, const char *type
)
655 unsigned char real_sha1
[20];
659 SHA1_Update(&c
, header
, 1+sprintf(header
, "%s %lu", type
, size
));
660 SHA1_Update(&c
, map
, size
);
661 SHA1_Final(real_sha1
, &c
);
662 return memcmp(sha1
, real_sha1
, 20) ? -1 : 0;
665 static void *map_sha1_file_internal(const unsigned char *sha1
,
671 char *filename
= find_sha1_file(sha1
, &st
);
677 fd
= open(filename
, O_RDONLY
| sha1_file_open_flag
);
679 /* See if it works without O_NOATIME */
680 switch (sha1_file_open_flag
) {
682 fd
= open(filename
, O_RDONLY
);
690 /* If it failed once, it will probably fail again.
691 * Stop using O_NOATIME
693 sha1_file_open_flag
= 0;
695 map
= mmap(NULL
, st
.st_size
, PROT_READ
, MAP_PRIVATE
, fd
, 0);
697 if (map
== MAP_FAILED
)
703 int unpack_sha1_header(z_stream
*stream
, void *map
, unsigned long mapsize
, void *buffer
, unsigned long size
)
705 /* Get the data stream */
706 memset(stream
, 0, sizeof(*stream
));
707 stream
->next_in
= map
;
708 stream
->avail_in
= mapsize
;
709 stream
->next_out
= buffer
;
710 stream
->avail_out
= size
;
713 return inflate(stream
, 0);
716 static void *unpack_sha1_rest(z_stream
*stream
, void *buffer
, unsigned long size
)
718 int bytes
= strlen(buffer
) + 1;
719 unsigned char *buf
= xmalloc(1+size
);
721 memcpy(buf
, buffer
+ bytes
, stream
->total_out
- bytes
);
722 bytes
= stream
->total_out
- bytes
;
724 stream
->next_out
= buf
+ bytes
;
725 stream
->avail_out
= size
- bytes
;
726 while (inflate(stream
, Z_FINISH
) == Z_OK
)
735 * We used to just use "sscanf()", but that's actually way
736 * too permissive for what we want to check. So do an anal
737 * object header parse by hand.
739 int parse_sha1_header(char *hdr
, char *type
, unsigned long *sizep
)
745 * The type can be at most ten bytes (including the
746 * terminating '\0' that we add), and is followed by
761 * The length must follow immediately, and be in canonical
762 * decimal format (ie "010" is not valid).
769 unsigned long c
= *hdr
- '0';
773 size
= size
* 10 + c
;
779 * The length must be followed by a zero byte
781 return *hdr
? -1 : 0;
784 void * unpack_sha1_file(void *map
, unsigned long mapsize
, char *type
, unsigned long *size
)
790 ret
= unpack_sha1_header(&stream
, map
, mapsize
, hdr
, sizeof(hdr
));
791 if (ret
< Z_OK
|| parse_sha1_header(hdr
, type
, size
) < 0)
794 return unpack_sha1_rest(&stream
, hdr
, *size
);
797 /* forward declaration for a mutually recursive function */
798 static int packed_object_info(struct pack_entry
*entry
,
799 char *type
, unsigned long *sizep
);
801 static int packed_delta_info(unsigned char *base_sha1
,
802 unsigned long delta_size
,
805 unsigned long *sizep
,
806 struct packed_git
*p
)
808 struct pack_entry base_ent
;
811 die("truncated pack file");
813 /* The base entry _must_ be in the same pack */
814 if (!find_pack_entry_one(base_sha1
, &base_ent
, p
))
815 die("failed to find delta-pack base object %s",
816 sha1_to_hex(base_sha1
));
818 /* We choose to only get the type of the base object and
819 * ignore potentially corrupt pack file that expects the delta
820 * based on a base with a wrong size. This saves tons of
824 if (packed_object_info(&base_ent
, type
, NULL
))
825 die("cannot get info for delta-pack base");
828 const unsigned char *data
;
829 unsigned char delta_head
[64];
830 unsigned long result_size
;
834 memset(&stream
, 0, sizeof(stream
));
836 data
= stream
.next_in
= base_sha1
+ 20;
837 stream
.avail_in
= left
- 20;
838 stream
.next_out
= delta_head
;
839 stream
.avail_out
= sizeof(delta_head
);
841 inflateInit(&stream
);
842 st
= inflate(&stream
, Z_FINISH
);
844 if ((st
!= Z_STREAM_END
) &&
845 stream
.total_out
!= sizeof(delta_head
))
846 die("delta data unpack-initial failed");
848 /* Examine the initial part of the delta to figure out
853 /* ignore base size */
854 get_delta_hdr_size(&data
, delta_head
+sizeof(delta_head
));
856 /* Read the result size */
857 result_size
= get_delta_hdr_size(&data
, delta_head
+sizeof(delta_head
));
858 *sizep
= result_size
;
863 static unsigned long unpack_object_header(struct packed_git
*p
, unsigned long offset
,
864 enum object_type
*type
, unsigned long *sizep
)
867 unsigned char *pack
, c
;
870 if (offset
>= p
->pack_size
)
871 die("object offset outside of pack file");
873 pack
= p
->pack_base
+ offset
;
876 *type
= (c
>> 4) & 7;
880 if (offset
>= p
->pack_size
)
881 die("object offset outside of pack file");
884 size
+= (c
& 0x7f) << shift
;
891 int check_reuse_pack_delta(struct packed_git
*p
, unsigned long offset
,
892 unsigned char *base
, unsigned long *sizep
,
893 enum object_type
*kindp
)
900 ptr
= unpack_object_header(p
, ptr
, kindp
, sizep
);
901 if (*kindp
!= OBJ_DELTA
)
903 memcpy(base
, p
->pack_base
+ ptr
, 20);
910 void packed_object_info_detail(struct pack_entry
*e
,
913 unsigned long *store_size
,
914 unsigned int *delta_chain_length
,
915 unsigned char *base_sha1
)
917 struct packed_git
*p
= e
->p
;
918 unsigned long offset
;
920 enum object_type kind
;
922 offset
= unpack_object_header(p
, e
->offset
, &kind
, size
);
923 pack
= p
->pack_base
+ offset
;
924 if (kind
!= OBJ_DELTA
)
925 *delta_chain_length
= 0;
927 unsigned int chain_length
= 0;
928 if (p
->pack_size
<= offset
+ 20)
929 die("pack file %s records an incomplete delta base",
931 memcpy(base_sha1
, pack
, 20);
933 struct pack_entry base_ent
;
936 find_pack_entry_one(pack
, &base_ent
, p
);
937 offset
= unpack_object_header(p
, base_ent
.offset
,
939 pack
= p
->pack_base
+ offset
;
941 } while (kind
== OBJ_DELTA
);
942 *delta_chain_length
= chain_length
;
946 strcpy(type
, commit_type
);
949 strcpy(type
, tree_type
);
952 strcpy(type
, blob_type
);
955 strcpy(type
, tag_type
);
958 die("corrupted pack file %s containing object of kind %d",
961 *store_size
= 0; /* notyet */
964 static int packed_object_info(struct pack_entry
*entry
,
965 char *type
, unsigned long *sizep
)
967 struct packed_git
*p
= entry
->p
;
968 unsigned long offset
, size
, left
;
970 enum object_type kind
;
973 if (use_packed_git(p
))
974 die("cannot map packed file");
976 offset
= unpack_object_header(p
, entry
->offset
, &kind
, &size
);
977 pack
= p
->pack_base
+ offset
;
978 left
= p
->pack_size
- offset
;
982 retval
= packed_delta_info(pack
, size
, left
, type
, sizep
, p
);
986 strcpy(type
, commit_type
);
989 strcpy(type
, tree_type
);
992 strcpy(type
, blob_type
);
995 strcpy(type
, tag_type
);
998 die("corrupted pack file %s containing object of kind %d",
1003 unuse_packed_git(p
);
1007 /* forward declaration for a mutually recursive function */
1008 static void *unpack_entry(struct pack_entry
*, char *, unsigned long *);
1010 static void *unpack_delta_entry(unsigned char *base_sha1
,
1011 unsigned long delta_size
,
1014 unsigned long *sizep
,
1015 struct packed_git
*p
)
1017 struct pack_entry base_ent
;
1018 void *data
, *delta_data
, *result
, *base
;
1019 unsigned long data_size
, result_size
, base_size
;
1024 die("truncated pack file");
1026 /* The base entry _must_ be in the same pack */
1027 if (!find_pack_entry_one(base_sha1
, &base_ent
, p
))
1028 die("failed to find delta-pack base object %s",
1029 sha1_to_hex(base_sha1
));
1030 base
= unpack_entry_gently(&base_ent
, type
, &base_size
);
1032 die("failed to read delta-pack base object %s",
1033 sha1_to_hex(base_sha1
));
1035 data
= base_sha1
+ 20;
1036 data_size
= left
- 20;
1037 delta_data
= xmalloc(delta_size
);
1039 memset(&stream
, 0, sizeof(stream
));
1041 stream
.next_in
= data
;
1042 stream
.avail_in
= data_size
;
1043 stream
.next_out
= delta_data
;
1044 stream
.avail_out
= delta_size
;
1046 inflateInit(&stream
);
1047 st
= inflate(&stream
, Z_FINISH
);
1048 inflateEnd(&stream
);
1049 if ((st
!= Z_STREAM_END
) || stream
.total_out
!= delta_size
)
1050 die("delta data unpack failed");
1052 result
= patch_delta(base
, base_size
,
1053 delta_data
, delta_size
,
1056 die("failed to apply delta");
1059 *sizep
= result_size
;
1063 static void *unpack_non_delta_entry(unsigned char *data
,
1069 unsigned char *buffer
;
1071 buffer
= xmalloc(size
+ 1);
1073 memset(&stream
, 0, sizeof(stream
));
1074 stream
.next_in
= data
;
1075 stream
.avail_in
= left
;
1076 stream
.next_out
= buffer
;
1077 stream
.avail_out
= size
;
1079 inflateInit(&stream
);
1080 st
= inflate(&stream
, Z_FINISH
);
1081 inflateEnd(&stream
);
1082 if ((st
!= Z_STREAM_END
) || stream
.total_out
!= size
) {
1090 static void *unpack_entry(struct pack_entry
*entry
,
1091 char *type
, unsigned long *sizep
)
1093 struct packed_git
*p
= entry
->p
;
1096 if (use_packed_git(p
))
1097 die("cannot map packed file");
1098 retval
= unpack_entry_gently(entry
, type
, sizep
);
1099 unuse_packed_git(p
);
1101 die("corrupted pack file %s", p
->pack_name
);
1105 /* The caller is responsible for use_packed_git()/unuse_packed_git() pair */
1106 void *unpack_entry_gently(struct pack_entry
*entry
,
1107 char *type
, unsigned long *sizep
)
1109 struct packed_git
*p
= entry
->p
;
1110 unsigned long offset
, size
, left
;
1111 unsigned char *pack
;
1112 enum object_type kind
;
1115 offset
= unpack_object_header(p
, entry
->offset
, &kind
, &size
);
1116 pack
= p
->pack_base
+ offset
;
1117 left
= p
->pack_size
- offset
;
1120 retval
= unpack_delta_entry(pack
, size
, left
, type
, sizep
, p
);
1123 strcpy(type
, commit_type
);
1126 strcpy(type
, tree_type
);
1129 strcpy(type
, blob_type
);
1132 strcpy(type
, tag_type
);
1138 retval
= unpack_non_delta_entry(pack
, size
, left
);
1142 int num_packed_objects(const struct packed_git
*p
)
1144 /* See check_packed_git_idx() */
1145 return (p
->index_size
- 20 - 20 - 4*256) / 24;
1148 int nth_packed_object_sha1(const struct packed_git
*p
, int n
,
1149 unsigned char* sha1
)
1151 void *index
= p
->index_base
+ 256;
1152 if (n
< 0 || num_packed_objects(p
) <= n
)
1154 memcpy(sha1
, (index
+ 24 * n
+ 4), 20);
1158 int find_pack_entry_one(const unsigned char *sha1
,
1159 struct pack_entry
*e
, struct packed_git
*p
)
1161 unsigned int *level1_ofs
= p
->index_base
;
1162 int hi
= ntohl(level1_ofs
[*sha1
]);
1163 int lo
= ((*sha1
== 0x0) ? 0 : ntohl(level1_ofs
[*sha1
- 1]));
1164 void *index
= p
->index_base
+ 256;
1167 int mi
= (lo
+ hi
) / 2;
1168 int cmp
= memcmp(index
+ 24 * mi
+ 4, sha1
, 20);
1170 e
->offset
= ntohl(*((unsigned int *)(index
+ 24 * mi
)));
1171 memcpy(e
->sha1
, sha1
, 20);
1183 static int find_pack_entry(const unsigned char *sha1
, struct pack_entry
*e
)
1185 struct packed_git
*p
;
1186 prepare_packed_git();
1188 for (p
= packed_git
; p
; p
= p
->next
) {
1189 if (find_pack_entry_one(sha1
, e
, p
))
1195 struct packed_git
*find_sha1_pack(const unsigned char *sha1
,
1196 struct packed_git
*packs
)
1198 struct packed_git
*p
;
1199 struct pack_entry e
;
1201 for (p
= packs
; p
; p
= p
->next
) {
1202 if (find_pack_entry_one(sha1
, &e
, p
))
1209 int sha1_object_info(const unsigned char *sha1
, char *type
, unsigned long *sizep
)
1212 unsigned long mapsize
, size
;
1217 map
= map_sha1_file_internal(sha1
, &mapsize
);
1219 struct pack_entry e
;
1221 if (find_pack_entry(sha1
, &e
))
1222 return packed_object_info(&e
, type
, sizep
);
1223 reprepare_packed_git();
1224 if (find_pack_entry(sha1
, &e
))
1225 return packed_object_info(&e
, type
, sizep
);
1226 return error("unable to find %s", sha1_to_hex(sha1
));
1228 if (unpack_sha1_header(&stream
, map
, mapsize
, hdr
, sizeof(hdr
)) < 0)
1229 status
= error("unable to unpack %s header",
1231 if (parse_sha1_header(hdr
, type
, &size
) < 0)
1232 status
= error("unable to parse %s header", sha1_to_hex(sha1
));
1238 inflateEnd(&stream
);
1239 munmap(map
, mapsize
);
1243 static void *read_packed_sha1(const unsigned char *sha1
, char *type
, unsigned long *size
)
1245 struct pack_entry e
;
1247 if (!find_pack_entry(sha1
, &e
)) {
1248 error("cannot read sha1_file for %s", sha1_to_hex(sha1
));
1251 return unpack_entry(&e
, type
, size
);
1254 void * read_sha1_file(const unsigned char *sha1
, char *type
, unsigned long *size
)
1256 unsigned long mapsize
;
1258 struct pack_entry e
;
1260 if (find_pack_entry(sha1
, &e
))
1261 return read_packed_sha1(sha1
, type
, size
);
1262 map
= map_sha1_file_internal(sha1
, &mapsize
);
1264 buf
= unpack_sha1_file(map
, mapsize
, type
, size
);
1265 munmap(map
, mapsize
);
1268 reprepare_packed_git();
1269 if (find_pack_entry(sha1
, &e
))
1270 return read_packed_sha1(sha1
, type
, size
);
1274 void *read_object_with_reference(const unsigned char *sha1
,
1275 const char *required_type
,
1276 unsigned long *size
,
1277 unsigned char *actual_sha1_return
)
1281 unsigned long isize
;
1282 unsigned char actual_sha1
[20];
1284 memcpy(actual_sha1
, sha1
, 20);
1286 int ref_length
= -1;
1287 const char *ref_type
= NULL
;
1289 buffer
= read_sha1_file(actual_sha1
, type
, &isize
);
1292 if (!strcmp(type
, required_type
)) {
1294 if (actual_sha1_return
)
1295 memcpy(actual_sha1_return
, actual_sha1
, 20);
1298 /* Handle references */
1299 else if (!strcmp(type
, commit_type
))
1301 else if (!strcmp(type
, tag_type
))
1302 ref_type
= "object ";
1307 ref_length
= strlen(ref_type
);
1309 if (memcmp(buffer
, ref_type
, ref_length
) ||
1310 get_sha1_hex(buffer
+ ref_length
, actual_sha1
)) {
1315 /* Now we have the ID of the referred-to object in
1316 * actual_sha1. Check again. */
1320 char *write_sha1_file_prepare(void *buf
,
1323 unsigned char *sha1
,
1329 /* Generate the header */
1330 *hdrlen
= sprintf((char *)hdr
, "%s %lu", type
, len
)+1;
1334 SHA1_Update(&c
, hdr
, *hdrlen
);
1335 SHA1_Update(&c
, buf
, len
);
1336 SHA1_Final(sha1
, &c
);
1338 return sha1_file_name(sha1
);
1342 * Link the tempfile to the final place, possibly creating the
1343 * last directory level as you do so.
1345 * Returns the errno on failure, 0 on success.
1347 static int link_temp_to_file(const char *tmpfile
, char *filename
)
1351 if (!link(tmpfile
, filename
))
1355 * Try to mkdir the last path component if that failed
1358 * Re-try the "link()" regardless of whether the mkdir
1359 * succeeds, since a race might mean that somebody
1363 if (ret
== ENOENT
) {
1364 char *dir
= strrchr(filename
, '/');
1367 mkdir(filename
, 0777);
1368 if (adjust_shared_perm(filename
))
1371 if (!link(tmpfile
, filename
))
1380 * Move the just written object into its final resting place
1382 int move_temp_to_file(const char *tmpfile
, char *filename
)
1384 int ret
= link_temp_to_file(tmpfile
, filename
);
1387 * Coda hack - coda doesn't like cross-directory links,
1388 * so we fall back to a rename, which will mean that it
1389 * won't be able to check collisions, but that's not a
1392 * The same holds for FAT formatted media.
1394 * When this succeeds, we just return 0. We have nothing
1397 if (ret
&& ret
!= EEXIST
) {
1398 if (!rename(tmpfile
, filename
))
1404 if (ret
!= EEXIST
) {
1405 fprintf(stderr
, "unable to write sha1 filename %s: %s\n", filename
, strerror(ret
));
1408 /* FIXME!!! Collision check here ? */
1414 static int write_buffer(int fd
, const void *buf
, size_t len
)
1419 size
= write(fd
, buf
, len
);
1421 return error("file write: disk full");
1423 if (errno
== EINTR
|| errno
== EAGAIN
)
1425 return error("file write error (%s)", strerror(errno
));
1433 int write_sha1_file(void *buf
, unsigned long len
, const char *type
, unsigned char *returnsha1
)
1436 unsigned char *compressed
;
1438 unsigned char sha1
[20];
1440 static char tmpfile
[PATH_MAX
];
1441 unsigned char hdr
[50];
1444 /* Normally if we have it in the pack then we do not bother writing
1445 * it out into .git/objects/??/?{38} file.
1447 filename
= write_sha1_file_prepare(buf
, len
, type
, sha1
, hdr
, &hdrlen
);
1449 memcpy(returnsha1
, sha1
, 20);
1450 if (has_sha1_file(sha1
))
1452 fd
= open(filename
, O_RDONLY
);
1455 * FIXME!!! We might do collision checking here, but we'd
1456 * need to uncompress the old file and check it. Later.
1462 if (errno
!= ENOENT
) {
1463 fprintf(stderr
, "sha1 file %s: %s\n", filename
, strerror(errno
));
1467 snprintf(tmpfile
, sizeof(tmpfile
), "%s/obj_XXXXXX", get_object_directory());
1469 fd
= mkstemp(tmpfile
);
1471 fprintf(stderr
, "unable to create temporary sha1 filename %s: %s\n", tmpfile
, strerror(errno
));
1476 memset(&stream
, 0, sizeof(stream
));
1477 deflateInit(&stream
, Z_BEST_COMPRESSION
);
1478 size
= deflateBound(&stream
, len
+hdrlen
);
1479 compressed
= xmalloc(size
);
1482 stream
.next_out
= compressed
;
1483 stream
.avail_out
= size
;
1485 /* First header.. */
1486 stream
.next_in
= hdr
;
1487 stream
.avail_in
= hdrlen
;
1488 while (deflate(&stream
, 0) == Z_OK
)
1491 /* Then the data itself.. */
1492 stream
.next_in
= buf
;
1493 stream
.avail_in
= len
;
1494 while (deflate(&stream
, Z_FINISH
) == Z_OK
)
1496 deflateEnd(&stream
);
1497 size
= stream
.total_out
;
1499 if (write_buffer(fd
, compressed
, size
) < 0)
1500 die("unable to write sha1 file");
1505 return move_temp_to_file(tmpfile
, filename
);
1509 * We need to unpack and recompress the object for writing
1510 * it out to a different file.
1512 static void *repack_object(const unsigned char *sha1
, unsigned long *objsize
)
1516 unsigned char *unpacked
;
1523 // need to unpack and recompress it by itself
1524 unpacked
= read_packed_sha1(sha1
, type
, &len
);
1526 hdrlen
= sprintf(hdr
, "%s %lu", type
, len
) + 1;
1529 memset(&stream
, 0, sizeof(stream
));
1530 deflateInit(&stream
, Z_BEST_COMPRESSION
);
1531 size
= deflateBound(&stream
, len
+ hdrlen
);
1532 buf
= xmalloc(size
);
1535 stream
.next_out
= buf
;
1536 stream
.avail_out
= size
;
1538 /* First header.. */
1539 stream
.next_in
= (void *)hdr
;
1540 stream
.avail_in
= hdrlen
;
1541 while (deflate(&stream
, 0) == Z_OK
)
1544 /* Then the data itself.. */
1545 stream
.next_in
= unpacked
;
1546 stream
.avail_in
= len
;
1547 while (deflate(&stream
, Z_FINISH
) == Z_OK
)
1549 deflateEnd(&stream
);
1552 *objsize
= stream
.total_out
;
1556 int write_sha1_to_fd(int fd
, const unsigned char *sha1
)
1559 unsigned long objsize
;
1560 void *buf
= map_sha1_file_internal(sha1
, &objsize
);
1563 retval
= write_buffer(fd
, buf
, objsize
);
1564 munmap(buf
, objsize
);
1568 buf
= repack_object(sha1
, &objsize
);
1569 retval
= write_buffer(fd
, buf
, objsize
);
1574 int write_sha1_from_fd(const unsigned char *sha1
, int fd
, char *buffer
,
1575 size_t bufsize
, size_t *bufposn
)
1577 char tmpfile
[PATH_MAX
];
1580 unsigned char real_sha1
[20];
1581 unsigned char discard
[4096];
1585 snprintf(tmpfile
, sizeof(tmpfile
), "%s/obj_XXXXXX", get_object_directory());
1587 local
= mkstemp(tmpfile
);
1589 return error("Couldn't open %s for %s",
1590 tmpfile
, sha1_to_hex(sha1
));
1592 memset(&stream
, 0, sizeof(stream
));
1594 inflateInit(&stream
);
1601 stream
.avail_in
= *bufposn
;
1602 stream
.next_in
= (unsigned char *) buffer
;
1604 stream
.next_out
= discard
;
1605 stream
.avail_out
= sizeof(discard
);
1606 ret
= inflate(&stream
, Z_SYNC_FLUSH
);
1607 SHA1_Update(&c
, discard
, sizeof(discard
) -
1609 } while (stream
.avail_in
&& ret
== Z_OK
);
1610 if (write_buffer(local
, buffer
, *bufposn
- stream
.avail_in
) < 0)
1611 die("unable to write sha1 file");
1612 memmove(buffer
, buffer
+ *bufposn
- stream
.avail_in
,
1614 *bufposn
= stream
.avail_in
;
1618 size
= read(fd
, buffer
+ *bufposn
, bufsize
- *bufposn
);
1623 return error("Connection closed?");
1624 perror("Reading from connection");
1629 inflateEnd(&stream
);
1632 SHA1_Final(real_sha1
, &c
);
1633 if (ret
!= Z_STREAM_END
) {
1635 return error("File %s corrupted", sha1_to_hex(sha1
));
1637 if (memcmp(sha1
, real_sha1
, 20)) {
1639 return error("File %s has bad hash", sha1_to_hex(sha1
));
1642 return move_temp_to_file(tmpfile
, sha1_file_name(sha1
));
1645 int has_pack_index(const unsigned char *sha1
)
1648 if (stat(sha1_pack_index_name(sha1
), &st
))
1653 int has_pack_file(const unsigned char *sha1
)
1656 if (stat(sha1_pack_name(sha1
), &st
))
1661 int has_sha1_pack(const unsigned char *sha1
)
1663 struct pack_entry e
;
1664 return find_pack_entry(sha1
, &e
);
1667 int has_sha1_file(const unsigned char *sha1
)
1670 struct pack_entry e
;
1672 if (find_pack_entry(sha1
, &e
))
1674 return find_sha1_file(sha1
, &st
) ? 1 : 0;
1678 * reads from fd as long as possible into a supplied buffer of size bytes.
1679 * If neccessary the buffer's size is increased using realloc()
1681 * returns 0 if anything went fine and -1 otherwise
1683 * NOTE: both buf and size may change, but even when -1 is returned
1684 * you still have to free() it yourself.
1686 int read_pipe(int fd
, char** return_buf
, unsigned long* return_size
)
1688 char* buf
= *return_buf
;
1689 unsigned long size
= *return_size
;
1691 unsigned long off
= 0;
1694 iret
= xread(fd
, buf
+ off
, size
- off
);
1699 buf
= realloc(buf
, size
);
1712 int index_pipe(unsigned char *sha1
, int fd
, const char *type
, int write_object
)
1714 unsigned long size
= 4096;
1715 char *buf
= malloc(size
);
1717 unsigned char hdr
[50];
1720 if (read_pipe(fd
, &buf
, &size
)) {
1728 ret
= write_sha1_file(buf
, size
, type
, sha1
);
1730 write_sha1_file_prepare(buf
, size
, type
, sha1
, hdr
, &hdrlen
);
1737 int index_fd(unsigned char *sha1
, int fd
, struct stat
*st
, int write_object
, const char *type
)
1739 unsigned long size
= st
->st_size
;
1742 unsigned char hdr
[50];
1747 buf
= mmap(NULL
, size
, PROT_READ
, MAP_PRIVATE
, fd
, 0);
1749 if (buf
== MAP_FAILED
)
1755 ret
= write_sha1_file(buf
, size
, type
, sha1
);
1757 write_sha1_file_prepare(buf
, size
, type
, sha1
, hdr
, &hdrlen
);
1765 int index_path(unsigned char *sha1
, const char *path
, struct stat
*st
, int write_object
)
1770 switch (st
->st_mode
& S_IFMT
) {
1772 fd
= open(path
, O_RDONLY
);
1774 return error("open(\"%s\"): %s", path
,
1776 if (index_fd(sha1
, fd
, st
, write_object
, NULL
) < 0)
1777 return error("%s: failed to insert into database",
1781 target
= xmalloc(st
->st_size
+1);
1782 if (readlink(path
, target
, st
->st_size
+1) != st
->st_size
) {
1783 char *errstr
= strerror(errno
);
1785 return error("readlink(\"%s\"): %s", path
,
1788 if (!write_object
) {
1789 unsigned char hdr
[50];
1791 write_sha1_file_prepare(target
, st
->st_size
, blob_type
,
1792 sha1
, hdr
, &hdrlen
);
1793 } else if (write_sha1_file(target
, st
->st_size
, blob_type
, sha1
))
1794 return error("%s: failed to insert into database",
1799 return error("%s: unsupported file type", path
);