2 Format of STDIN stream:
14 new_blob ::= 'blob' lf
17 file_content ::= data;
19 new_commit ::= 'commit' sp ref_str lf
21 ('author' sp name '<' email '>' when lf)?
22 'committer' sp name '<' email '>' when lf
24 ('from' sp (ref_str | hexsha1 | sha1exp_str | idnum) lf)?
25 ('merge' sp (ref_str | hexsha1 | sha1exp_str | idnum) lf)*
30 file_change ::= file_clr
36 file_clr ::= 'deleteall' lf;
37 file_del ::= 'D' sp path_str lf;
38 file_rnm ::= 'R' sp path_str sp path_str lf;
39 file_cpy ::= 'C' sp path_str sp path_str lf;
40 file_obm ::= 'M' sp mode sp (hexsha1 | idnum) sp path_str lf;
41 file_inm ::= 'M' sp mode sp 'inline' sp path_str lf
44 new_tag ::= 'tag' sp tag_str lf
45 'from' sp (ref_str | hexsha1 | sha1exp_str | idnum) lf
46 'tagger' sp name '<' email '>' when lf
50 reset_branch ::= 'reset' sp ref_str lf
51 ('from' sp (ref_str | hexsha1 | sha1exp_str | idnum) lf)?
54 checkpoint ::= 'checkpoint' lf
57 progress ::= 'progress' sp not_lf* lf
60 # note: the first idnum in a stream should be 1 and subsequent
61 # idnums should not have gaps between values as this will cause
62 # the stream parser to reserve space for the gapped values. An
63 # idnum can be updated in the future to a new object by issuing
64 # a new mark directive with the old idnum.
66 mark ::= 'mark' sp idnum lf;
67 data ::= (delimited_data | exact_data)
70 # note: delim may be any string but must not contain lf.
71 # data_line may contain any data but must not be exactly
73 delimited_data ::= 'data' sp '<<' delim lf
77 # note: declen indicates the length of binary_data in bytes.
78 # declen does not include the lf preceeding the binary data.
80 exact_data ::= 'data' sp declen lf
83 # note: quoted strings are C-style quoting supporting \c for
84 # common escapes of 'c' (e..g \n, \t, \\, \") or \nnn where nnn
85 # is the signed byte value in octal. Note that the only
86 # characters which must actually be escaped to protect the
87 # stream formatting is: \, " and LF. Otherwise these values
91 sha1exp_str ::= sha1exp;
93 path_str ::= path | '"' quoted(path) '"' ;
94 mode ::= '100644' | '644'
99 declen ::= # unsigned 32 bit value, ascii base10 notation;
100 bigint ::= # unsigned integer value, ascii base10 notation;
101 binary_data ::= # file content, not interpreted;
103 when ::= raw_when | rfc2822_when;
104 raw_when ::= ts sp tz;
105 rfc2822_when ::= # Valid RFC 2822 date and time;
107 sp ::= # ASCII space character;
108 lf ::= # ASCII newline (LF) character;
110 # note: a colon (':') must precede the numerical value assigned to
111 # an idnum. This is to distinguish it from a ref or tag name as
112 # GIT does not permit ':' in ref or tag strings.
114 idnum ::= ':' bigint;
115 path ::= # GIT style file path, e.g. "a/b/c";
116 ref ::= # GIT ref name, e.g. "refs/heads/MOZ_GECKO_EXPERIMENT";
117 tag ::= # GIT tag name, e.g. "FIREFOX_1_5";
118 sha1exp ::= # Any valid GIT SHA1 expression;
119 hexsha1 ::= # SHA1 in hexadecimal format;
121 # note: name and email are UTF8 strings, however name must not
122 # contain '<' or lf and email must not contain any of the
123 # following: '<', '>', lf.
125 name ::= # valid GIT author/committer name;
126 email ::= # valid GIT author/committer email;
127 ts ::= # time since the epoch in seconds, ascii base10 notation;
128 tz ::= # GIT style timezone;
130 # note: comments may appear anywhere in the input, except
131 # within a data command. Any form of the data command
132 # always escapes the related input from comment processing.
134 # In case it is not clear, the '#' that starts the comment
135 # must be the first character on that the line (an lf have
138 comment ::= '#' not_lf* lf;
139 not_lf ::= # Any byte that is not ASCII newline (LF);
151 #include "csum-file.h"
155 #define PACK_ID_BITS 16
156 #define MAX_PACK_ID ((1<<PACK_ID_BITS)-1)
160 struct object_entry
*next
;
162 unsigned type
: TYPE_BITS
;
163 unsigned pack_id
: PACK_ID_BITS
;
164 unsigned char sha1
[20];
167 struct object_entry_pool
169 struct object_entry_pool
*next_pool
;
170 struct object_entry
*next_free
;
171 struct object_entry
*end
;
172 struct object_entry entries
[FLEX_ARRAY
]; /* more */
178 struct object_entry
*marked
[1024];
179 struct mark_set
*sets
[1024];
195 struct mem_pool
*next_pool
;
198 char space
[FLEX_ARRAY
]; /* more */
203 struct atom_str
*next_atom
;
204 unsigned short str_len
;
205 char str_dat
[FLEX_ARRAY
]; /* more */
211 struct tree_content
*tree
;
212 struct atom_str
* name
;
216 unsigned char sha1
[20];
222 unsigned int entry_capacity
; /* must match avail_tree_content */
223 unsigned int entry_count
;
224 unsigned int delta_depth
;
225 struct tree_entry
*entries
[FLEX_ARRAY
]; /* more */
228 struct avail_tree_content
230 unsigned int entry_capacity
; /* must match tree_content */
231 struct avail_tree_content
*next_avail
;
236 struct branch
*table_next_branch
;
237 struct branch
*active_next_branch
;
239 struct tree_entry branch_tree
;
240 uintmax_t last_commit
;
242 unsigned pack_id
: PACK_ID_BITS
;
243 unsigned char sha1
[20];
248 struct tag
*next_tag
;
250 unsigned int pack_id
;
251 unsigned char sha1
[20];
262 struct hash_list
*next
;
263 unsigned char sha1
[20];
272 struct recent_command
274 struct recent_command
*prev
;
275 struct recent_command
*next
;
279 /* Configured limits on output */
280 static unsigned long max_depth
= 10;
281 static off_t max_packsize
= (1LL << 32) - 1;
282 static int force_update
;
284 /* Stats and misc. counters */
285 static uintmax_t alloc_count
;
286 static uintmax_t marks_set_count
;
287 static uintmax_t object_count_by_type
[1 << TYPE_BITS
];
288 static uintmax_t duplicate_count_by_type
[1 << TYPE_BITS
];
289 static uintmax_t delta_count_by_type
[1 << TYPE_BITS
];
290 static unsigned long object_count
;
291 static unsigned long branch_count
;
292 static unsigned long branch_load_count
;
294 static FILE *pack_edges
;
297 static size_t mem_pool_alloc
= 2*1024*1024 - sizeof(struct mem_pool
);
298 static size_t total_allocd
;
299 static struct mem_pool
*mem_pool
;
301 /* Atom management */
302 static unsigned int atom_table_sz
= 4451;
303 static unsigned int atom_cnt
;
304 static struct atom_str
**atom_table
;
306 /* The .pack file being generated */
307 static unsigned int pack_id
;
308 static struct packed_git
*pack_data
;
309 static struct packed_git
**all_packs
;
310 static unsigned long pack_size
;
312 /* Table of objects we've written. */
313 static unsigned int object_entry_alloc
= 5000;
314 static struct object_entry_pool
*blocks
;
315 static struct object_entry
*object_table
[1 << 16];
316 static struct mark_set
*marks
;
317 static const char* mark_file
;
320 static struct last_object last_blob
;
322 /* Tree management */
323 static unsigned int tree_entry_alloc
= 1000;
324 static void *avail_tree_entry
;
325 static unsigned int avail_tree_table_sz
= 100;
326 static struct avail_tree_content
**avail_tree_table
;
327 static struct dbuf old_tree
;
328 static struct dbuf new_tree
;
331 static unsigned long max_active_branches
= 5;
332 static unsigned long cur_active_branches
;
333 static unsigned long branch_table_sz
= 1039;
334 static struct branch
**branch_table
;
335 static struct branch
*active_branches
;
338 static struct tag
*first_tag
;
339 static struct tag
*last_tag
;
341 /* Input stream parsing */
342 static whenspec_type whenspec
= WHENSPEC_RAW
;
343 static struct strbuf command_buf
;
344 static int unread_command_buf
;
345 static struct recent_command cmd_hist
= {&cmd_hist
, &cmd_hist
, NULL
};
346 static struct recent_command
*cmd_tail
= &cmd_hist
;
347 static struct recent_command
*rc_free
;
348 static unsigned int cmd_save
= 100;
349 static uintmax_t next_mark
;
350 static struct dbuf new_data
;
352 static void write_branch_report(FILE *rpt
, struct branch
*b
)
354 fprintf(rpt
, "%s:\n", b
->name
);
356 fprintf(rpt
, " status :");
358 fputs(" active", rpt
);
359 if (b
->branch_tree
.tree
)
360 fputs(" loaded", rpt
);
361 if (is_null_sha1(b
->branch_tree
.versions
[1].sha1
))
362 fputs(" dirty", rpt
);
365 fprintf(rpt
, " tip commit : %s\n", sha1_to_hex(b
->sha1
));
366 fprintf(rpt
, " old tree : %s\n", sha1_to_hex(b
->branch_tree
.versions
[0].sha1
));
367 fprintf(rpt
, " cur tree : %s\n", sha1_to_hex(b
->branch_tree
.versions
[1].sha1
));
368 fprintf(rpt
, " commit clock: %" PRIuMAX
"\n", b
->last_commit
);
370 fputs(" last pack : ", rpt
);
371 if (b
->pack_id
< MAX_PACK_ID
)
372 fprintf(rpt
, "%u", b
->pack_id
);
378 static void write_crash_report(const char *err
, va_list params
)
380 char *loc
= git_path("fast_import_crash_%d", getpid());
381 FILE *rpt
= fopen(loc
, "w");
384 struct recent_command
*rc
;
387 error("can't write crash report %s: %s", loc
, strerror(errno
));
391 fprintf(stderr
, "fast-import: dumping crash report to %s\n", loc
);
393 fprintf(rpt
, "fast-import crash report:\n");
394 fprintf(rpt
, " fast-import process: %d\n", getpid());
395 fprintf(rpt
, " parent process : %d\n", getppid());
396 fprintf(rpt
, " at %s\n", show_date(time(NULL
), 0, DATE_LOCAL
));
399 fputs("fatal: ", rpt
);
400 vfprintf(rpt
, err
, params
);
404 fputs("Most Recent Commands Before Crash\n", rpt
);
405 fputs("---------------------------------\n", rpt
);
406 for (rc
= cmd_hist
.next
; rc
!= &cmd_hist
; rc
= rc
->next
) {
407 if (rc
->next
== &cmd_hist
)
416 fputs("Active Branch LRU\n", rpt
);
417 fputs("-----------------\n", rpt
);
418 fprintf(rpt
, " active_branches = %lu cur, %lu max\n",
420 max_active_branches
);
422 fputs(" pos clock name\n", rpt
);
423 fputs(" ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n", rpt
);
424 for (b
= active_branches
, lu
= 0; b
; b
= b
->active_next_branch
)
425 fprintf(rpt
, " %2lu) %6" PRIuMAX
" %s\n",
426 ++lu
, b
->last_commit
, b
->name
);
429 fputs("Inactive Branches\n", rpt
);
430 fputs("-----------------\n", rpt
);
431 for (lu
= 0; lu
< branch_table_sz
; lu
++) {
432 for (b
= branch_table
[lu
]; b
; b
= b
->table_next_branch
)
433 write_branch_report(rpt
, b
);
437 fputs("-------------------\n", rpt
);
438 fputs("END OF CRASH REPORT\n", rpt
);
442 static NORETURN
void die_nicely(const char *err
, va_list params
)
446 fputs("fatal: ", stderr
);
447 vfprintf(stderr
, err
, params
);
452 write_crash_report(err
, params
);
458 static void alloc_objects(unsigned int cnt
)
460 struct object_entry_pool
*b
;
462 b
= xmalloc(sizeof(struct object_entry_pool
)
463 + cnt
* sizeof(struct object_entry
));
464 b
->next_pool
= blocks
;
465 b
->next_free
= b
->entries
;
466 b
->end
= b
->entries
+ cnt
;
471 static struct object_entry
*new_object(unsigned char *sha1
)
473 struct object_entry
*e
;
475 if (blocks
->next_free
== blocks
->end
)
476 alloc_objects(object_entry_alloc
);
478 e
= blocks
->next_free
++;
479 hashcpy(e
->sha1
, sha1
);
483 static struct object_entry
*find_object(unsigned char *sha1
)
485 unsigned int h
= sha1
[0] << 8 | sha1
[1];
486 struct object_entry
*e
;
487 for (e
= object_table
[h
]; e
; e
= e
->next
)
488 if (!hashcmp(sha1
, e
->sha1
))
493 static struct object_entry
*insert_object(unsigned char *sha1
)
495 unsigned int h
= sha1
[0] << 8 | sha1
[1];
496 struct object_entry
*e
= object_table
[h
];
497 struct object_entry
*p
= NULL
;
500 if (!hashcmp(sha1
, e
->sha1
))
506 e
= new_object(sha1
);
516 static unsigned int hc_str(const char *s
, size_t len
)
524 static void *pool_alloc(size_t len
)
529 for (p
= mem_pool
; p
; p
= p
->next_pool
)
530 if ((p
->end
- p
->next_free
>= len
))
534 if (len
>= (mem_pool_alloc
/2)) {
538 total_allocd
+= sizeof(struct mem_pool
) + mem_pool_alloc
;
539 p
= xmalloc(sizeof(struct mem_pool
) + mem_pool_alloc
);
540 p
->next_pool
= mem_pool
;
541 p
->next_free
= p
->space
;
542 p
->end
= p
->next_free
+ mem_pool_alloc
;
547 /* round out to a pointer alignment */
548 if (len
& (sizeof(void*) - 1))
549 len
+= sizeof(void*) - (len
& (sizeof(void*) - 1));
554 static void *pool_calloc(size_t count
, size_t size
)
556 size_t len
= count
* size
;
557 void *r
= pool_alloc(len
);
562 static char *pool_strdup(const char *s
)
564 char *r
= pool_alloc(strlen(s
) + 1);
569 static void size_dbuf(struct dbuf
*b
, size_t maxlen
)
572 if (b
->capacity
>= maxlen
)
576 b
->capacity
= ((maxlen
/ 1024) + 1) * 1024;
577 b
->buffer
= xmalloc(b
->capacity
);
580 static void insert_mark(uintmax_t idnum
, struct object_entry
*oe
)
582 struct mark_set
*s
= marks
;
583 while ((idnum
>> s
->shift
) >= 1024) {
584 s
= pool_calloc(1, sizeof(struct mark_set
));
585 s
->shift
= marks
->shift
+ 10;
586 s
->data
.sets
[0] = marks
;
590 uintmax_t i
= idnum
>> s
->shift
;
591 idnum
-= i
<< s
->shift
;
592 if (!s
->data
.sets
[i
]) {
593 s
->data
.sets
[i
] = pool_calloc(1, sizeof(struct mark_set
));
594 s
->data
.sets
[i
]->shift
= s
->shift
- 10;
598 if (!s
->data
.marked
[idnum
])
600 s
->data
.marked
[idnum
] = oe
;
603 static struct object_entry
*find_mark(uintmax_t idnum
)
605 uintmax_t orig_idnum
= idnum
;
606 struct mark_set
*s
= marks
;
607 struct object_entry
*oe
= NULL
;
608 if ((idnum
>> s
->shift
) < 1024) {
609 while (s
&& s
->shift
) {
610 uintmax_t i
= idnum
>> s
->shift
;
611 idnum
-= i
<< s
->shift
;
615 oe
= s
->data
.marked
[idnum
];
618 die("mark :%" PRIuMAX
" not declared", orig_idnum
);
622 static struct atom_str
*to_atom(const char *s
, unsigned short len
)
624 unsigned int hc
= hc_str(s
, len
) % atom_table_sz
;
627 for (c
= atom_table
[hc
]; c
; c
= c
->next_atom
)
628 if (c
->str_len
== len
&& !strncmp(s
, c
->str_dat
, len
))
631 c
= pool_alloc(sizeof(struct atom_str
) + len
+ 1);
633 strncpy(c
->str_dat
, s
, len
);
635 c
->next_atom
= atom_table
[hc
];
641 static struct branch
*lookup_branch(const char *name
)
643 unsigned int hc
= hc_str(name
, strlen(name
)) % branch_table_sz
;
646 for (b
= branch_table
[hc
]; b
; b
= b
->table_next_branch
)
647 if (!strcmp(name
, b
->name
))
652 static struct branch
*new_branch(const char *name
)
654 unsigned int hc
= hc_str(name
, strlen(name
)) % branch_table_sz
;
655 struct branch
* b
= lookup_branch(name
);
658 die("Invalid attempt to create duplicate branch: %s", name
);
659 switch (check_ref_format(name
)) {
660 case 0: break; /* its valid */
661 case -2: break; /* valid, but too few '/', allow anyway */
663 die("Branch name doesn't conform to GIT standards: %s", name
);
666 b
= pool_calloc(1, sizeof(struct branch
));
667 b
->name
= pool_strdup(name
);
668 b
->table_next_branch
= branch_table
[hc
];
669 b
->branch_tree
.versions
[0].mode
= S_IFDIR
;
670 b
->branch_tree
.versions
[1].mode
= S_IFDIR
;
672 b
->pack_id
= MAX_PACK_ID
;
673 branch_table
[hc
] = b
;
678 static unsigned int hc_entries(unsigned int cnt
)
680 cnt
= cnt
& 7 ? (cnt
/ 8) + 1 : cnt
/ 8;
681 return cnt
< avail_tree_table_sz
? cnt
: avail_tree_table_sz
- 1;
684 static struct tree_content
*new_tree_content(unsigned int cnt
)
686 struct avail_tree_content
*f
, *l
= NULL
;
687 struct tree_content
*t
;
688 unsigned int hc
= hc_entries(cnt
);
690 for (f
= avail_tree_table
[hc
]; f
; l
= f
, f
= f
->next_avail
)
691 if (f
->entry_capacity
>= cnt
)
696 l
->next_avail
= f
->next_avail
;
698 avail_tree_table
[hc
] = f
->next_avail
;
700 cnt
= cnt
& 7 ? ((cnt
/ 8) + 1) * 8 : cnt
;
701 f
= pool_alloc(sizeof(*t
) + sizeof(t
->entries
[0]) * cnt
);
702 f
->entry_capacity
= cnt
;
705 t
= (struct tree_content
*)f
;
711 static void release_tree_entry(struct tree_entry
*e
);
712 static void release_tree_content(struct tree_content
*t
)
714 struct avail_tree_content
*f
= (struct avail_tree_content
*)t
;
715 unsigned int hc
= hc_entries(f
->entry_capacity
);
716 f
->next_avail
= avail_tree_table
[hc
];
717 avail_tree_table
[hc
] = f
;
720 static void release_tree_content_recursive(struct tree_content
*t
)
723 for (i
= 0; i
< t
->entry_count
; i
++)
724 release_tree_entry(t
->entries
[i
]);
725 release_tree_content(t
);
728 static struct tree_content
*grow_tree_content(
729 struct tree_content
*t
,
732 struct tree_content
*r
= new_tree_content(t
->entry_count
+ amt
);
733 r
->entry_count
= t
->entry_count
;
734 r
->delta_depth
= t
->delta_depth
;
735 memcpy(r
->entries
,t
->entries
,t
->entry_count
*sizeof(t
->entries
[0]));
736 release_tree_content(t
);
740 static struct tree_entry
*new_tree_entry(void)
742 struct tree_entry
*e
;
744 if (!avail_tree_entry
) {
745 unsigned int n
= tree_entry_alloc
;
746 total_allocd
+= n
* sizeof(struct tree_entry
);
747 avail_tree_entry
= e
= xmalloc(n
* sizeof(struct tree_entry
));
749 *((void**)e
) = e
+ 1;
755 e
= avail_tree_entry
;
756 avail_tree_entry
= *((void**)e
);
760 static void release_tree_entry(struct tree_entry
*e
)
763 release_tree_content_recursive(e
->tree
);
764 *((void**)e
) = avail_tree_entry
;
765 avail_tree_entry
= e
;
768 static struct tree_content
*dup_tree_content(struct tree_content
*s
)
770 struct tree_content
*d
;
771 struct tree_entry
*a
, *b
;
776 d
= new_tree_content(s
->entry_count
);
777 for (i
= 0; i
< s
->entry_count
; i
++) {
779 b
= new_tree_entry();
780 memcpy(b
, a
, sizeof(*a
));
781 if (a
->tree
&& is_null_sha1(b
->versions
[1].sha1
))
782 b
->tree
= dup_tree_content(a
->tree
);
787 d
->entry_count
= s
->entry_count
;
788 d
->delta_depth
= s
->delta_depth
;
793 static void start_packfile(void)
795 static char tmpfile
[PATH_MAX
];
796 struct packed_git
*p
;
797 struct pack_header hdr
;
800 snprintf(tmpfile
, sizeof(tmpfile
),
801 "%s/tmp_pack_XXXXXX", get_object_directory());
802 pack_fd
= xmkstemp(tmpfile
);
803 p
= xcalloc(1, sizeof(*p
) + strlen(tmpfile
) + 2);
804 strcpy(p
->pack_name
, tmpfile
);
805 p
->pack_fd
= pack_fd
;
807 hdr
.hdr_signature
= htonl(PACK_SIGNATURE
);
808 hdr
.hdr_version
= htonl(2);
810 write_or_die(p
->pack_fd
, &hdr
, sizeof(hdr
));
813 pack_size
= sizeof(hdr
);
816 all_packs
= xrealloc(all_packs
, sizeof(*all_packs
) * (pack_id
+ 1));
817 all_packs
[pack_id
] = p
;
820 static int oecmp (const void *a_
, const void *b_
)
822 struct object_entry
*a
= *((struct object_entry
**)a_
);
823 struct object_entry
*b
= *((struct object_entry
**)b_
);
824 return hashcmp(a
->sha1
, b
->sha1
);
827 static char *create_index(void)
829 static char tmpfile
[PATH_MAX
];
832 struct object_entry
**idx
, **c
, **last
, *e
;
833 struct object_entry_pool
*o
;
837 /* Build the sorted table of object IDs. */
838 idx
= xmalloc(object_count
* sizeof(struct object_entry
*));
840 for (o
= blocks
; o
; o
= o
->next_pool
)
841 for (e
= o
->next_free
; e
-- != o
->entries
;)
842 if (pack_id
== e
->pack_id
)
844 last
= idx
+ object_count
;
846 die("internal consistency error creating the index");
847 qsort(idx
, object_count
, sizeof(struct object_entry
*), oecmp
);
849 /* Generate the fan-out array. */
851 for (i
= 0; i
< 256; i
++) {
852 struct object_entry
**next
= c
;;
853 while (next
< last
) {
854 if ((*next
)->sha1
[0] != i
)
858 array
[i
] = htonl(next
- idx
);
862 snprintf(tmpfile
, sizeof(tmpfile
),
863 "%s/tmp_idx_XXXXXX", get_object_directory());
864 idx_fd
= xmkstemp(tmpfile
);
865 f
= sha1fd(idx_fd
, tmpfile
);
866 sha1write(f
, array
, 256 * sizeof(int));
868 for (c
= idx
; c
!= last
; c
++) {
869 uint32_t offset
= htonl((*c
)->offset
);
870 sha1write(f
, &offset
, 4);
871 sha1write(f
, (*c
)->sha1
, sizeof((*c
)->sha1
));
872 SHA1_Update(&ctx
, (*c
)->sha1
, 20);
874 sha1write(f
, pack_data
->sha1
, sizeof(pack_data
->sha1
));
875 sha1close(f
, NULL
, 1);
877 SHA1_Final(pack_data
->sha1
, &ctx
);
881 static char *keep_pack(char *curr_index_name
)
883 static char name
[PATH_MAX
];
884 static const char *keep_msg
= "fast-import";
887 chmod(pack_data
->pack_name
, 0444);
888 chmod(curr_index_name
, 0444);
890 snprintf(name
, sizeof(name
), "%s/pack/pack-%s.keep",
891 get_object_directory(), sha1_to_hex(pack_data
->sha1
));
892 keep_fd
= open(name
, O_RDWR
|O_CREAT
|O_EXCL
, 0600);
894 die("cannot create keep file");
895 write(keep_fd
, keep_msg
, strlen(keep_msg
));
898 snprintf(name
, sizeof(name
), "%s/pack/pack-%s.pack",
899 get_object_directory(), sha1_to_hex(pack_data
->sha1
));
900 if (move_temp_to_file(pack_data
->pack_name
, name
))
901 die("cannot store pack file");
903 snprintf(name
, sizeof(name
), "%s/pack/pack-%s.idx",
904 get_object_directory(), sha1_to_hex(pack_data
->sha1
));
905 if (move_temp_to_file(curr_index_name
, name
))
906 die("cannot store index file");
910 static void unkeep_all_packs(void)
912 static char name
[PATH_MAX
];
915 for (k
= 0; k
< pack_id
; k
++) {
916 struct packed_git
*p
= all_packs
[k
];
917 snprintf(name
, sizeof(name
), "%s/pack/pack-%s.keep",
918 get_object_directory(), sha1_to_hex(p
->sha1
));
923 static void end_packfile(void)
925 struct packed_git
*old_p
= pack_data
, *new_p
;
933 fixup_pack_header_footer(pack_data
->pack_fd
, pack_data
->sha1
,
934 pack_data
->pack_name
, object_count
);
935 close(pack_data
->pack_fd
);
936 idx_name
= keep_pack(create_index());
938 /* Register the packfile with core git's machinary. */
939 new_p
= add_packed_git(idx_name
, strlen(idx_name
), 1);
941 die("core git rejected index %s", idx_name
);
942 new_p
->windows
= old_p
->windows
;
943 all_packs
[pack_id
] = new_p
;
944 install_packed_git(new_p
);
946 /* Print the boundary */
948 fprintf(pack_edges
, "%s:", new_p
->pack_name
);
949 for (i
= 0; i
< branch_table_sz
; i
++) {
950 for (b
= branch_table
[i
]; b
; b
= b
->table_next_branch
) {
951 if (b
->pack_id
== pack_id
)
952 fprintf(pack_edges
, " %s", sha1_to_hex(b
->sha1
));
955 for (t
= first_tag
; t
; t
= t
->next_tag
) {
956 if (t
->pack_id
== pack_id
)
957 fprintf(pack_edges
, " %s", sha1_to_hex(t
->sha1
));
959 fputc('\n', pack_edges
);
966 unlink(old_p
->pack_name
);
969 /* We can't carry a delta across packfiles. */
970 free(last_blob
.data
);
971 last_blob
.data
= NULL
;
973 last_blob
.offset
= 0;
977 static void cycle_packfile(void)
983 static size_t encode_header(
984 enum object_type type
,
991 if (type
< OBJ_COMMIT
|| type
> OBJ_REF_DELTA
)
992 die("bad type %d", type
);
994 c
= (type
<< 4) | (size
& 15);
1006 static int store_object(
1007 enum object_type type
,
1010 struct last_object
*last
,
1011 unsigned char *sha1out
,
1015 struct object_entry
*e
;
1016 unsigned char hdr
[96];
1017 unsigned char sha1
[20];
1018 unsigned long hdrlen
, deltalen
;
1022 hdrlen
= sprintf((char*)hdr
,"%s %lu", typename(type
),
1023 (unsigned long)datlen
) + 1;
1025 SHA1_Update(&c
, hdr
, hdrlen
);
1026 SHA1_Update(&c
, dat
, datlen
);
1027 SHA1_Final(sha1
, &c
);
1029 hashcpy(sha1out
, sha1
);
1031 e
= insert_object(sha1
);
1033 insert_mark(mark
, e
);
1035 duplicate_count_by_type
[type
]++;
1037 } else if (find_sha1_pack(sha1
, packed_git
)) {
1039 e
->pack_id
= MAX_PACK_ID
;
1040 e
->offset
= 1; /* just not zero! */
1041 duplicate_count_by_type
[type
]++;
1045 if (last
&& last
->data
&& last
->depth
< max_depth
) {
1046 delta
= diff_delta(last
->data
, last
->len
,
1049 if (delta
&& deltalen
>= datlen
) {
1056 memset(&s
, 0, sizeof(s
));
1057 deflateInit(&s
, zlib_compression_level
);
1060 s
.avail_in
= deltalen
;
1063 s
.avail_in
= datlen
;
1065 s
.avail_out
= deflateBound(&s
, s
.avail_in
);
1066 s
.next_out
= out
= xmalloc(s
.avail_out
);
1067 while (deflate(&s
, Z_FINISH
) == Z_OK
)
1071 /* Determine if we should auto-checkpoint. */
1072 if ((pack_size
+ 60 + s
.total_out
) > max_packsize
1073 || (pack_size
+ 60 + s
.total_out
) < pack_size
) {
1075 /* This new object needs to *not* have the current pack_id. */
1076 e
->pack_id
= pack_id
+ 1;
1079 /* We cannot carry a delta into the new pack. */
1084 memset(&s
, 0, sizeof(s
));
1085 deflateInit(&s
, zlib_compression_level
);
1087 s
.avail_in
= datlen
;
1088 s
.avail_out
= deflateBound(&s
, s
.avail_in
);
1089 s
.next_out
= out
= xrealloc(out
, s
.avail_out
);
1090 while (deflate(&s
, Z_FINISH
) == Z_OK
)
1097 e
->pack_id
= pack_id
;
1098 e
->offset
= pack_size
;
1100 object_count_by_type
[type
]++;
1103 unsigned long ofs
= e
->offset
- last
->offset
;
1104 unsigned pos
= sizeof(hdr
) - 1;
1106 delta_count_by_type
[type
]++;
1109 hdrlen
= encode_header(OBJ_OFS_DELTA
, deltalen
, hdr
);
1110 write_or_die(pack_data
->pack_fd
, hdr
, hdrlen
);
1111 pack_size
+= hdrlen
;
1113 hdr
[pos
] = ofs
& 127;
1115 hdr
[--pos
] = 128 | (--ofs
& 127);
1116 write_or_die(pack_data
->pack_fd
, hdr
+ pos
, sizeof(hdr
) - pos
);
1117 pack_size
+= sizeof(hdr
) - pos
;
1121 hdrlen
= encode_header(type
, datlen
, hdr
);
1122 write_or_die(pack_data
->pack_fd
, hdr
, hdrlen
);
1123 pack_size
+= hdrlen
;
1126 write_or_die(pack_data
->pack_fd
, out
, s
.total_out
);
1127 pack_size
+= s
.total_out
;
1135 last
->offset
= e
->offset
;
1141 static void *gfi_unpack_entry(
1142 struct object_entry
*oe
,
1143 unsigned long *sizep
)
1145 enum object_type type
;
1146 struct packed_git
*p
= all_packs
[oe
->pack_id
];
1148 p
->pack_size
= pack_size
+ 20;
1149 return unpack_entry(p
, oe
->offset
, &type
, sizep
);
1152 static const char *get_mode(const char *str
, uint16_t *modep
)
1157 while ((c
= *str
++) != ' ') {
1158 if (c
< '0' || c
> '7')
1160 mode
= (mode
<< 3) + (c
- '0');
1166 static void load_tree(struct tree_entry
*root
)
1168 unsigned char* sha1
= root
->versions
[1].sha1
;
1169 struct object_entry
*myoe
;
1170 struct tree_content
*t
;
1175 root
->tree
= t
= new_tree_content(8);
1176 if (is_null_sha1(sha1
))
1179 myoe
= find_object(sha1
);
1180 if (myoe
&& myoe
->pack_id
!= MAX_PACK_ID
) {
1181 if (myoe
->type
!= OBJ_TREE
)
1182 die("Not a tree: %s", sha1_to_hex(sha1
));
1184 buf
= gfi_unpack_entry(myoe
, &size
);
1186 enum object_type type
;
1187 buf
= read_sha1_file(sha1
, &type
, &size
);
1188 if (!buf
|| type
!= OBJ_TREE
)
1189 die("Can't load tree %s", sha1_to_hex(sha1
));
1193 while (c
!= (buf
+ size
)) {
1194 struct tree_entry
*e
= new_tree_entry();
1196 if (t
->entry_count
== t
->entry_capacity
)
1197 root
->tree
= t
= grow_tree_content(t
, t
->entry_count
);
1198 t
->entries
[t
->entry_count
++] = e
;
1201 c
= get_mode(c
, &e
->versions
[1].mode
);
1203 die("Corrupt mode in %s", sha1_to_hex(sha1
));
1204 e
->versions
[0].mode
= e
->versions
[1].mode
;
1205 e
->name
= to_atom(c
, strlen(c
));
1206 c
+= e
->name
->str_len
+ 1;
1207 hashcpy(e
->versions
[0].sha1
, (unsigned char*)c
);
1208 hashcpy(e
->versions
[1].sha1
, (unsigned char*)c
);
1214 static int tecmp0 (const void *_a
, const void *_b
)
1216 struct tree_entry
*a
= *((struct tree_entry
**)_a
);
1217 struct tree_entry
*b
= *((struct tree_entry
**)_b
);
1218 return base_name_compare(
1219 a
->name
->str_dat
, a
->name
->str_len
, a
->versions
[0].mode
,
1220 b
->name
->str_dat
, b
->name
->str_len
, b
->versions
[0].mode
);
1223 static int tecmp1 (const void *_a
, const void *_b
)
1225 struct tree_entry
*a
= *((struct tree_entry
**)_a
);
1226 struct tree_entry
*b
= *((struct tree_entry
**)_b
);
1227 return base_name_compare(
1228 a
->name
->str_dat
, a
->name
->str_len
, a
->versions
[1].mode
,
1229 b
->name
->str_dat
, b
->name
->str_len
, b
->versions
[1].mode
);
1232 static void mktree(struct tree_content
*t
,
1242 qsort(t
->entries
,t
->entry_count
,sizeof(t
->entries
[0]),tecmp0
);
1244 qsort(t
->entries
,t
->entry_count
,sizeof(t
->entries
[0]),tecmp1
);
1246 for (i
= 0; i
< t
->entry_count
; i
++) {
1247 if (t
->entries
[i
]->versions
[v
].mode
)
1248 maxlen
+= t
->entries
[i
]->name
->str_len
+ 34;
1251 size_dbuf(b
, maxlen
);
1253 for (i
= 0; i
< t
->entry_count
; i
++) {
1254 struct tree_entry
*e
= t
->entries
[i
];
1255 if (!e
->versions
[v
].mode
)
1257 c
+= sprintf(c
, "%o", (unsigned int)e
->versions
[v
].mode
);
1259 strcpy(c
, e
->name
->str_dat
);
1260 c
+= e
->name
->str_len
+ 1;
1261 hashcpy((unsigned char*)c
, e
->versions
[v
].sha1
);
1264 *szp
= c
- (char*)b
->buffer
;
1267 static void store_tree(struct tree_entry
*root
)
1269 struct tree_content
*t
= root
->tree
;
1270 unsigned int i
, j
, del
;
1271 unsigned long new_len
;
1272 struct last_object lo
;
1273 struct object_entry
*le
;
1275 if (!is_null_sha1(root
->versions
[1].sha1
))
1278 for (i
= 0; i
< t
->entry_count
; i
++) {
1279 if (t
->entries
[i
]->tree
)
1280 store_tree(t
->entries
[i
]);
1283 le
= find_object(root
->versions
[0].sha1
);
1284 if (!S_ISDIR(root
->versions
[0].mode
)
1286 || le
->pack_id
!= pack_id
) {
1291 mktree(t
, 0, &lo
.len
, &old_tree
);
1292 lo
.data
= old_tree
.buffer
;
1293 lo
.offset
= le
->offset
;
1294 lo
.depth
= t
->delta_depth
;
1298 mktree(t
, 1, &new_len
, &new_tree
);
1299 store_object(OBJ_TREE
, new_tree
.buffer
, new_len
,
1300 &lo
, root
->versions
[1].sha1
, 0);
1302 t
->delta_depth
= lo
.depth
;
1303 for (i
= 0, j
= 0, del
= 0; i
< t
->entry_count
; i
++) {
1304 struct tree_entry
*e
= t
->entries
[i
];
1305 if (e
->versions
[1].mode
) {
1306 e
->versions
[0].mode
= e
->versions
[1].mode
;
1307 hashcpy(e
->versions
[0].sha1
, e
->versions
[1].sha1
);
1308 t
->entries
[j
++] = e
;
1310 release_tree_entry(e
);
1314 t
->entry_count
-= del
;
1317 static int tree_content_set(
1318 struct tree_entry
*root
,
1320 const unsigned char *sha1
,
1321 const uint16_t mode
,
1322 struct tree_content
*subtree
)
1324 struct tree_content
*t
= root
->tree
;
1327 struct tree_entry
*e
;
1329 slash1
= strchr(p
, '/');
1335 die("Empty path component found in input");
1336 if (!slash1
&& !S_ISDIR(mode
) && subtree
)
1337 die("Non-directories cannot have subtrees");
1339 for (i
= 0; i
< t
->entry_count
; i
++) {
1341 if (e
->name
->str_len
== n
&& !strncmp(p
, e
->name
->str_dat
, n
)) {
1344 && e
->versions
[1].mode
== mode
1345 && !hashcmp(e
->versions
[1].sha1
, sha1
))
1347 e
->versions
[1].mode
= mode
;
1348 hashcpy(e
->versions
[1].sha1
, sha1
);
1350 release_tree_content_recursive(e
->tree
);
1352 hashclr(root
->versions
[1].sha1
);
1355 if (!S_ISDIR(e
->versions
[1].mode
)) {
1356 e
->tree
= new_tree_content(8);
1357 e
->versions
[1].mode
= S_IFDIR
;
1361 if (tree_content_set(e
, slash1
+ 1, sha1
, mode
, subtree
)) {
1362 hashclr(root
->versions
[1].sha1
);
1369 if (t
->entry_count
== t
->entry_capacity
)
1370 root
->tree
= t
= grow_tree_content(t
, t
->entry_count
);
1371 e
= new_tree_entry();
1372 e
->name
= to_atom(p
, n
);
1373 e
->versions
[0].mode
= 0;
1374 hashclr(e
->versions
[0].sha1
);
1375 t
->entries
[t
->entry_count
++] = e
;
1377 e
->tree
= new_tree_content(8);
1378 e
->versions
[1].mode
= S_IFDIR
;
1379 tree_content_set(e
, slash1
+ 1, sha1
, mode
, subtree
);
1382 e
->versions
[1].mode
= mode
;
1383 hashcpy(e
->versions
[1].sha1
, sha1
);
1385 hashclr(root
->versions
[1].sha1
);
1389 static int tree_content_remove(
1390 struct tree_entry
*root
,
1392 struct tree_entry
*backup_leaf
)
1394 struct tree_content
*t
= root
->tree
;
1397 struct tree_entry
*e
;
1399 slash1
= strchr(p
, '/');
1405 for (i
= 0; i
< t
->entry_count
; i
++) {
1407 if (e
->name
->str_len
== n
&& !strncmp(p
, e
->name
->str_dat
, n
)) {
1408 if (!slash1
|| !S_ISDIR(e
->versions
[1].mode
))
1412 if (tree_content_remove(e
, slash1
+ 1, backup_leaf
)) {
1413 for (n
= 0; n
< e
->tree
->entry_count
; n
++) {
1414 if (e
->tree
->entries
[n
]->versions
[1].mode
) {
1415 hashclr(root
->versions
[1].sha1
);
1429 memcpy(backup_leaf
, e
, sizeof(*backup_leaf
));
1431 release_tree_content_recursive(e
->tree
);
1433 e
->versions
[1].mode
= 0;
1434 hashclr(e
->versions
[1].sha1
);
1435 hashclr(root
->versions
[1].sha1
);
1439 static int tree_content_get(
1440 struct tree_entry
*root
,
1442 struct tree_entry
*leaf
)
1444 struct tree_content
*t
= root
->tree
;
1447 struct tree_entry
*e
;
1449 slash1
= strchr(p
, '/');
1455 for (i
= 0; i
< t
->entry_count
; i
++) {
1457 if (e
->name
->str_len
== n
&& !strncmp(p
, e
->name
->str_dat
, n
)) {
1459 memcpy(leaf
, e
, sizeof(*leaf
));
1460 if (e
->tree
&& is_null_sha1(e
->versions
[1].sha1
))
1461 leaf
->tree
= dup_tree_content(e
->tree
);
1466 if (!S_ISDIR(e
->versions
[1].mode
))
1470 return tree_content_get(e
, slash1
+ 1, leaf
);
1476 static int update_branch(struct branch
*b
)
1478 static const char *msg
= "fast-import";
1479 struct ref_lock
*lock
;
1480 unsigned char old_sha1
[20];
1482 if (read_ref(b
->name
, old_sha1
))
1484 lock
= lock_any_ref_for_update(b
->name
, old_sha1
, 0);
1486 return error("Unable to lock %s", b
->name
);
1487 if (!force_update
&& !is_null_sha1(old_sha1
)) {
1488 struct commit
*old_cmit
, *new_cmit
;
1490 old_cmit
= lookup_commit_reference_gently(old_sha1
, 0);
1491 new_cmit
= lookup_commit_reference_gently(b
->sha1
, 0);
1492 if (!old_cmit
|| !new_cmit
) {
1494 return error("Branch %s is missing commits.", b
->name
);
1497 if (!in_merge_bases(old_cmit
, &new_cmit
, 1)) {
1499 warning("Not updating %s"
1500 " (new tip %s does not contain %s)",
1501 b
->name
, sha1_to_hex(b
->sha1
), sha1_to_hex(old_sha1
));
1505 if (write_ref_sha1(lock
, b
->sha1
, msg
) < 0)
1506 return error("Unable to update %s", b
->name
);
1510 static void dump_branches(void)
1515 for (i
= 0; i
< branch_table_sz
; i
++) {
1516 for (b
= branch_table
[i
]; b
; b
= b
->table_next_branch
)
1517 failure
|= update_branch(b
);
1521 static void dump_tags(void)
1523 static const char *msg
= "fast-import";
1525 struct ref_lock
*lock
;
1526 char ref_name
[PATH_MAX
];
1528 for (t
= first_tag
; t
; t
= t
->next_tag
) {
1529 sprintf(ref_name
, "tags/%s", t
->name
);
1530 lock
= lock_ref_sha1(ref_name
, NULL
);
1531 if (!lock
|| write_ref_sha1(lock
, t
->sha1
, msg
) < 0)
1532 failure
|= error("Unable to update %s", ref_name
);
1536 static void dump_marks_helper(FILE *f
,
1542 for (k
= 0; k
< 1024; k
++) {
1543 if (m
->data
.sets
[k
])
1544 dump_marks_helper(f
, (base
+ k
) << m
->shift
,
1548 for (k
= 0; k
< 1024; k
++) {
1549 if (m
->data
.marked
[k
])
1550 fprintf(f
, ":%" PRIuMAX
" %s\n", base
+ k
,
1551 sha1_to_hex(m
->data
.marked
[k
]->sha1
));
1556 static void dump_marks(void)
1558 static struct lock_file mark_lock
;
1565 mark_fd
= hold_lock_file_for_update(&mark_lock
, mark_file
, 0);
1567 failure
|= error("Unable to write marks file %s: %s",
1568 mark_file
, strerror(errno
));
1572 f
= fdopen(mark_fd
, "w");
1574 rollback_lock_file(&mark_lock
);
1575 failure
|= error("Unable to write marks file %s: %s",
1576 mark_file
, strerror(errno
));
1580 dump_marks_helper(f
, 0, marks
);
1582 if (commit_lock_file(&mark_lock
))
1583 failure
|= error("Unable to write marks file %s: %s",
1584 mark_file
, strerror(errno
));
1587 static void read_next_command(void)
1590 if (unread_command_buf
) {
1591 unread_command_buf
= 0;
1592 if (command_buf
.eof
)
1595 struct recent_command
*rc
;
1597 command_buf
.buf
= NULL
;
1598 read_line(&command_buf
, stdin
, '\n');
1599 if (command_buf
.eof
)
1607 cmd_hist
.next
= rc
->next
;
1608 cmd_hist
.next
->prev
= &cmd_hist
;
1612 rc
->buf
= command_buf
.buf
;
1613 rc
->prev
= cmd_tail
;
1614 rc
->next
= cmd_hist
.prev
;
1615 rc
->prev
->next
= rc
;
1618 } while (command_buf
.buf
[0] == '#');
1621 static void skip_optional_lf()
1623 int term_char
= fgetc(stdin
);
1624 if (term_char
!= '\n' && term_char
!= EOF
)
1625 ungetc(term_char
, stdin
);
1628 static void cmd_mark(void)
1630 if (!prefixcmp(command_buf
.buf
, "mark :")) {
1631 next_mark
= strtoumax(command_buf
.buf
+ 6, NULL
, 10);
1632 read_next_command();
1638 static void *cmd_data (size_t *size
)
1643 if (prefixcmp(command_buf
.buf
, "data "))
1644 die("Expected 'data n' command, found: %s", command_buf
.buf
);
1646 if (!prefixcmp(command_buf
.buf
+ 5, "<<")) {
1647 char *term
= xstrdup(command_buf
.buf
+ 5 + 2);
1648 size_t sz
= 8192, term_len
= command_buf
.len
- 5 - 2;
1650 buffer
= xmalloc(sz
);
1651 command_buf
.buf
= NULL
;
1653 read_line(&command_buf
, stdin
, '\n');
1654 if (command_buf
.eof
)
1655 die("EOF in data (terminator '%s' not found)", term
);
1656 if (term_len
== command_buf
.len
1657 && !strcmp(term
, command_buf
.buf
))
1659 ALLOC_GROW(buffer
, length
+ command_buf
.len
, sz
);
1660 memcpy(buffer
+ length
,
1662 command_buf
.len
- 1);
1663 length
+= command_buf
.len
- 1;
1664 buffer
[length
++] = '\n';
1670 length
= strtoul(command_buf
.buf
+ 5, NULL
, 10);
1671 buffer
= xmalloc(length
);
1672 while (n
< length
) {
1673 size_t s
= fread(buffer
+ n
, 1, length
- n
, stdin
);
1674 if (!s
&& feof(stdin
))
1675 die("EOF in data (%lu bytes remaining)",
1676 (unsigned long)(length
- n
));
1686 static int validate_raw_date(const char *src
, char *result
, int maxlen
)
1688 const char *orig_src
= src
;
1691 strtoul(src
, &endp
, 10);
1692 if (endp
== src
|| *endp
!= ' ')
1696 if (*src
!= '-' && *src
!= '+')
1700 strtoul(src
+ 1, &endp
, 10);
1701 if (endp
== src
|| *endp
|| (endp
- orig_src
) >= maxlen
)
1704 strcpy(result
, orig_src
);
1708 static char *parse_ident(const char *buf
)
1714 gt
= strrchr(buf
, '>');
1716 die("Missing > in ident string: %s", buf
);
1719 die("Missing space after > in ident string: %s", buf
);
1721 name_len
= gt
- buf
;
1722 ident
= xmalloc(name_len
+ 24);
1723 strncpy(ident
, buf
, name_len
);
1727 if (validate_raw_date(gt
, ident
+ name_len
, 24) < 0)
1728 die("Invalid raw date \"%s\" in ident: %s", gt
, buf
);
1730 case WHENSPEC_RFC2822
:
1731 if (parse_date(gt
, ident
+ name_len
, 24) < 0)
1732 die("Invalid rfc2822 date \"%s\" in ident: %s", gt
, buf
);
1735 if (strcmp("now", gt
))
1736 die("Date in ident must be 'now': %s", buf
);
1737 datestamp(ident
+ name_len
, 24);
1744 static void cmd_new_blob(void)
1749 read_next_command();
1753 if (store_object(OBJ_BLOB
, d
, l
, &last_blob
, NULL
, next_mark
))
1757 static void unload_one_branch(void)
1759 while (cur_active_branches
1760 && cur_active_branches
>= max_active_branches
) {
1761 uintmax_t min_commit
= ULONG_MAX
;
1762 struct branch
*e
, *l
= NULL
, *p
= NULL
;
1764 for (e
= active_branches
; e
; e
= e
->active_next_branch
) {
1765 if (e
->last_commit
< min_commit
) {
1767 min_commit
= e
->last_commit
;
1773 e
= p
->active_next_branch
;
1774 p
->active_next_branch
= e
->active_next_branch
;
1776 e
= active_branches
;
1777 active_branches
= e
->active_next_branch
;
1780 e
->active_next_branch
= NULL
;
1781 if (e
->branch_tree
.tree
) {
1782 release_tree_content_recursive(e
->branch_tree
.tree
);
1783 e
->branch_tree
.tree
= NULL
;
1785 cur_active_branches
--;
1789 static void load_branch(struct branch
*b
)
1791 load_tree(&b
->branch_tree
);
1794 b
->active_next_branch
= active_branches
;
1795 active_branches
= b
;
1796 cur_active_branches
++;
1797 branch_load_count
++;
1801 static void file_change_m(struct branch
*b
)
1803 const char *p
= command_buf
.buf
+ 2;
1806 struct object_entry
*oe
= oe
;
1807 unsigned char sha1
[20];
1808 uint16_t mode
, inline_data
= 0;
1810 p
= get_mode(p
, &mode
);
1812 die("Corrupt mode: %s", command_buf
.buf
);
1814 case S_IFREG
| 0644:
1815 case S_IFREG
| 0755:
1822 die("Corrupt mode: %s", command_buf
.buf
);
1827 oe
= find_mark(strtoumax(p
+ 1, &x
, 10));
1828 hashcpy(sha1
, oe
->sha1
);
1830 } else if (!prefixcmp(p
, "inline")) {
1834 if (get_sha1_hex(p
, sha1
))
1835 die("Invalid SHA1: %s", command_buf
.buf
);
1836 oe
= find_object(sha1
);
1840 die("Missing space after SHA1: %s", command_buf
.buf
);
1842 p_uq
= unquote_c_style(p
, &endp
);
1845 die("Garbage after path in: %s", command_buf
.buf
);
1853 p
= p_uq
= xstrdup(p
);
1854 read_next_command();
1856 if (store_object(OBJ_BLOB
, d
, l
, &last_blob
, sha1
, 0))
1859 if (oe
->type
!= OBJ_BLOB
)
1860 die("Not a blob (actually a %s): %s",
1861 command_buf
.buf
, typename(oe
->type
));
1863 enum object_type type
= sha1_object_info(sha1
, NULL
);
1865 die("Blob not found: %s", command_buf
.buf
);
1866 if (type
!= OBJ_BLOB
)
1867 die("Not a blob (actually a %s): %s",
1868 typename(type
), command_buf
.buf
);
1871 tree_content_set(&b
->branch_tree
, p
, sha1
, S_IFREG
| mode
, NULL
);
1875 static void file_change_d(struct branch
*b
)
1877 const char *p
= command_buf
.buf
+ 2;
1881 p_uq
= unquote_c_style(p
, &endp
);
1884 die("Garbage after path in: %s", command_buf
.buf
);
1887 tree_content_remove(&b
->branch_tree
, p
, NULL
);
1891 static void file_change_cr(struct branch
*b
, int rename
)
1896 struct tree_entry leaf
;
1898 s
= command_buf
.buf
+ 2;
1899 s_uq
= unquote_c_style(s
, &endp
);
1902 die("Missing space after source: %s", command_buf
.buf
);
1905 endp
= strchr(s
, ' ');
1907 die("Missing space after source: %s", command_buf
.buf
);
1908 s_uq
= xmalloc(endp
- s
+ 1);
1909 memcpy(s_uq
, s
, endp
- s
);
1916 die("Missing dest: %s", command_buf
.buf
);
1919 d_uq
= unquote_c_style(d
, &endp
);
1922 die("Garbage after dest in: %s", command_buf
.buf
);
1926 memset(&leaf
, 0, sizeof(leaf
));
1928 tree_content_remove(&b
->branch_tree
, s
, &leaf
);
1930 tree_content_get(&b
->branch_tree
, s
, &leaf
);
1931 if (!leaf
.versions
[1].mode
)
1932 die("Path %s not in branch", s
);
1933 tree_content_set(&b
->branch_tree
, d
,
1934 leaf
.versions
[1].sha1
,
1935 leaf
.versions
[1].mode
,
1942 static void file_change_deleteall(struct branch
*b
)
1944 release_tree_content_recursive(b
->branch_tree
.tree
);
1945 hashclr(b
->branch_tree
.versions
[0].sha1
);
1946 hashclr(b
->branch_tree
.versions
[1].sha1
);
1947 load_tree(&b
->branch_tree
);
1950 static void cmd_from_commit(struct branch
*b
, char *buf
, unsigned long size
)
1952 if (!buf
|| size
< 46)
1953 die("Not a valid commit: %s", sha1_to_hex(b
->sha1
));
1954 if (memcmp("tree ", buf
, 5)
1955 || get_sha1_hex(buf
+ 5, b
->branch_tree
.versions
[1].sha1
))
1956 die("The commit %s is corrupt", sha1_to_hex(b
->sha1
));
1957 hashcpy(b
->branch_tree
.versions
[0].sha1
,
1958 b
->branch_tree
.versions
[1].sha1
);
1961 static void cmd_from_existing(struct branch
*b
)
1963 if (is_null_sha1(b
->sha1
)) {
1964 hashclr(b
->branch_tree
.versions
[0].sha1
);
1965 hashclr(b
->branch_tree
.versions
[1].sha1
);
1970 buf
= read_object_with_reference(b
->sha1
,
1971 commit_type
, &size
, b
->sha1
);
1972 cmd_from_commit(b
, buf
, size
);
1977 static int cmd_from(struct branch
*b
)
1982 if (prefixcmp(command_buf
.buf
, "from "))
1985 if (b
->branch_tree
.tree
) {
1986 release_tree_content_recursive(b
->branch_tree
.tree
);
1987 b
->branch_tree
.tree
= NULL
;
1990 from
= strchr(command_buf
.buf
, ' ') + 1;
1991 s
= lookup_branch(from
);
1993 die("Can't create a branch from itself: %s", b
->name
);
1995 unsigned char *t
= s
->branch_tree
.versions
[1].sha1
;
1996 hashcpy(b
->sha1
, s
->sha1
);
1997 hashcpy(b
->branch_tree
.versions
[0].sha1
, t
);
1998 hashcpy(b
->branch_tree
.versions
[1].sha1
, t
);
1999 } else if (*from
== ':') {
2000 uintmax_t idnum
= strtoumax(from
+ 1, NULL
, 10);
2001 struct object_entry
*oe
= find_mark(idnum
);
2002 if (oe
->type
!= OBJ_COMMIT
)
2003 die("Mark :%" PRIuMAX
" not a commit", idnum
);
2004 hashcpy(b
->sha1
, oe
->sha1
);
2005 if (oe
->pack_id
!= MAX_PACK_ID
) {
2007 char *buf
= gfi_unpack_entry(oe
, &size
);
2008 cmd_from_commit(b
, buf
, size
);
2011 cmd_from_existing(b
);
2012 } else if (!get_sha1(from
, b
->sha1
))
2013 cmd_from_existing(b
);
2015 die("Invalid ref name or SHA1 expression: %s", from
);
2017 read_next_command();
2021 static struct hash_list
*cmd_merge(unsigned int *count
)
2023 struct hash_list
*list
= NULL
, *n
, *e
= e
;
2028 while (!prefixcmp(command_buf
.buf
, "merge ")) {
2029 from
= strchr(command_buf
.buf
, ' ') + 1;
2030 n
= xmalloc(sizeof(*n
));
2031 s
= lookup_branch(from
);
2033 hashcpy(n
->sha1
, s
->sha1
);
2034 else if (*from
== ':') {
2035 uintmax_t idnum
= strtoumax(from
+ 1, NULL
, 10);
2036 struct object_entry
*oe
= find_mark(idnum
);
2037 if (oe
->type
!= OBJ_COMMIT
)
2038 die("Mark :%" PRIuMAX
" not a commit", idnum
);
2039 hashcpy(n
->sha1
, oe
->sha1
);
2040 } else if (!get_sha1(from
, n
->sha1
)) {
2042 char *buf
= read_object_with_reference(n
->sha1
,
2043 commit_type
, &size
, n
->sha1
);
2044 if (!buf
|| size
< 46)
2045 die("Not a valid commit: %s", from
);
2048 die("Invalid ref name or SHA1 expression: %s", from
);
2057 read_next_command();
2062 static void cmd_new_commit(void)
2068 char *author
= NULL
;
2069 char *committer
= NULL
;
2070 struct hash_list
*merge_list
= NULL
;
2071 unsigned int merge_count
;
2073 /* Obtain the branch name from the rest of our command */
2074 sp
= strchr(command_buf
.buf
, ' ') + 1;
2075 b
= lookup_branch(sp
);
2079 read_next_command();
2081 if (!prefixcmp(command_buf
.buf
, "author ")) {
2082 author
= parse_ident(command_buf
.buf
+ 7);
2083 read_next_command();
2085 if (!prefixcmp(command_buf
.buf
, "committer ")) {
2086 committer
= parse_ident(command_buf
.buf
+ 10);
2087 read_next_command();
2090 die("Expected committer but didn't get one");
2091 msg
= cmd_data(&msglen
);
2092 read_next_command();
2094 merge_list
= cmd_merge(&merge_count
);
2096 /* ensure the branch is active/loaded */
2097 if (!b
->branch_tree
.tree
|| !max_active_branches
) {
2098 unload_one_branch();
2103 while (!command_buf
.eof
&& command_buf
.len
> 1) {
2104 if (!prefixcmp(command_buf
.buf
, "M "))
2106 else if (!prefixcmp(command_buf
.buf
, "D "))
2108 else if (!prefixcmp(command_buf
.buf
, "R "))
2109 file_change_cr(b
, 1);
2110 else if (!prefixcmp(command_buf
.buf
, "C "))
2111 file_change_cr(b
, 0);
2112 else if (!strcmp("deleteall", command_buf
.buf
))
2113 file_change_deleteall(b
);
2115 unread_command_buf
= 1;
2118 read_next_command();
2121 /* build the tree and the commit */
2122 store_tree(&b
->branch_tree
);
2123 hashcpy(b
->branch_tree
.versions
[0].sha1
,
2124 b
->branch_tree
.versions
[1].sha1
);
2125 size_dbuf(&new_data
, 114 + msglen
2128 ? strlen(author
) + strlen(committer
)
2129 : 2 * strlen(committer
)));
2130 sp
= new_data
.buffer
;
2131 sp
+= sprintf(sp
, "tree %s\n",
2132 sha1_to_hex(b
->branch_tree
.versions
[1].sha1
));
2133 if (!is_null_sha1(b
->sha1
))
2134 sp
+= sprintf(sp
, "parent %s\n", sha1_to_hex(b
->sha1
));
2135 while (merge_list
) {
2136 struct hash_list
*next
= merge_list
->next
;
2137 sp
+= sprintf(sp
, "parent %s\n", sha1_to_hex(merge_list
->sha1
));
2141 sp
+= sprintf(sp
, "author %s\n", author
? author
: committer
);
2142 sp
+= sprintf(sp
, "committer %s\n", committer
);
2144 memcpy(sp
, msg
, msglen
);
2150 if (!store_object(OBJ_COMMIT
,
2151 new_data
.buffer
, sp
- (char*)new_data
.buffer
,
2152 NULL
, b
->sha1
, next_mark
))
2153 b
->pack_id
= pack_id
;
2154 b
->last_commit
= object_count_by_type
[OBJ_COMMIT
];
2157 static void cmd_new_tag(void)
2166 uintmax_t from_mark
= 0;
2167 unsigned char sha1
[20];
2169 /* Obtain the new tag name from the rest of our command */
2170 sp
= strchr(command_buf
.buf
, ' ') + 1;
2171 t
= pool_alloc(sizeof(struct tag
));
2173 t
->name
= pool_strdup(sp
);
2175 last_tag
->next_tag
= t
;
2179 read_next_command();
2182 if (prefixcmp(command_buf
.buf
, "from "))
2183 die("Expected from command, got %s", command_buf
.buf
);
2184 from
= strchr(command_buf
.buf
, ' ') + 1;
2185 s
= lookup_branch(from
);
2187 hashcpy(sha1
, s
->sha1
);
2188 } else if (*from
== ':') {
2189 struct object_entry
*oe
;
2190 from_mark
= strtoumax(from
+ 1, NULL
, 10);
2191 oe
= find_mark(from_mark
);
2192 if (oe
->type
!= OBJ_COMMIT
)
2193 die("Mark :%" PRIuMAX
" not a commit", from_mark
);
2194 hashcpy(sha1
, oe
->sha1
);
2195 } else if (!get_sha1(from
, sha1
)) {
2199 buf
= read_object_with_reference(sha1
,
2200 commit_type
, &size
, sha1
);
2201 if (!buf
|| size
< 46)
2202 die("Not a valid commit: %s", from
);
2205 die("Invalid ref name or SHA1 expression: %s", from
);
2206 read_next_command();
2209 if (prefixcmp(command_buf
.buf
, "tagger "))
2210 die("Expected tagger command, got %s", command_buf
.buf
);
2211 tagger
= parse_ident(command_buf
.buf
+ 7);
2213 /* tag payload/message */
2214 read_next_command();
2215 msg
= cmd_data(&msglen
);
2217 /* build the tag object */
2218 size_dbuf(&new_data
, 67+strlen(t
->name
)+strlen(tagger
)+msglen
);
2219 sp
= new_data
.buffer
;
2220 sp
+= sprintf(sp
, "object %s\n", sha1_to_hex(sha1
));
2221 sp
+= sprintf(sp
, "type %s\n", commit_type
);
2222 sp
+= sprintf(sp
, "tag %s\n", t
->name
);
2223 sp
+= sprintf(sp
, "tagger %s\n", tagger
);
2225 memcpy(sp
, msg
, msglen
);
2230 if (store_object(OBJ_TAG
, new_data
.buffer
,
2231 sp
- (char*)new_data
.buffer
,
2233 t
->pack_id
= MAX_PACK_ID
;
2235 t
->pack_id
= pack_id
;
2238 static void cmd_reset_branch(void)
2243 /* Obtain the branch name from the rest of our command */
2244 sp
= strchr(command_buf
.buf
, ' ') + 1;
2245 b
= lookup_branch(sp
);
2248 hashclr(b
->branch_tree
.versions
[0].sha1
);
2249 hashclr(b
->branch_tree
.versions
[1].sha1
);
2250 if (b
->branch_tree
.tree
) {
2251 release_tree_content_recursive(b
->branch_tree
.tree
);
2252 b
->branch_tree
.tree
= NULL
;
2257 read_next_command();
2258 if (!cmd_from(b
) && command_buf
.len
> 1)
2259 unread_command_buf
= 1;
2262 static void cmd_checkpoint(void)
2273 static void cmd_progress(void)
2275 fwrite(command_buf
.buf
, 1, command_buf
.len
- 1, stdout
);
2276 fputc('\n', stdout
);
2281 static void import_marks(const char *input_file
)
2284 FILE *f
= fopen(input_file
, "r");
2286 die("cannot read %s: %s", input_file
, strerror(errno
));
2287 while (fgets(line
, sizeof(line
), f
)) {
2290 unsigned char sha1
[20];
2291 struct object_entry
*e
;
2293 end
= strchr(line
, '\n');
2294 if (line
[0] != ':' || !end
)
2295 die("corrupt mark line: %s", line
);
2297 mark
= strtoumax(line
+ 1, &end
, 10);
2298 if (!mark
|| end
== line
+ 1
2299 || *end
!= ' ' || get_sha1(end
+ 1, sha1
))
2300 die("corrupt mark line: %s", line
);
2301 e
= find_object(sha1
);
2303 enum object_type type
= sha1_object_info(sha1
, NULL
);
2305 die("object not found: %s", sha1_to_hex(sha1
));
2306 e
= insert_object(sha1
);
2308 e
->pack_id
= MAX_PACK_ID
;
2309 e
->offset
= 1; /* just not zero! */
2311 insert_mark(mark
, e
);
2316 static const char fast_import_usage
[] =
2317 "git-fast-import [--date-format=f] [--max-pack-size=n] [--depth=n] [--active-branches=n] [--export-marks=marks.file]";
2319 int main(int argc
, const char **argv
)
2321 unsigned int i
, show_stats
= 1;
2323 git_config(git_default_config
);
2324 alloc_objects(object_entry_alloc
);
2325 strbuf_init(&command_buf
);
2326 atom_table
= xcalloc(atom_table_sz
, sizeof(struct atom_str
*));
2327 branch_table
= xcalloc(branch_table_sz
, sizeof(struct branch
*));
2328 avail_tree_table
= xcalloc(avail_tree_table_sz
, sizeof(struct avail_tree_content
*));
2329 marks
= pool_calloc(1, sizeof(struct mark_set
));
2331 for (i
= 1; i
< argc
; i
++) {
2332 const char *a
= argv
[i
];
2334 if (*a
!= '-' || !strcmp(a
, "--"))
2336 else if (!prefixcmp(a
, "--date-format=")) {
2337 const char *fmt
= a
+ 14;
2338 if (!strcmp(fmt
, "raw"))
2339 whenspec
= WHENSPEC_RAW
;
2340 else if (!strcmp(fmt
, "rfc2822"))
2341 whenspec
= WHENSPEC_RFC2822
;
2342 else if (!strcmp(fmt
, "now"))
2343 whenspec
= WHENSPEC_NOW
;
2345 die("unknown --date-format argument %s", fmt
);
2347 else if (!prefixcmp(a
, "--max-pack-size="))
2348 max_packsize
= strtoumax(a
+ 16, NULL
, 0) * 1024 * 1024;
2349 else if (!prefixcmp(a
, "--depth="))
2350 max_depth
= strtoul(a
+ 8, NULL
, 0);
2351 else if (!prefixcmp(a
, "--active-branches="))
2352 max_active_branches
= strtoul(a
+ 18, NULL
, 0);
2353 else if (!prefixcmp(a
, "--import-marks="))
2354 import_marks(a
+ 15);
2355 else if (!prefixcmp(a
, "--export-marks="))
2357 else if (!prefixcmp(a
, "--export-pack-edges=")) {
2360 pack_edges
= fopen(a
+ 20, "a");
2362 die("Cannot open %s: %s", a
+ 20, strerror(errno
));
2363 } else if (!strcmp(a
, "--force"))
2365 else if (!strcmp(a
, "--quiet"))
2367 else if (!strcmp(a
, "--stats"))
2370 die("unknown option %s", a
);
2373 usage(fast_import_usage
);
2375 rc_free
= pool_alloc(cmd_save
* sizeof(*rc_free
));
2376 for (i
= 0; i
< (cmd_save
- 1); i
++)
2377 rc_free
[i
].next
= &rc_free
[i
+ 1];
2378 rc_free
[cmd_save
- 1].next
= NULL
;
2380 prepare_packed_git();
2382 set_die_routine(die_nicely
);
2384 read_next_command();
2385 if (command_buf
.eof
)
2387 else if (!strcmp("blob", command_buf
.buf
))
2389 else if (!prefixcmp(command_buf
.buf
, "commit "))
2391 else if (!prefixcmp(command_buf
.buf
, "tag "))
2393 else if (!prefixcmp(command_buf
.buf
, "reset "))
2395 else if (!strcmp("checkpoint", command_buf
.buf
))
2397 else if (!prefixcmp(command_buf
.buf
, "progress "))
2400 die("Unsupported command: %s", command_buf
.buf
);
2413 uintmax_t total_count
= 0, duplicate_count
= 0;
2414 for (i
= 0; i
< ARRAY_SIZE(object_count_by_type
); i
++)
2415 total_count
+= object_count_by_type
[i
];
2416 for (i
= 0; i
< ARRAY_SIZE(duplicate_count_by_type
); i
++)
2417 duplicate_count
+= duplicate_count_by_type
[i
];
2419 fprintf(stderr
, "%s statistics:\n", argv
[0]);
2420 fprintf(stderr
, "---------------------------------------------------------------------\n");
2421 fprintf(stderr
, "Alloc'd objects: %10" PRIuMAX
"\n", alloc_count
);
2422 fprintf(stderr
, "Total objects: %10" PRIuMAX
" (%10" PRIuMAX
" duplicates )\n", total_count
, duplicate_count
);
2423 fprintf(stderr
, " blobs : %10" PRIuMAX
" (%10" PRIuMAX
" duplicates %10" PRIuMAX
" deltas)\n", object_count_by_type
[OBJ_BLOB
], duplicate_count_by_type
[OBJ_BLOB
], delta_count_by_type
[OBJ_BLOB
]);
2424 fprintf(stderr
, " trees : %10" PRIuMAX
" (%10" PRIuMAX
" duplicates %10" PRIuMAX
" deltas)\n", object_count_by_type
[OBJ_TREE
], duplicate_count_by_type
[OBJ_TREE
], delta_count_by_type
[OBJ_TREE
]);
2425 fprintf(stderr
, " commits: %10" PRIuMAX
" (%10" PRIuMAX
" duplicates %10" PRIuMAX
" deltas)\n", object_count_by_type
[OBJ_COMMIT
], duplicate_count_by_type
[OBJ_COMMIT
], delta_count_by_type
[OBJ_COMMIT
]);
2426 fprintf(stderr
, " tags : %10" PRIuMAX
" (%10" PRIuMAX
" duplicates %10" PRIuMAX
" deltas)\n", object_count_by_type
[OBJ_TAG
], duplicate_count_by_type
[OBJ_TAG
], delta_count_by_type
[OBJ_TAG
]);
2427 fprintf(stderr
, "Total branches: %10lu (%10lu loads )\n", branch_count
, branch_load_count
);
2428 fprintf(stderr
, " marks: %10" PRIuMAX
" (%10" PRIuMAX
" unique )\n", (((uintmax_t)1) << marks
->shift
) * 1024, marks_set_count
);
2429 fprintf(stderr
, " atoms: %10u\n", atom_cnt
);
2430 fprintf(stderr
, "Memory total: %10" PRIuMAX
" KiB\n", (total_allocd
+ alloc_count
*sizeof(struct object_entry
))/1024);
2431 fprintf(stderr
, " pools: %10lu KiB\n", (unsigned long)(total_allocd
/1024));
2432 fprintf(stderr
, " objects: %10" PRIuMAX
" KiB\n", (alloc_count
*sizeof(struct object_entry
))/1024);
2433 fprintf(stderr
, "---------------------------------------------------------------------\n");
2435 fprintf(stderr
, "---------------------------------------------------------------------\n");
2436 fprintf(stderr
, "\n");
2439 return failure
? 1 : 0;