7 #define PREV_BUF_SIZE 4096
13 struct packed_git
*packs
;
14 struct alt_base
*next
;
17 enum object_request_state
{
26 struct walker
*walker
;
27 unsigned char sha1
[20];
28 struct alt_base
*repo
;
30 char filename
[PATH_MAX
];
31 char tmpfile
[PATH_MAX
];
33 enum object_request_state state
;
35 char errorstr
[CURL_ERROR_SIZE
];
37 unsigned char real_sha1
[20];
42 struct active_request_slot
*slot
;
43 struct object_request
*next
;
46 struct alternates_request
{
47 struct walker
*walker
;
50 struct strbuf
*buffer
;
51 struct active_request_slot
*slot
;
61 static struct object_request
*object_queue_head
;
63 static size_t fwrite_sha1_file(void *ptr
, size_t eltsize
, size_t nmemb
,
66 unsigned char expn
[4096];
67 size_t size
= eltsize
* nmemb
;
69 struct object_request
*obj_req
= (struct object_request
*)data
;
71 ssize_t retval
= xwrite(obj_req
->local
,
72 (char *) ptr
+ posn
, size
- posn
);
76 } while (posn
< size
);
78 obj_req
->stream
.avail_in
= size
;
79 obj_req
->stream
.next_in
= ptr
;
81 obj_req
->stream
.next_out
= expn
;
82 obj_req
->stream
.avail_out
= sizeof(expn
);
83 obj_req
->zret
= git_inflate(&obj_req
->stream
, Z_SYNC_FLUSH
);
84 git_SHA1_Update(&obj_req
->c
, expn
,
85 sizeof(expn
) - obj_req
->stream
.avail_out
);
86 } while (obj_req
->stream
.avail_in
&& obj_req
->zret
== Z_OK
);
91 static void fetch_alternates(struct walker
*walker
, const char *base
);
93 static void process_object_response(void *callback_data
);
95 static void start_object_request(struct walker
*walker
,
96 struct object_request
*obj_req
)
98 char *hex
= sha1_to_hex(obj_req
->sha1
);
99 char prevfile
[PATH_MAX
];
103 unsigned char prev_buf
[PREV_BUF_SIZE
];
104 ssize_t prev_read
= 0;
106 char range
[RANGE_HEADER_SIZE
];
107 struct curl_slist
*range_header
= NULL
;
108 struct active_request_slot
*slot
;
110 snprintf(prevfile
, sizeof(prevfile
), "%s.prev", obj_req
->filename
);
111 unlink_or_warn(prevfile
);
112 rename(obj_req
->tmpfile
, prevfile
);
113 unlink_or_warn(obj_req
->tmpfile
);
115 if (obj_req
->local
!= -1)
116 error("fd leakage in start: %d", obj_req
->local
);
117 obj_req
->local
= open(obj_req
->tmpfile
,
118 O_WRONLY
| O_CREAT
| O_EXCL
, 0666);
120 * This could have failed due to the "lazy directory creation";
121 * try to mkdir the last path component.
123 if (obj_req
->local
< 0 && errno
== ENOENT
) {
124 char *dir
= strrchr(obj_req
->tmpfile
, '/');
127 mkdir(obj_req
->tmpfile
, 0777);
130 obj_req
->local
= open(obj_req
->tmpfile
,
131 O_WRONLY
| O_CREAT
| O_EXCL
, 0666);
134 if (obj_req
->local
< 0) {
135 obj_req
->state
= ABORTED
;
136 error("Couldn't create temporary file %s for %s: %s",
137 obj_req
->tmpfile
, obj_req
->filename
, strerror(errno
));
141 memset(&obj_req
->stream
, 0, sizeof(obj_req
->stream
));
143 git_inflate_init(&obj_req
->stream
);
145 git_SHA1_Init(&obj_req
->c
);
147 url
= xmalloc(strlen(obj_req
->repo
->base
) + 51);
148 obj_req
->url
= xmalloc(strlen(obj_req
->repo
->base
) + 51);
149 strcpy(url
, obj_req
->repo
->base
);
150 posn
= url
+ strlen(obj_req
->repo
->base
);
151 strcpy(posn
, "/objects/");
153 memcpy(posn
, hex
, 2);
156 strcpy(posn
, hex
+ 2);
157 strcpy(obj_req
->url
, url
);
160 * If a previous temp file is present, process what was already
163 prevlocal
= open(prevfile
, O_RDONLY
);
164 if (prevlocal
!= -1) {
166 prev_read
= xread(prevlocal
, prev_buf
, PREV_BUF_SIZE
);
168 if (fwrite_sha1_file(prev_buf
,
171 obj_req
) == prev_read
)
172 prev_posn
+= prev_read
;
176 } while (prev_read
> 0);
179 unlink_or_warn(prevfile
);
182 * Reset inflate/SHA1 if there was an error reading the previous temp
183 * file; also rewind to the beginning of the local file.
185 if (prev_read
== -1) {
186 memset(&obj_req
->stream
, 0, sizeof(obj_req
->stream
));
187 git_inflate_init(&obj_req
->stream
);
188 git_SHA1_Init(&obj_req
->c
);
191 lseek(obj_req
->local
, 0, SEEK_SET
);
192 ftruncate(obj_req
->local
, 0);
196 slot
= get_active_slot();
197 slot
->callback_func
= process_object_response
;
198 slot
->callback_data
= obj_req
;
199 obj_req
->slot
= slot
;
201 curl_easy_setopt(slot
->curl
, CURLOPT_FILE
, obj_req
);
202 curl_easy_setopt(slot
->curl
, CURLOPT_WRITEFUNCTION
, fwrite_sha1_file
);
203 curl_easy_setopt(slot
->curl
, CURLOPT_ERRORBUFFER
, obj_req
->errorstr
);
204 curl_easy_setopt(slot
->curl
, CURLOPT_URL
, url
);
205 curl_easy_setopt(slot
->curl
, CURLOPT_HTTPHEADER
, no_pragma_header
);
208 * If we have successfully processed data from a previous fetch
209 * attempt, only fetch the data we don't already have.
212 if (walker
->get_verbosely
)
214 "Resuming fetch of object %s at byte %ld\n",
216 sprintf(range
, "Range: bytes=%ld-", prev_posn
);
217 range_header
= curl_slist_append(range_header
, range
);
218 curl_easy_setopt(slot
->curl
,
219 CURLOPT_HTTPHEADER
, range_header
);
222 /* Try to get the request started, abort the request on error */
223 obj_req
->state
= ACTIVE
;
224 if (!start_active_slot(slot
)) {
225 obj_req
->state
= ABORTED
;
226 obj_req
->slot
= NULL
;
227 close(obj_req
->local
);
234 static void finish_object_request(struct object_request
*obj_req
)
238 close(obj_req
->local
);
241 if (obj_req
->http_code
== 416) {
242 fprintf(stderr
, "Warning: requested range invalid; we may already have all the data.\n");
243 } else if (obj_req
->curl_result
!= CURLE_OK
) {
244 if (stat(obj_req
->tmpfile
, &st
) == 0)
246 unlink_or_warn(obj_req
->tmpfile
);
250 git_inflate_end(&obj_req
->stream
);
251 git_SHA1_Final(obj_req
->real_sha1
, &obj_req
->c
);
252 if (obj_req
->zret
!= Z_STREAM_END
) {
253 unlink_or_warn(obj_req
->tmpfile
);
256 if (hashcmp(obj_req
->sha1
, obj_req
->real_sha1
)) {
257 unlink_or_warn(obj_req
->tmpfile
);
261 move_temp_to_file(obj_req
->tmpfile
, obj_req
->filename
);
263 if (obj_req
->rename
== 0)
264 walker_say(obj_req
->walker
, "got %s\n", sha1_to_hex(obj_req
->sha1
));
267 static void process_object_response(void *callback_data
)
269 struct object_request
*obj_req
=
270 (struct object_request
*)callback_data
;
271 struct walker
*walker
= obj_req
->walker
;
272 struct walker_data
*data
= walker
->data
;
273 struct alt_base
*alt
= data
->alt
;
275 obj_req
->curl_result
= obj_req
->slot
->curl_result
;
276 obj_req
->http_code
= obj_req
->slot
->http_code
;
277 obj_req
->slot
= NULL
;
278 obj_req
->state
= COMPLETE
;
280 /* Use alternates if necessary */
281 if (missing_target(obj_req
)) {
282 fetch_alternates(walker
, alt
->base
);
283 if (obj_req
->repo
->next
!= NULL
) {
286 close(obj_req
->local
);
288 start_object_request(walker
, obj_req
);
293 finish_object_request(obj_req
);
296 static void release_object_request(struct object_request
*obj_req
)
298 struct object_request
*entry
= object_queue_head
;
300 if (obj_req
->local
!= -1)
301 error("fd leakage in release: %d", obj_req
->local
);
302 if (obj_req
== object_queue_head
) {
303 object_queue_head
= obj_req
->next
;
305 while (entry
->next
!= NULL
&& entry
->next
!= obj_req
)
307 if (entry
->next
== obj_req
)
308 entry
->next
= entry
->next
->next
;
315 #ifdef USE_CURL_MULTI
316 static int fill_active_slot(struct walker
*walker
)
318 struct object_request
*obj_req
;
320 for (obj_req
= object_queue_head
; obj_req
; obj_req
= obj_req
->next
) {
321 if (obj_req
->state
== WAITING
) {
322 if (has_sha1_file(obj_req
->sha1
))
323 obj_req
->state
= COMPLETE
;
325 start_object_request(walker
, obj_req
);
334 static void prefetch(struct walker
*walker
, unsigned char *sha1
)
336 struct object_request
*newreq
;
337 struct object_request
*tail
;
338 struct walker_data
*data
= walker
->data
;
339 char *filename
= sha1_file_name(sha1
);
341 newreq
= xmalloc(sizeof(*newreq
));
342 newreq
->walker
= walker
;
343 hashcpy(newreq
->sha1
, sha1
);
344 newreq
->repo
= data
->alt
;
347 newreq
->state
= WAITING
;
348 snprintf(newreq
->filename
, sizeof(newreq
->filename
), "%s", filename
);
349 snprintf(newreq
->tmpfile
, sizeof(newreq
->tmpfile
),
350 "%s.temp", filename
);
354 http_is_verbose
= walker
->get_verbosely
;
356 if (object_queue_head
== NULL
) {
357 object_queue_head
= newreq
;
359 tail
= object_queue_head
;
360 while (tail
->next
!= NULL
)
365 #ifdef USE_CURL_MULTI
371 static void process_alternates_response(void *callback_data
)
373 struct alternates_request
*alt_req
=
374 (struct alternates_request
*)callback_data
;
375 struct walker
*walker
= alt_req
->walker
;
376 struct walker_data
*cdata
= walker
->data
;
377 struct active_request_slot
*slot
= alt_req
->slot
;
378 struct alt_base
*tail
= cdata
->alt
;
379 const char *base
= alt_req
->base
;
380 static const char null_byte
= '\0';
384 if (alt_req
->http_specific
) {
385 if (slot
->curl_result
!= CURLE_OK
||
386 !alt_req
->buffer
->len
) {
388 /* Try reusing the slot to get non-http alternates */
389 alt_req
->http_specific
= 0;
390 sprintf(alt_req
->url
, "%s/objects/info/alternates",
392 curl_easy_setopt(slot
->curl
, CURLOPT_URL
,
396 if (slot
->finished
!= NULL
)
397 (*slot
->finished
) = 0;
398 if (!start_active_slot(slot
)) {
399 cdata
->got_alternates
= -1;
401 if (slot
->finished
!= NULL
)
402 (*slot
->finished
) = 1;
406 } else if (slot
->curl_result
!= CURLE_OK
) {
407 if (!missing_target(slot
)) {
408 cdata
->got_alternates
= -1;
413 fwrite_buffer(&null_byte
, 1, 1, alt_req
->buffer
);
414 alt_req
->buffer
->len
--;
415 data
= alt_req
->buffer
->buf
;
417 while (i
< alt_req
->buffer
->len
) {
419 while (posn
< alt_req
->buffer
->len
&& data
[posn
] != '\n')
421 if (data
[posn
] == '\n') {
424 struct alt_base
*newalt
;
426 if (data
[i
] == '/') {
429 * http://git.host/pub/scm/linux.git/
431 * so memcpy(dst, base, serverlen) will
432 * copy up to "...git.host".
434 const char *colon_ss
= strstr(base
,"://");
436 serverlen
= (strchr(colon_ss
+ 3, '/')
440 } else if (!memcmp(data
+ i
, "../", 3)) {
442 * Relative URL; chop the corresponding
443 * number of subpath from base (and ../
444 * from data), and concatenate the result.
446 * The code first drops ../ from data, and
447 * then drops one ../ from data and one path
448 * from base. IOW, one extra ../ is dropped
449 * from data than path is dropped from base.
451 * This is not wrong. The alternate in
452 * http://git.host/pub/scm/linux.git/
454 * http://git.host/pub/scm/linus.git/
455 * is ../../linus.git/objects/. You need
456 * two ../../ to borrow from your direct
460 serverlen
= strlen(base
);
461 while (i
+ 2 < posn
&&
462 !memcmp(data
+ i
, "../", 3)) {
465 } while (serverlen
&&
466 base
[serverlen
- 1] != '/');
469 /* If the server got removed, give up. */
470 okay
= strchr(base
, ':') - base
+ 3 <
472 } else if (alt_req
->http_specific
) {
473 char *colon
= strchr(data
+ i
, ':');
474 char *slash
= strchr(data
+ i
, '/');
475 if (colon
&& slash
&& colon
< data
+ posn
&&
476 slash
< data
+ posn
&& colon
< slash
) {
480 /* skip "objects\n" at end */
482 target
= xmalloc(serverlen
+ posn
- i
- 6);
483 memcpy(target
, base
, serverlen
);
484 memcpy(target
+ serverlen
, data
+ i
,
486 target
[serverlen
+ posn
- i
- 7] = 0;
487 if (walker
->get_verbosely
)
489 "Also look at %s\n", target
);
490 newalt
= xmalloc(sizeof(*newalt
));
492 newalt
->base
= target
;
493 newalt
->got_indices
= 0;
494 newalt
->packs
= NULL
;
496 while (tail
->next
!= NULL
)
504 cdata
->got_alternates
= 1;
507 static void fetch_alternates(struct walker
*walker
, const char *base
)
509 struct strbuf buffer
= STRBUF_INIT
;
511 struct active_request_slot
*slot
;
512 struct alternates_request alt_req
;
513 struct walker_data
*cdata
= walker
->data
;
516 * If another request has already started fetching alternates,
517 * wait for them to arrive and return to processing this request's
520 #ifdef USE_CURL_MULTI
521 while (cdata
->got_alternates
== 0) {
526 /* Nothing to do if they've already been fetched */
527 if (cdata
->got_alternates
== 1)
530 /* Start the fetch */
531 cdata
->got_alternates
= 0;
533 if (walker
->get_verbosely
)
534 fprintf(stderr
, "Getting alternates list for %s\n", base
);
536 url
= xmalloc(strlen(base
) + 31);
537 sprintf(url
, "%s/objects/info/http-alternates", base
);
540 * Use a callback to process the result, since another request
541 * may fail and need to have alternates loaded before continuing
543 slot
= get_active_slot();
544 slot
->callback_func
= process_alternates_response
;
545 alt_req
.walker
= walker
;
546 slot
->callback_data
= &alt_req
;
548 curl_easy_setopt(slot
->curl
, CURLOPT_FILE
, &buffer
);
549 curl_easy_setopt(slot
->curl
, CURLOPT_WRITEFUNCTION
, fwrite_buffer
);
550 curl_easy_setopt(slot
->curl
, CURLOPT_URL
, url
);
554 alt_req
.buffer
= &buffer
;
555 alt_req
.http_specific
= 1;
558 if (start_active_slot(slot
))
559 run_active_slot(slot
);
561 cdata
->got_alternates
= -1;
563 strbuf_release(&buffer
);
567 static int fetch_indices(struct walker
*walker
, struct alt_base
*repo
)
571 if (repo
->got_indices
)
574 if (walker
->get_verbosely
)
575 fprintf(stderr
, "Getting pack list for %s\n", repo
->base
);
577 switch (http_get_info_packs(repo
->base
, &repo
->packs
)) {
579 case HTTP_MISSING_TARGET
:
580 repo
->got_indices
= 1;
584 repo
->got_indices
= 0;
591 static int fetch_pack(struct walker
*walker
, struct alt_base
*repo
, unsigned char *sha1
)
594 struct packed_git
*target
;
595 struct packed_git
**lst
;
598 char tmpfile
[PATH_MAX
];
601 char range
[RANGE_HEADER_SIZE
];
602 struct curl_slist
*range_header
= NULL
;
604 struct active_request_slot
*slot
;
605 struct slot_results results
;
607 if (fetch_indices(walker
, repo
))
609 target
= find_sha1_pack(sha1
, repo
->packs
);
613 if (walker
->get_verbosely
) {
614 fprintf(stderr
, "Getting pack %s\n",
615 sha1_to_hex(target
->sha1
));
616 fprintf(stderr
, " which contains %s\n",
620 url
= xmalloc(strlen(repo
->base
) + 65);
621 sprintf(url
, "%s/objects/pack/pack-%s.pack",
622 repo
->base
, sha1_to_hex(target
->sha1
));
624 filename
= sha1_pack_name(target
->sha1
);
625 snprintf(tmpfile
, sizeof(tmpfile
), "%s.temp", filename
);
626 packfile
= fopen(tmpfile
, "a");
628 return error("Unable to open local file %s for pack",
631 slot
= get_active_slot();
632 slot
->results
= &results
;
633 curl_easy_setopt(slot
->curl
, CURLOPT_FILE
, packfile
);
634 curl_easy_setopt(slot
->curl
, CURLOPT_WRITEFUNCTION
, fwrite
);
635 curl_easy_setopt(slot
->curl
, CURLOPT_URL
, url
);
636 curl_easy_setopt(slot
->curl
, CURLOPT_HTTPHEADER
, no_pragma_header
);
637 slot
->local
= packfile
;
640 * If there is data present from a previous transfer attempt,
641 * resume where it left off
643 prev_posn
= ftell(packfile
);
645 if (walker
->get_verbosely
)
647 "Resuming fetch of pack %s at byte %ld\n",
648 sha1_to_hex(target
->sha1
), prev_posn
);
649 sprintf(range
, "Range: bytes=%ld-", prev_posn
);
650 range_header
= curl_slist_append(range_header
, range
);
651 curl_easy_setopt(slot
->curl
, CURLOPT_HTTPHEADER
, range_header
);
654 if (start_active_slot(slot
)) {
655 run_active_slot(slot
);
656 if (results
.curl_result
!= CURLE_OK
) {
659 return error("Unable to get pack file %s\n%s", url
,
665 return error("Unable to start request");
668 target
->pack_size
= ftell(packfile
);
672 ret
= move_temp_to_file(tmpfile
, filename
);
677 while (*lst
!= target
)
678 lst
= &((*lst
)->next
);
681 if (verify_pack(target
))
683 install_packed_git(target
);
688 static void abort_object_request(struct object_request
*obj_req
)
690 if (obj_req
->local
>= 0) {
691 close(obj_req
->local
);
694 unlink_or_warn(obj_req
->tmpfile
);
696 release_active_slot(obj_req
->slot
);
697 obj_req
->slot
= NULL
;
699 release_object_request(obj_req
);
702 static int fetch_object(struct walker
*walker
, struct alt_base
*repo
, unsigned char *sha1
)
704 char *hex
= sha1_to_hex(sha1
);
706 struct object_request
*obj_req
= object_queue_head
;
708 while (obj_req
!= NULL
&& hashcmp(obj_req
->sha1
, sha1
))
709 obj_req
= obj_req
->next
;
711 return error("Couldn't find request for %s in the queue", hex
);
713 if (has_sha1_file(obj_req
->sha1
)) {
714 abort_object_request(obj_req
);
718 #ifdef USE_CURL_MULTI
719 while (obj_req
->state
== WAITING
)
722 start_object_request(walker
, obj_req
);
725 while (obj_req
->state
== ACTIVE
)
726 run_active_slot(obj_req
->slot
);
728 if (obj_req
->local
!= -1) {
729 close(obj_req
->local
);
733 if (obj_req
->state
== ABORTED
) {
734 ret
= error("Request for %s aborted", hex
);
735 } else if (obj_req
->curl_result
!= CURLE_OK
&&
736 obj_req
->http_code
!= 416) {
737 if (missing_target(obj_req
))
738 ret
= -1; /* Be silent, it is probably in a pack. */
740 ret
= error("%s (curl_result = %d, http_code = %ld, sha1 = %s)",
741 obj_req
->errorstr
, obj_req
->curl_result
,
742 obj_req
->http_code
, hex
);
743 } else if (obj_req
->zret
!= Z_STREAM_END
) {
744 walker
->corrupt_object_found
++;
745 ret
= error("File %s (%s) corrupt", hex
, obj_req
->url
);
746 } else if (hashcmp(obj_req
->sha1
, obj_req
->real_sha1
)) {
747 ret
= error("File %s has bad hash", hex
);
748 } else if (obj_req
->rename
< 0) {
749 ret
= error("unable to write sha1 filename %s",
753 release_object_request(obj_req
);
757 static int fetch(struct walker
*walker
, unsigned char *sha1
)
759 struct walker_data
*data
= walker
->data
;
760 struct alt_base
*altbase
= data
->alt
;
762 if (!fetch_object(walker
, altbase
, sha1
))
765 if (!fetch_pack(walker
, altbase
, sha1
))
767 fetch_alternates(walker
, data
->alt
->base
);
768 altbase
= altbase
->next
;
770 return error("Unable to find %s under %s", sha1_to_hex(sha1
),
774 static int fetch_ref(struct walker
*walker
, struct ref
*ref
)
776 struct walker_data
*data
= walker
->data
;
777 return http_fetch_ref(data
->alt
->base
, ref
);
780 static void cleanup(struct walker
*walker
)
785 struct walker
*get_http_walker(const char *url
, struct remote
*remote
)
788 struct walker_data
*data
= xmalloc(sizeof(struct walker_data
));
789 struct walker
*walker
= xmalloc(sizeof(struct walker
));
793 data
->alt
= xmalloc(sizeof(*data
->alt
));
794 data
->alt
->base
= xmalloc(strlen(url
) + 1);
795 strcpy(data
->alt
->base
, url
);
796 for (s
= data
->alt
->base
+ strlen(data
->alt
->base
) - 1; *s
== '/'; --s
)
799 data
->alt
->got_indices
= 0;
800 data
->alt
->packs
= NULL
;
801 data
->alt
->next
= NULL
;
802 data
->got_alternates
= -1;
804 walker
->corrupt_object_found
= 0;
805 walker
->fetch
= fetch
;
806 walker
->fetch_ref
= fetch_ref
;
807 walker
->prefetch
= prefetch
;
808 walker
->cleanup
= cleanup
;
811 #ifdef USE_CURL_MULTI
812 add_fill_function(walker
, (int (*)(void *)) fill_active_slot
);