9 #if LIBCURL_VERSION_NUM >= 0x070908
10 #define USE_CURL_MULTI
11 #define DEFAULT_MAX_REQUESTS 5
14 #if LIBCURL_VERSION_NUM < 0x070704
15 #define curl_global_cleanup() do { /* nothing */ } while(0)
17 #if LIBCURL_VERSION_NUM < 0x070800
18 #define curl_global_init(a) do { /* nothing */ } while(0)
21 #if LIBCURL_VERSION_NUM < 0x070c04
22 #define NO_CURL_EASY_DUPHANDLE
25 #define PREV_BUF_SIZE 4096
26 #define RANGE_HEADER_SIZE 30
28 static int active_requests
= 0;
29 static int data_received
;
32 static int max_requests
= -1;
35 #ifndef NO_CURL_EASY_DUPHANDLE
36 static CURL
*curl_default
;
38 static struct curl_slist
*pragma_header
;
39 static struct curl_slist
*no_pragma_header
;
40 static struct curl_slist
*no_range_header
;
41 static char curl_errorstr
[CURL_ERROR_SIZE
];
47 struct packed_git
*packs
;
48 struct alt_base
*next
;
51 static struct alt_base
*alt
= NULL
;
60 struct transfer_request
62 unsigned char sha1
[20];
63 struct alt_base
*repo
;
65 char filename
[PATH_MAX
];
66 char tmpfile
[PATH_MAX
];
68 enum transfer_state state
;
70 char errorstr
[CURL_ERROR_SIZE
];
72 unsigned char real_sha1
[20];
77 struct active_request_slot
*slot
;
78 struct transfer_request
*next
;
81 struct active_request_slot
88 struct active_request_slot
*next
;
91 static struct transfer_request
*request_queue_head
= NULL
;
92 static struct active_request_slot
*active_queue_head
= NULL
;
94 static int curl_ssl_verify
= -1;
95 static char *ssl_cert
= NULL
;
96 #if LIBCURL_VERSION_NUM >= 0x070902
97 static char *ssl_key
= NULL
;
99 #if LIBCURL_VERSION_NUM >= 0x070908
100 static char *ssl_capath
= NULL
;
102 static char *ssl_cainfo
= NULL
;
103 static long curl_low_speed_limit
= -1;
104 static long curl_low_speed_time
= -1;
113 static int http_options(const char *var
, const char *value
)
115 if (!strcmp("http.sslverify", var
)) {
116 if (curl_ssl_verify
== -1) {
117 curl_ssl_verify
= git_config_bool(var
, value
);
122 if (!strcmp("http.sslcert", var
)) {
123 if (ssl_cert
== NULL
) {
124 ssl_cert
= xmalloc(strlen(value
)+1);
125 strcpy(ssl_cert
, value
);
129 #if LIBCURL_VERSION_NUM >= 0x070902
130 if (!strcmp("http.sslkey", var
)) {
131 if (ssl_key
== NULL
) {
132 ssl_key
= xmalloc(strlen(value
)+1);
133 strcpy(ssl_key
, value
);
138 #if LIBCURL_VERSION_NUM >= 0x070908
139 if (!strcmp("http.sslcapath", var
)) {
140 if (ssl_capath
== NULL
) {
141 ssl_capath
= xmalloc(strlen(value
)+1);
142 strcpy(ssl_capath
, value
);
147 if (!strcmp("http.sslcainfo", var
)) {
148 if (ssl_cainfo
== NULL
) {
149 ssl_cainfo
= xmalloc(strlen(value
)+1);
150 strcpy(ssl_cainfo
, value
);
155 #ifdef USE_CURL_MULTI
156 if (!strcmp("http.maxrequests", var
)) {
157 if (max_requests
== -1)
158 max_requests
= git_config_int(var
, value
);
163 if (!strcmp("http.lowspeedlimit", var
)) {
164 if (curl_low_speed_limit
== -1)
165 curl_low_speed_limit
= (long)git_config_int(var
, value
);
168 if (!strcmp("http.lowspeedtime", var
)) {
169 if (curl_low_speed_time
== -1)
170 curl_low_speed_time
= (long)git_config_int(var
, value
);
174 /* Fall back on the default ones */
175 return git_default_config(var
, value
);
178 static size_t fwrite_buffer(void *ptr
, size_t eltsize
, size_t nmemb
,
179 struct buffer
*buffer
)
181 size_t size
= eltsize
* nmemb
;
182 if (size
> buffer
->size
- buffer
->posn
)
183 size
= buffer
->size
- buffer
->posn
;
184 memcpy(buffer
->buffer
+ buffer
->posn
, ptr
, size
);
185 buffer
->posn
+= size
;
190 static size_t fwrite_buffer_dynamic(const void *ptr
, size_t eltsize
,
191 size_t nmemb
, struct buffer
*buffer
)
193 size_t size
= eltsize
* nmemb
;
194 if (size
> buffer
->size
- buffer
->posn
) {
195 buffer
->size
= buffer
->size
* 3 / 2;
196 if (buffer
->size
< buffer
->posn
+ size
)
197 buffer
->size
= buffer
->posn
+ size
;
198 buffer
->buffer
= xrealloc(buffer
->buffer
, buffer
->size
);
200 memcpy(buffer
->buffer
+ buffer
->posn
, ptr
, size
);
201 buffer
->posn
+= size
;
206 static size_t fwrite_sha1_file(void *ptr
, size_t eltsize
, size_t nmemb
,
209 unsigned char expn
[4096];
210 size_t size
= eltsize
* nmemb
;
212 struct transfer_request
*request
= (struct transfer_request
*)data
;
214 ssize_t retval
= write(request
->local
,
215 ptr
+ posn
, size
- posn
);
219 } while (posn
< size
);
221 request
->stream
.avail_in
= size
;
222 request
->stream
.next_in
= ptr
;
224 request
->stream
.next_out
= expn
;
225 request
->stream
.avail_out
= sizeof(expn
);
226 request
->zret
= inflate(&request
->stream
, Z_SYNC_FLUSH
);
227 SHA1_Update(&request
->c
, expn
,
228 sizeof(expn
) - request
->stream
.avail_out
);
229 } while (request
->stream
.avail_in
&& request
->zret
== Z_OK
);
234 #ifdef USE_CURL_MULTI
235 static void process_curl_messages(void);
236 static void process_request_queue(void);
239 static CURL
* get_curl_handle(void)
241 CURL
* result
= curl_easy_init();
243 curl_easy_setopt(result
, CURLOPT_SSL_VERIFYPEER
, curl_ssl_verify
);
244 #if LIBCURL_VERSION_NUM >= 0x070907
245 curl_easy_setopt(result
, CURLOPT_NETRC
, CURL_NETRC_OPTIONAL
);
248 if (ssl_cert
!= NULL
)
249 curl_easy_setopt(result
, CURLOPT_SSLCERT
, ssl_cert
);
250 #if LIBCURL_VERSION_NUM >= 0x070902
252 curl_easy_setopt(result
, CURLOPT_SSLKEY
, ssl_key
);
254 #if LIBCURL_VERSION_NUM >= 0x070908
255 if (ssl_capath
!= NULL
)
256 curl_easy_setopt(result
, CURLOPT_CAPATH
, ssl_capath
);
258 if (ssl_cainfo
!= NULL
)
259 curl_easy_setopt(result
, CURLOPT_CAINFO
, ssl_cainfo
);
260 curl_easy_setopt(result
, CURLOPT_FAILONERROR
, 1);
262 if (curl_low_speed_limit
> 0 && curl_low_speed_time
> 0) {
263 curl_easy_setopt(result
, CURLOPT_LOW_SPEED_LIMIT
,
264 curl_low_speed_limit
);
265 curl_easy_setopt(result
, CURLOPT_LOW_SPEED_TIME
,
266 curl_low_speed_time
);
272 static struct active_request_slot
*get_active_slot(void)
274 struct active_request_slot
*slot
= active_queue_head
;
275 struct active_request_slot
*newslot
;
277 #ifdef USE_CURL_MULTI
280 /* Wait for a slot to open up if the queue is full */
281 while (active_requests
>= max_requests
) {
282 curl_multi_perform(curlm
, &num_transfers
);
283 if (num_transfers
< active_requests
) {
284 process_curl_messages();
289 while (slot
!= NULL
&& slot
->in_use
) {
293 newslot
= xmalloc(sizeof(*newslot
));
294 newslot
->curl
= NULL
;
296 newslot
->next
= NULL
;
298 slot
= active_queue_head
;
300 active_queue_head
= newslot
;
302 while (slot
->next
!= NULL
) {
305 slot
->next
= newslot
;
310 if (slot
->curl
== NULL
) {
311 #ifdef NO_CURL_EASY_DUPHANDLE
312 slot
->curl
= get_curl_handle();
314 slot
->curl
= curl_easy_duphandle(curl_default
);
322 curl_easy_setopt(slot
->curl
, CURLOPT_HTTPHEADER
, pragma_header
);
323 curl_easy_setopt(slot
->curl
, CURLOPT_HTTPHEADER
, no_range_header
);
324 curl_easy_setopt(slot
->curl
, CURLOPT_ERRORBUFFER
, curl_errorstr
);
329 static int start_active_slot(struct active_request_slot
*slot
)
331 #ifdef USE_CURL_MULTI
332 CURLMcode curlm_result
= curl_multi_add_handle(curlm
, slot
->curl
);
334 if (curlm_result
!= CURLM_OK
&&
335 curlm_result
!= CURLM_CALL_MULTI_PERFORM
) {
344 static void run_active_slot(struct active_request_slot
*slot
)
346 #ifdef USE_CURL_MULTI
354 struct timeval select_timeout
;
355 CURLMcode curlm_result
;
357 while (!slot
->done
) {
360 curlm_result
= curl_multi_perform(curlm
,
362 } while (curlm_result
== CURLM_CALL_MULTI_PERFORM
);
363 if (num_transfers
< active_requests
) {
364 process_curl_messages();
365 process_request_queue();
368 if (!data_received
&& slot
->local
!= NULL
) {
369 current_pos
= ftell(slot
->local
);
370 if (current_pos
> last_pos
)
372 last_pos
= current_pos
;
375 if (!slot
->done
&& !data_received
) {
380 select_timeout
.tv_sec
= 0;
381 select_timeout
.tv_usec
= 50000;
382 select(max_fd
, &readfds
, &writefds
,
383 &excfds
, &select_timeout
);
387 slot
->curl_result
= curl_easy_perform(slot
->curl
);
392 static void start_request(struct transfer_request
*request
)
394 char *hex
= sha1_to_hex(request
->sha1
);
395 char prevfile
[PATH_MAX
];
399 unsigned char prev_buf
[PREV_BUF_SIZE
];
400 ssize_t prev_read
= 0;
402 char range
[RANGE_HEADER_SIZE
];
403 struct curl_slist
*range_header
= NULL
;
404 struct active_request_slot
*slot
;
406 snprintf(prevfile
, sizeof(prevfile
), "%s.prev", request
->filename
);
408 rename(request
->tmpfile
, prevfile
);
409 unlink(request
->tmpfile
);
411 request
->local
= open(request
->tmpfile
,
412 O_WRONLY
| O_CREAT
| O_EXCL
, 0666);
413 /* This could have failed due to the "lazy directory creation";
414 * try to mkdir the last path component.
416 if (request
->local
< 0 && errno
== ENOENT
) {
417 char *dir
= strrchr(request
->tmpfile
, '/');
420 mkdir(request
->tmpfile
, 0777);
423 request
->local
= open(request
->tmpfile
,
424 O_WRONLY
| O_CREAT
| O_EXCL
, 0666);
427 if (request
->local
< 0) {
428 request
->state
= ABORTED
;
429 error("Couldn't create temporary file %s for %s: %s\n",
430 request
->tmpfile
, request
->filename
, strerror(errno
));
434 memset(&request
->stream
, 0, sizeof(request
->stream
));
436 inflateInit(&request
->stream
);
438 SHA1_Init(&request
->c
);
440 url
= xmalloc(strlen(request
->repo
->base
) + 50);
441 request
->url
= xmalloc(strlen(request
->repo
->base
) + 50);
442 strcpy(url
, request
->repo
->base
);
443 posn
= url
+ strlen(request
->repo
->base
);
444 strcpy(posn
, "objects/");
446 memcpy(posn
, hex
, 2);
449 strcpy(posn
, hex
+ 2);
450 strcpy(request
->url
, url
);
452 /* If a previous temp file is present, process what was already
454 prevlocal
= open(prevfile
, O_RDONLY
);
455 if (prevlocal
!= -1) {
457 prev_read
= read(prevlocal
, prev_buf
, PREV_BUF_SIZE
);
459 if (fwrite_sha1_file(prev_buf
,
462 request
) == prev_read
) {
463 prev_posn
+= prev_read
;
468 } while (prev_read
> 0);
473 /* Reset inflate/SHA1 if there was an error reading the previous temp
474 file; also rewind to the beginning of the local file. */
475 if (prev_read
== -1) {
476 memset(&request
->stream
, 0, sizeof(request
->stream
));
477 inflateInit(&request
->stream
);
478 SHA1_Init(&request
->c
);
481 lseek(request
->local
, SEEK_SET
, 0);
482 ftruncate(request
->local
, 0);
486 slot
= get_active_slot();
487 curl_easy_setopt(slot
->curl
, CURLOPT_FILE
, request
);
488 curl_easy_setopt(slot
->curl
, CURLOPT_WRITEFUNCTION
, fwrite_sha1_file
);
489 curl_easy_setopt(slot
->curl
, CURLOPT_ERRORBUFFER
, request
->errorstr
);
490 curl_easy_setopt(slot
->curl
, CURLOPT_URL
, url
);
491 curl_easy_setopt(slot
->curl
, CURLOPT_HTTPHEADER
, no_pragma_header
);
493 /* If we have successfully processed data from a previous fetch
494 attempt, only fetch the data we don't already have. */
498 "Resuming fetch of object %s at byte %ld\n",
500 sprintf(range
, "Range: bytes=%ld-", prev_posn
);
501 range_header
= curl_slist_append(range_header
, range
);
502 curl_easy_setopt(slot
->curl
,
503 CURLOPT_HTTPHEADER
, range_header
);
506 /* Try to get the request started, abort the request on error */
507 if (!start_active_slot(slot
)) {
508 request
->state
= ABORTED
;
509 close(request
->local
);
514 request
->slot
= slot
;
515 request
->state
= ACTIVE
;
518 static void finish_request(struct transfer_request
*request
)
520 fchmod(request
->local
, 0444);
521 close(request
->local
);
523 if (request
->http_code
== 416) {
524 fprintf(stderr
, "Warning: requested range invalid; we may already have all the data.\n");
525 } else if (request
->curl_result
!= CURLE_OK
) {
529 inflateEnd(&request
->stream
);
530 SHA1_Final(request
->real_sha1
, &request
->c
);
531 if (request
->zret
!= Z_STREAM_END
) {
532 unlink(request
->tmpfile
);
535 if (memcmp(request
->sha1
, request
->real_sha1
, 20)) {
536 unlink(request
->tmpfile
);
540 move_temp_to_file(request
->tmpfile
, request
->filename
);
542 if (request
->rename
== 0)
543 pull_say("got %s\n", sha1_to_hex(request
->sha1
));
546 static void release_request(struct transfer_request
*request
)
548 struct transfer_request
*entry
= request_queue_head
;
550 if (request
== request_queue_head
) {
551 request_queue_head
= request
->next
;
553 while (entry
->next
!= NULL
&& entry
->next
!= request
)
555 if (entry
->next
== request
)
556 entry
->next
= entry
->next
->next
;
563 #ifdef USE_CURL_MULTI
564 void process_curl_messages(void)
567 struct active_request_slot
*slot
;
568 struct transfer_request
*request
= NULL
;
569 CURLMsg
*curl_message
= curl_multi_info_read(curlm
, &num_messages
);
571 while (curl_message
!= NULL
) {
572 if (curl_message
->msg
== CURLMSG_DONE
) {
573 slot
= active_queue_head
;
574 while (slot
!= NULL
&&
575 slot
->curl
!= curl_message
->easy_handle
)
578 curl_multi_remove_handle(curlm
, slot
->curl
);
582 slot
->curl_result
= curl_message
->data
.result
;
583 request
= request_queue_head
;
584 while (request
!= NULL
&&
585 request
->slot
!= slot
)
586 request
= request
->next
;
588 fprintf(stderr
, "Received DONE message for unknown request!\n");
590 if (request
!= NULL
) {
591 request
->curl_result
=
592 curl_message
->data
.result
;
593 curl_easy_getinfo(slot
->curl
,
595 &request
->http_code
);
596 request
->slot
= NULL
;
598 /* Use alternates if necessary */
599 if (request
->http_code
== 404 &&
600 request
->repo
->next
!= NULL
) {
601 request
->repo
= request
->repo
->next
;
602 start_request(request
);
604 finish_request(request
);
605 request
->state
= COMPLETE
;
609 fprintf(stderr
, "Unknown CURL message received: %d\n",
610 (int)curl_message
->msg
);
612 curl_message
= curl_multi_info_read(curlm
, &num_messages
);
616 void process_request_queue(void)
618 struct transfer_request
*request
= request_queue_head
;
619 struct active_request_slot
*slot
= active_queue_head
;
622 while (active_requests
< max_requests
&& request
!= NULL
) {
623 if (request
->state
== WAITING
) {
624 if (has_sha1_file(request
->sha1
))
625 release_request(request
);
627 start_request(request
);
628 curl_multi_perform(curlm
, &num_transfers
);
630 request
= request
->next
;
633 while (slot
!= NULL
) {
634 if (!slot
->in_use
&& slot
->curl
!= NULL
) {
635 curl_easy_cleanup(slot
->curl
);
643 void prefetch(unsigned char *sha1
)
645 struct transfer_request
*newreq
;
646 struct transfer_request
*tail
;
647 char *filename
= sha1_file_name(sha1
);
649 newreq
= xmalloc(sizeof(*newreq
));
650 memcpy(newreq
->sha1
, sha1
, 20);
654 newreq
->state
= WAITING
;
655 snprintf(newreq
->filename
, sizeof(newreq
->filename
), "%s", filename
);
656 snprintf(newreq
->tmpfile
, sizeof(newreq
->tmpfile
),
657 "%s.temp", filename
);
660 if (request_queue_head
== NULL
) {
661 request_queue_head
= newreq
;
663 tail
= request_queue_head
;
664 while (tail
->next
!= NULL
) {
669 #ifdef USE_CURL_MULTI
670 process_request_queue();
671 process_curl_messages();
675 static int fetch_index(struct alt_base
*repo
, unsigned char *sha1
)
677 char *hex
= sha1_to_hex(sha1
);
680 char tmpfile
[PATH_MAX
];
682 char range
[RANGE_HEADER_SIZE
];
683 struct curl_slist
*range_header
= NULL
;
686 struct active_request_slot
*slot
;
688 if (has_pack_index(sha1
))
692 fprintf(stderr
, "Getting index for pack %s\n", hex
);
694 url
= xmalloc(strlen(repo
->base
) + 64);
695 sprintf(url
, "%s/objects/pack/pack-%s.idx", repo
->base
, hex
);
697 filename
= sha1_pack_index_name(sha1
);
698 snprintf(tmpfile
, sizeof(tmpfile
), "%s.temp", filename
);
699 indexfile
= fopen(tmpfile
, "a");
701 return error("Unable to open local file %s for pack index",
704 slot
= get_active_slot();
705 curl_easy_setopt(slot
->curl
, CURLOPT_FILE
, indexfile
);
706 curl_easy_setopt(slot
->curl
, CURLOPT_WRITEFUNCTION
, fwrite
);
707 curl_easy_setopt(slot
->curl
, CURLOPT_URL
, url
);
708 curl_easy_setopt(slot
->curl
, CURLOPT_HTTPHEADER
, no_pragma_header
);
709 slot
->local
= indexfile
;
711 /* If there is data present from a previous transfer attempt,
712 resume where it left off */
713 prev_posn
= ftell(indexfile
);
717 "Resuming fetch of index for pack %s at byte %ld\n",
719 sprintf(range
, "Range: bytes=%ld-", prev_posn
);
720 range_header
= curl_slist_append(range_header
, range
);
721 curl_easy_setopt(slot
->curl
, CURLOPT_HTTPHEADER
, range_header
);
724 if (start_active_slot(slot
)) {
725 run_active_slot(slot
);
726 if (slot
->curl_result
!= CURLE_OK
) {
728 return error("Unable to get pack index %s\n%s", url
,
732 return error("Unable to start request");
737 return move_temp_to_file(tmpfile
, filename
);
740 static int setup_index(struct alt_base
*repo
, unsigned char *sha1
)
742 struct packed_git
*new_pack
;
743 if (has_pack_file(sha1
))
744 return 0; // don't list this as something we can get
746 if (fetch_index(repo
, sha1
))
749 new_pack
= parse_pack_index(sha1
);
750 new_pack
->next
= repo
->packs
;
751 repo
->packs
= new_pack
;
755 static int fetch_alternates(char *base
)
758 struct buffer buffer
;
762 int http_specific
= 1;
763 struct alt_base
*tail
= alt
;
764 static const char null_byte
= '\0';
766 struct active_request_slot
*slot
;
768 data
= xmalloc(4096);
771 buffer
.buffer
= data
;
774 fprintf(stderr
, "Getting alternates list\n");
776 url
= xmalloc(strlen(base
) + 31);
777 sprintf(url
, "%s/objects/info/http-alternates", base
);
779 slot
= get_active_slot();
780 curl_easy_setopt(slot
->curl
, CURLOPT_FILE
, &buffer
);
781 curl_easy_setopt(slot
->curl
, CURLOPT_WRITEFUNCTION
,
782 fwrite_buffer_dynamic
);
783 curl_easy_setopt(slot
->curl
, CURLOPT_URL
, url
);
784 if (start_active_slot(slot
)) {
785 run_active_slot(slot
);
786 if (slot
->curl_result
!= CURLE_OK
|| !buffer
.posn
) {
789 sprintf(url
, "%s/objects/info/alternates", base
);
791 slot
= get_active_slot();
792 curl_easy_setopt(slot
->curl
, CURLOPT_FILE
, &buffer
);
793 curl_easy_setopt(slot
->curl
, CURLOPT_WRITEFUNCTION
,
794 fwrite_buffer_dynamic
);
795 curl_easy_setopt(slot
->curl
, CURLOPT_URL
, url
);
796 if (start_active_slot(slot
)) {
797 run_active_slot(slot
);
798 if (slot
->curl_result
!= CURLE_OK
) {
809 fwrite_buffer_dynamic(&null_byte
, 1, 1, &buffer
);
811 data
= buffer
.buffer
;
813 while (i
< buffer
.posn
) {
815 while (posn
< buffer
.posn
&& data
[posn
] != '\n')
817 if (data
[posn
] == '\n') {
820 struct alt_base
*newalt
;
822 if (data
[i
] == '/') {
823 serverlen
= strchr(base
+ 8, '/') - base
;
825 } else if (!memcmp(data
+ i
, "../", 3)) {
827 serverlen
= strlen(base
);
828 while (i
+ 2 < posn
&&
829 !memcmp(data
+ i
, "../", 3)) {
832 } while (serverlen
&&
833 base
[serverlen
- 1] != '/');
836 // If the server got removed, give up.
837 okay
= strchr(base
, ':') - base
+ 3 <
839 } else if (http_specific
) {
840 char *colon
= strchr(data
+ i
, ':');
841 char *slash
= strchr(data
+ i
, '/');
842 if (colon
&& slash
&& colon
< data
+ posn
&&
843 slash
< data
+ posn
&& colon
< slash
) {
847 // skip 'objects' at end
849 target
= xmalloc(serverlen
+ posn
- i
- 6);
850 strncpy(target
, base
, serverlen
);
851 strncpy(target
+ serverlen
, data
+ i
,
853 target
[serverlen
+ posn
- i
- 7] = '\0';
856 "Also look at %s\n", target
);
857 newalt
= xmalloc(sizeof(*newalt
));
859 newalt
->base
= target
;
860 newalt
->got_indices
= 0;
861 newalt
->packs
= NULL
;
862 while (tail
->next
!= NULL
)
875 static int fetch_indices(struct alt_base
*repo
)
877 unsigned char sha1
[20];
879 struct buffer buffer
;
883 struct active_request_slot
*slot
;
885 if (repo
->got_indices
)
888 data
= xmalloc(4096);
891 buffer
.buffer
= data
;
894 fprintf(stderr
, "Getting pack list\n");
896 url
= xmalloc(strlen(repo
->base
) + 21);
897 sprintf(url
, "%s/objects/info/packs", repo
->base
);
899 slot
= get_active_slot();
900 curl_easy_setopt(slot
->curl
, CURLOPT_FILE
, &buffer
);
901 curl_easy_setopt(slot
->curl
, CURLOPT_WRITEFUNCTION
,
902 fwrite_buffer_dynamic
);
903 curl_easy_setopt(slot
->curl
, CURLOPT_URL
, url
);
904 curl_easy_setopt(slot
->curl
, CURLOPT_HTTPHEADER
, NULL
);
905 if (start_active_slot(slot
)) {
906 run_active_slot(slot
);
907 if (slot
->curl_result
!= CURLE_OK
) {
909 return error("%s", curl_errorstr
);
913 return error("Unable to start request");
916 data
= buffer
.buffer
;
917 while (i
< buffer
.posn
) {
921 if (i
+ 52 < buffer
.posn
&&
922 !strncmp(data
+ i
, " pack-", 6) &&
923 !strncmp(data
+ i
+ 46, ".pack\n", 6)) {
924 get_sha1_hex(data
+ i
+ 6, sha1
);
925 setup_index(repo
, sha1
);
930 while (data
[i
] != '\n')
937 repo
->got_indices
= 1;
941 static int fetch_pack(struct alt_base
*repo
, unsigned char *sha1
)
944 struct packed_git
*target
;
945 struct packed_git
**lst
;
948 char tmpfile
[PATH_MAX
];
951 char range
[RANGE_HEADER_SIZE
];
952 struct curl_slist
*range_header
= NULL
;
954 struct active_request_slot
*slot
;
956 if (fetch_indices(repo
))
958 target
= find_sha1_pack(sha1
, repo
->packs
);
963 fprintf(stderr
, "Getting pack %s\n",
964 sha1_to_hex(target
->sha1
));
965 fprintf(stderr
, " which contains %s\n",
969 url
= xmalloc(strlen(repo
->base
) + 65);
970 sprintf(url
, "%s/objects/pack/pack-%s.pack",
971 repo
->base
, sha1_to_hex(target
->sha1
));
973 filename
= sha1_pack_name(target
->sha1
);
974 snprintf(tmpfile
, sizeof(tmpfile
), "%s.temp", filename
);
975 packfile
= fopen(tmpfile
, "a");
977 return error("Unable to open local file %s for pack",
980 slot
= get_active_slot();
981 curl_easy_setopt(slot
->curl
, CURLOPT_FILE
, packfile
);
982 curl_easy_setopt(slot
->curl
, CURLOPT_WRITEFUNCTION
, fwrite
);
983 curl_easy_setopt(slot
->curl
, CURLOPT_URL
, url
);
984 curl_easy_setopt(slot
->curl
, CURLOPT_HTTPHEADER
, no_pragma_header
);
985 slot
->local
= packfile
;
987 /* If there is data present from a previous transfer attempt,
988 resume where it left off */
989 prev_posn
= ftell(packfile
);
993 "Resuming fetch of pack %s at byte %ld\n",
994 sha1_to_hex(target
->sha1
), prev_posn
);
995 sprintf(range
, "Range: bytes=%ld-", prev_posn
);
996 range_header
= curl_slist_append(range_header
, range
);
997 curl_easy_setopt(slot
->curl
, CURLOPT_HTTPHEADER
, range_header
);
1000 if (start_active_slot(slot
)) {
1001 run_active_slot(slot
);
1002 if (slot
->curl_result
!= CURLE_OK
) {
1004 return error("Unable to get pack file %s\n%s", url
,
1008 return error("Unable to start request");
1013 ret
= move_temp_to_file(tmpfile
, filename
);
1018 while (*lst
!= target
)
1019 lst
= &((*lst
)->next
);
1020 *lst
= (*lst
)->next
;
1022 if (verify_pack(target
, 0))
1024 install_packed_git(target
);
1029 static int fetch_object(struct alt_base
*repo
, unsigned char *sha1
)
1031 char *hex
= sha1_to_hex(sha1
);
1033 struct transfer_request
*request
= request_queue_head
;
1035 while (request
!= NULL
&& memcmp(request
->sha1
, sha1
, 20))
1036 request
= request
->next
;
1037 if (request
== NULL
)
1038 return error("Couldn't find request for %s in the queue", hex
);
1040 if (has_sha1_file(request
->sha1
)) {
1041 release_request(request
);
1045 #ifdef USE_CURL_MULTI
1046 while (request
->state
== WAITING
) {
1048 curl_multi_perform(curlm
, &num_transfers
);
1049 if (num_transfers
< active_requests
) {
1050 process_curl_messages();
1051 process_request_queue();
1055 start_request(request
);
1058 while (request
->state
== ACTIVE
) {
1059 run_active_slot(request
->slot
);
1060 #ifndef USE_CURL_MULTI
1061 request
->curl_result
= request
->slot
->curl_result
;
1062 curl_easy_getinfo(request
->slot
->curl
,
1064 &request
->http_code
);
1065 request
->slot
= NULL
;
1067 /* Use alternates if necessary */
1068 if (request
->http_code
== 404 &&
1069 request
->repo
->next
!= NULL
) {
1070 request
->repo
= request
->repo
->next
;
1071 start_request(request
);
1073 finish_request(request
);
1074 request
->state
= COMPLETE
;
1079 if (request
->state
== ABORTED
) {
1080 release_request(request
);
1081 return error("Request for %s aborted", hex
);
1084 if (request
->curl_result
!= CURLE_OK
&& request
->http_code
!= 416) {
1085 ret
= error("%s (curl_result = %d, http_code = %ld, sha1 = %s)",
1086 request
->errorstr
, request
->curl_result
,
1087 request
->http_code
, hex
);
1088 release_request(request
);
1092 if (request
->zret
!= Z_STREAM_END
) {
1093 ret
= error("File %s (%s) corrupt\n", hex
, request
->url
);
1094 release_request(request
);
1098 if (memcmp(request
->sha1
, request
->real_sha1
, 20)) {
1099 release_request(request
);
1100 return error("File %s has bad hash\n", hex
);
1103 if (request
->rename
< 0) {
1104 ret
= error("unable to write sha1 filename %s: %s",
1106 strerror(request
->rename
));
1107 release_request(request
);
1111 release_request(request
);
1115 int fetch(unsigned char *sha1
)
1117 struct alt_base
*altbase
= alt
;
1119 if (!fetch_object(altbase
, sha1
))
1122 if (!fetch_pack(altbase
, sha1
))
1124 altbase
= altbase
->next
;
1126 return error("Unable to find %s under %s\n", sha1_to_hex(sha1
),
1130 static inline int needs_quote(int ch
)
1133 case '/': case '-': case '.':
1134 case 'A'...'Z': case 'a'...'z': case '0'...'9':
1141 static inline int hex(int v
)
1143 if (v
< 10) return '0' + v
;
1144 else return 'A' + v
- 10;
1147 static char *quote_ref_url(const char *base
, const char *ref
)
1151 int len
, baselen
, ch
;
1153 baselen
= strlen(base
);
1154 len
= baselen
+ 6; /* "refs/" + NUL */
1155 for (cp
= ref
; (ch
= *cp
) != 0; cp
++, len
++)
1156 if (needs_quote(ch
))
1157 len
+= 2; /* extra two hex plus replacement % */
1158 qref
= xmalloc(len
);
1159 memcpy(qref
, base
, baselen
);
1160 memcpy(qref
+ baselen
, "refs/", 5);
1161 for (cp
= ref
, dp
= qref
+ baselen
+ 5; (ch
= *cp
) != 0; cp
++) {
1162 if (needs_quote(ch
)) {
1164 *dp
++ = hex((ch
>> 4) & 0xF);
1165 *dp
++ = hex(ch
& 0xF);
1175 int fetch_ref(char *ref
, unsigned char *sha1
)
1179 struct buffer buffer
;
1180 char *base
= alt
->base
;
1181 struct active_request_slot
*slot
;
1184 buffer
.buffer
= hex
;
1187 url
= quote_ref_url(base
, ref
);
1188 slot
= get_active_slot();
1189 curl_easy_setopt(slot
->curl
, CURLOPT_FILE
, &buffer
);
1190 curl_easy_setopt(slot
->curl
, CURLOPT_WRITEFUNCTION
, fwrite_buffer
);
1191 curl_easy_setopt(slot
->curl
, CURLOPT_HTTPHEADER
, NULL
);
1192 curl_easy_setopt(slot
->curl
, CURLOPT_URL
, url
);
1193 if (start_active_slot(slot
)) {
1194 run_active_slot(slot
);
1195 if (slot
->curl_result
!= CURLE_OK
)
1196 return error("Couldn't get %s for %s\n%s",
1197 url
, ref
, curl_errorstr
);
1199 return error("Unable to start request");
1203 get_sha1_hex(hex
, sha1
);
1207 int main(int argc
, char **argv
)
1212 struct active_request_slot
*slot
;
1213 char *low_speed_limit
;
1214 char *low_speed_time
;
1216 while (arg
< argc
&& argv
[arg
][0] == '-') {
1217 if (argv
[arg
][1] == 't') {
1219 } else if (argv
[arg
][1] == 'c') {
1221 } else if (argv
[arg
][1] == 'a') {
1225 } else if (argv
[arg
][1] == 'v') {
1227 } else if (argv
[arg
][1] == 'w') {
1228 write_ref
= argv
[arg
+ 1];
1230 } else if (!strcmp(argv
[arg
], "--recover")) {
1235 if (argc
< arg
+ 2) {
1236 usage("git-http-fetch [-c] [-t] [-a] [-d] [-v] [--recover] [-w ref] commit-id url");
1239 commit_id
= argv
[arg
];
1240 url
= argv
[arg
+ 1];
1242 curl_global_init(CURL_GLOBAL_ALL
);
1244 #ifdef USE_CURL_MULTI
1246 char *http_max_requests
= getenv("GIT_HTTP_MAX_REQUESTS");
1247 if (http_max_requests
!= NULL
)
1248 max_requests
= atoi(http_max_requests
);
1251 curlm
= curl_multi_init();
1252 if (curlm
== NULL
) {
1253 fprintf(stderr
, "Error creating curl multi handle.\n");
1258 if (getenv("GIT_SSL_NO_VERIFY"))
1259 curl_ssl_verify
= 0;
1261 ssl_cert
= getenv("GIT_SSL_CERT");
1262 #if LIBCURL_VERSION_NUM >= 0x070902
1263 ssl_key
= getenv("GIT_SSL_KEY");
1265 #if LIBCURL_VERSION_NUM >= 0x070908
1266 ssl_capath
= getenv("GIT_SSL_CAPATH");
1268 ssl_cainfo
= getenv("GIT_SSL_CAINFO");
1270 low_speed_limit
= getenv("GIT_HTTP_LOW_SPEED_LIMIT");
1271 if (low_speed_limit
!= NULL
)
1272 curl_low_speed_limit
= strtol(low_speed_limit
, NULL
, 10);
1273 low_speed_time
= getenv("GIT_HTTP_LOW_SPEED_TIME");
1274 if (low_speed_time
!= NULL
)
1275 curl_low_speed_time
= strtol(low_speed_time
, NULL
, 10);
1277 git_config(http_options
);
1279 if (curl_ssl_verify
== -1)
1280 curl_ssl_verify
= 1;
1282 #ifdef USE_CURL_MULTI
1283 if (max_requests
< 1)
1284 max_requests
= DEFAULT_MAX_REQUESTS
;
1287 pragma_header
= curl_slist_append(pragma_header
, "Pragma: no-cache");
1288 no_pragma_header
= curl_slist_append(no_pragma_header
, "Pragma:");
1289 no_range_header
= curl_slist_append(no_range_header
, "Range:");
1291 #ifndef NO_CURL_EASY_DUPHANDLE
1292 curl_default
= get_curl_handle();
1295 alt
= xmalloc(sizeof(*alt
));
1297 alt
->got_indices
= 0;
1300 fetch_alternates(alt
->base
);
1302 if (pull(commit_id
))
1305 curl_slist_free_all(pragma_header
);
1306 curl_slist_free_all(no_pragma_header
);
1307 curl_slist_free_all(no_range_header
);
1308 #ifndef NO_CURL_EASY_DUPHANDLE
1309 curl_easy_cleanup(curl_default
);
1311 slot
= active_queue_head
;
1312 while (slot
!= NULL
) {
1313 if (slot
->curl
!= NULL
)
1314 curl_easy_cleanup(slot
->curl
);
1317 #ifdef USE_CURL_MULTI
1318 curl_multi_cleanup(curlm
);
1320 curl_global_cleanup();