Fix for multiple alternates requests in http-fetch
[tgit.git] / http-fetch.c
blob99b6cc7e419f14c3678282725c6a03de3d8947ec
1 #include "cache.h"
2 #include "commit.h"
3 #include "pack.h"
4 #include "fetch.h"
6 #include <curl/curl.h>
7 #include <curl/easy.h>
9 #if LIBCURL_VERSION_NUM >= 0x070908
10 #define USE_CURL_MULTI
11 #define DEFAULT_MAX_REQUESTS 5
12 #endif
14 #if LIBCURL_VERSION_NUM < 0x070704
15 #define curl_global_cleanup() do { /* nothing */ } while(0)
16 #endif
17 #if LIBCURL_VERSION_NUM < 0x070800
18 #define curl_global_init(a) do { /* nothing */ } while(0)
19 #endif
21 #if LIBCURL_VERSION_NUM < 0x070c04
22 #define NO_CURL_EASY_DUPHANDLE
23 #endif
25 #define PREV_BUF_SIZE 4096
26 #define RANGE_HEADER_SIZE 30
28 static int got_alternates = -1;
29 static int active_requests = 0;
30 static int data_received;
32 #ifdef USE_CURL_MULTI
33 static int max_requests = -1;
34 static CURLM *curlm;
35 #endif
36 #ifndef NO_CURL_EASY_DUPHANDLE
37 static CURL *curl_default;
38 #endif
39 static struct curl_slist *pragma_header;
40 static struct curl_slist *no_pragma_header;
41 static struct curl_slist *no_range_header;
42 static char curl_errorstr[CURL_ERROR_SIZE];
44 struct alt_base
46 char *base;
47 int got_indices;
48 struct packed_git *packs;
49 struct alt_base *next;
52 static struct alt_base *alt = NULL;
54 enum transfer_state {
55 WAITING,
56 ABORTED,
57 ACTIVE,
58 COMPLETE,
61 struct transfer_request
63 unsigned char sha1[20];
64 struct alt_base *repo;
65 char *url;
66 char filename[PATH_MAX];
67 char tmpfile[PATH_MAX];
68 int local;
69 enum transfer_state state;
70 CURLcode curl_result;
71 char errorstr[CURL_ERROR_SIZE];
72 long http_code;
73 unsigned char real_sha1[20];
74 SHA_CTX c;
75 z_stream stream;
76 int zret;
77 int rename;
78 struct active_request_slot *slot;
79 struct transfer_request *next;
82 struct active_request_slot
84 CURL *curl;
85 FILE *local;
86 int in_use;
87 int done;
88 CURLcode curl_result;
89 long http_code;
90 void *callback_data;
91 void (*callback_func)(void *data);
92 struct active_request_slot *next;
95 struct alt_request {
96 char *base;
97 char *url;
98 struct buffer *buffer;
99 struct active_request_slot *slot;
100 int http_specific;
103 static struct transfer_request *request_queue_head = NULL;
104 static struct active_request_slot *active_queue_head = NULL;
106 static int curl_ssl_verify = -1;
107 static char *ssl_cert = NULL;
108 #if LIBCURL_VERSION_NUM >= 0x070902
109 static char *ssl_key = NULL;
110 #endif
111 #if LIBCURL_VERSION_NUM >= 0x070908
112 static char *ssl_capath = NULL;
113 #endif
114 static char *ssl_cainfo = NULL;
115 static long curl_low_speed_limit = -1;
116 static long curl_low_speed_time = -1;
118 struct buffer
120 size_t posn;
121 size_t size;
122 void *buffer;
125 static int http_options(const char *var, const char *value)
127 if (!strcmp("http.sslverify", var)) {
128 if (curl_ssl_verify == -1) {
129 curl_ssl_verify = git_config_bool(var, value);
131 return 0;
134 if (!strcmp("http.sslcert", var)) {
135 if (ssl_cert == NULL) {
136 ssl_cert = xmalloc(strlen(value)+1);
137 strcpy(ssl_cert, value);
139 return 0;
141 #if LIBCURL_VERSION_NUM >= 0x070902
142 if (!strcmp("http.sslkey", var)) {
143 if (ssl_key == NULL) {
144 ssl_key = xmalloc(strlen(value)+1);
145 strcpy(ssl_key, value);
147 return 0;
149 #endif
150 #if LIBCURL_VERSION_NUM >= 0x070908
151 if (!strcmp("http.sslcapath", var)) {
152 if (ssl_capath == NULL) {
153 ssl_capath = xmalloc(strlen(value)+1);
154 strcpy(ssl_capath, value);
156 return 0;
158 #endif
159 if (!strcmp("http.sslcainfo", var)) {
160 if (ssl_cainfo == NULL) {
161 ssl_cainfo = xmalloc(strlen(value)+1);
162 strcpy(ssl_cainfo, value);
164 return 0;
167 #ifdef USE_CURL_MULTI
168 if (!strcmp("http.maxrequests", var)) {
169 if (max_requests == -1)
170 max_requests = git_config_int(var, value);
171 return 0;
173 #endif
175 if (!strcmp("http.lowspeedlimit", var)) {
176 if (curl_low_speed_limit == -1)
177 curl_low_speed_limit = (long)git_config_int(var, value);
178 return 0;
180 if (!strcmp("http.lowspeedtime", var)) {
181 if (curl_low_speed_time == -1)
182 curl_low_speed_time = (long)git_config_int(var, value);
183 return 0;
186 /* Fall back on the default ones */
187 return git_default_config(var, value);
190 static size_t fwrite_buffer(void *ptr, size_t eltsize, size_t nmemb,
191 struct buffer *buffer)
193 size_t size = eltsize * nmemb;
194 if (size > buffer->size - buffer->posn)
195 size = buffer->size - buffer->posn;
196 memcpy(buffer->buffer + buffer->posn, ptr, size);
197 buffer->posn += size;
198 data_received++;
199 return size;
202 static size_t fwrite_buffer_dynamic(const void *ptr, size_t eltsize,
203 size_t nmemb, struct buffer *buffer)
205 size_t size = eltsize * nmemb;
206 if (size > buffer->size - buffer->posn) {
207 buffer->size = buffer->size * 3 / 2;
208 if (buffer->size < buffer->posn + size)
209 buffer->size = buffer->posn + size;
210 buffer->buffer = xrealloc(buffer->buffer, buffer->size);
212 memcpy(buffer->buffer + buffer->posn, ptr, size);
213 buffer->posn += size;
214 data_received++;
215 return size;
218 static size_t fwrite_sha1_file(void *ptr, size_t eltsize, size_t nmemb,
219 void *data)
221 unsigned char expn[4096];
222 size_t size = eltsize * nmemb;
223 int posn = 0;
224 struct transfer_request *request = (struct transfer_request *)data;
225 do {
226 ssize_t retval = write(request->local,
227 ptr + posn, size - posn);
228 if (retval < 0)
229 return posn;
230 posn += retval;
231 } while (posn < size);
233 request->stream.avail_in = size;
234 request->stream.next_in = ptr;
235 do {
236 request->stream.next_out = expn;
237 request->stream.avail_out = sizeof(expn);
238 request->zret = inflate(&request->stream, Z_SYNC_FLUSH);
239 SHA1_Update(&request->c, expn,
240 sizeof(expn) - request->stream.avail_out);
241 } while (request->stream.avail_in && request->zret == Z_OK);
242 data_received++;
243 return size;
246 #ifdef USE_CURL_MULTI
247 static void process_curl_messages(void);
248 static void process_request_queue(void);
249 #endif
250 static void fetch_alternates(char *base);
252 static CURL* get_curl_handle(void)
254 CURL* result = curl_easy_init();
256 curl_easy_setopt(result, CURLOPT_SSL_VERIFYPEER, curl_ssl_verify);
257 #if LIBCURL_VERSION_NUM >= 0x070907
258 curl_easy_setopt(result, CURLOPT_NETRC, CURL_NETRC_OPTIONAL);
259 #endif
261 if (ssl_cert != NULL)
262 curl_easy_setopt(result, CURLOPT_SSLCERT, ssl_cert);
263 #if LIBCURL_VERSION_NUM >= 0x070902
264 if (ssl_key != NULL)
265 curl_easy_setopt(result, CURLOPT_SSLKEY, ssl_key);
266 #endif
267 #if LIBCURL_VERSION_NUM >= 0x070908
268 if (ssl_capath != NULL)
269 curl_easy_setopt(result, CURLOPT_CAPATH, ssl_capath);
270 #endif
271 if (ssl_cainfo != NULL)
272 curl_easy_setopt(result, CURLOPT_CAINFO, ssl_cainfo);
273 curl_easy_setopt(result, CURLOPT_FAILONERROR, 1);
275 if (curl_low_speed_limit > 0 && curl_low_speed_time > 0) {
276 curl_easy_setopt(result, CURLOPT_LOW_SPEED_LIMIT,
277 curl_low_speed_limit);
278 curl_easy_setopt(result, CURLOPT_LOW_SPEED_TIME,
279 curl_low_speed_time);
282 curl_easy_setopt(result, CURLOPT_FOLLOWLOCATION, 1);
284 return result;
287 static struct active_request_slot *get_active_slot(void)
289 struct active_request_slot *slot = active_queue_head;
290 struct active_request_slot *newslot;
292 #ifdef USE_CURL_MULTI
293 int num_transfers;
295 /* Wait for a slot to open up if the queue is full */
296 while (active_requests >= max_requests) {
297 curl_multi_perform(curlm, &num_transfers);
298 if (num_transfers < active_requests) {
299 process_curl_messages();
302 #endif
304 while (slot != NULL && slot->in_use) {
305 slot = slot->next;
307 if (slot == NULL) {
308 newslot = xmalloc(sizeof(*newslot));
309 newslot->curl = NULL;
310 newslot->in_use = 0;
311 newslot->next = NULL;
313 slot = active_queue_head;
314 if (slot == NULL) {
315 active_queue_head = newslot;
316 } else {
317 while (slot->next != NULL) {
318 slot = slot->next;
320 slot->next = newslot;
322 slot = newslot;
325 if (slot->curl == NULL) {
326 #ifdef NO_CURL_EASY_DUPHANDLE
327 slot->curl = get_curl_handle();
328 #else
329 slot->curl = curl_easy_duphandle(curl_default);
330 #endif
333 active_requests++;
334 slot->in_use = 1;
335 slot->done = 0;
336 slot->local = NULL;
337 slot->callback_data = NULL;
338 slot->callback_func = NULL;
339 curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, pragma_header);
340 curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, no_range_header);
341 curl_easy_setopt(slot->curl, CURLOPT_ERRORBUFFER, curl_errorstr);
343 return slot;
346 static int start_active_slot(struct active_request_slot *slot)
348 #ifdef USE_CURL_MULTI
349 CURLMcode curlm_result = curl_multi_add_handle(curlm, slot->curl);
351 if (curlm_result != CURLM_OK &&
352 curlm_result != CURLM_CALL_MULTI_PERFORM) {
353 active_requests--;
354 slot->in_use = 0;
355 return 0;
357 #endif
358 return 1;
361 static void run_active_slot(struct active_request_slot *slot)
363 #ifdef USE_CURL_MULTI
364 int num_transfers;
365 long last_pos = 0;
366 long current_pos;
367 fd_set readfds;
368 fd_set writefds;
369 fd_set excfds;
370 int max_fd;
371 struct timeval select_timeout;
372 CURLMcode curlm_result;
374 while (!slot->done) {
375 data_received = 0;
376 do {
377 curlm_result = curl_multi_perform(curlm,
378 &num_transfers);
379 } while (curlm_result == CURLM_CALL_MULTI_PERFORM);
380 if (num_transfers < active_requests) {
381 process_curl_messages();
382 process_request_queue();
385 if (!data_received && slot->local != NULL) {
386 current_pos = ftell(slot->local);
387 if (current_pos > last_pos)
388 data_received++;
389 last_pos = current_pos;
392 if (!slot->done && !data_received) {
393 max_fd = 0;
394 FD_ZERO(&readfds);
395 FD_ZERO(&writefds);
396 FD_ZERO(&excfds);
397 select_timeout.tv_sec = 0;
398 select_timeout.tv_usec = 50000;
399 select(max_fd, &readfds, &writefds,
400 &excfds, &select_timeout);
403 #else
404 slot->curl_result = curl_easy_perform(slot->curl);
405 active_requests--;
406 #endif
409 static void start_request(struct transfer_request *request)
411 char *hex = sha1_to_hex(request->sha1);
412 char prevfile[PATH_MAX];
413 char *url;
414 char *posn;
415 int prevlocal;
416 unsigned char prev_buf[PREV_BUF_SIZE];
417 ssize_t prev_read = 0;
418 long prev_posn = 0;
419 char range[RANGE_HEADER_SIZE];
420 struct curl_slist *range_header = NULL;
421 struct active_request_slot *slot;
423 snprintf(prevfile, sizeof(prevfile), "%s.prev", request->filename);
424 unlink(prevfile);
425 rename(request->tmpfile, prevfile);
426 unlink(request->tmpfile);
428 request->local = open(request->tmpfile,
429 O_WRONLY | O_CREAT | O_EXCL, 0666);
430 /* This could have failed due to the "lazy directory creation";
431 * try to mkdir the last path component.
433 if (request->local < 0 && errno == ENOENT) {
434 char *dir = strrchr(request->tmpfile, '/');
435 if (dir) {
436 *dir = 0;
437 mkdir(request->tmpfile, 0777);
438 *dir = '/';
440 request->local = open(request->tmpfile,
441 O_WRONLY | O_CREAT | O_EXCL, 0666);
444 if (request->local < 0) {
445 request->state = ABORTED;
446 error("Couldn't create temporary file %s for %s: %s\n",
447 request->tmpfile, request->filename, strerror(errno));
448 return;
451 memset(&request->stream, 0, sizeof(request->stream));
453 inflateInit(&request->stream);
455 SHA1_Init(&request->c);
457 url = xmalloc(strlen(request->repo->base) + 50);
458 request->url = xmalloc(strlen(request->repo->base) + 50);
459 strcpy(url, request->repo->base);
460 posn = url + strlen(request->repo->base);
461 strcpy(posn, "objects/");
462 posn += 8;
463 memcpy(posn, hex, 2);
464 posn += 2;
465 *(posn++) = '/';
466 strcpy(posn, hex + 2);
467 strcpy(request->url, url);
469 /* If a previous temp file is present, process what was already
470 fetched. */
471 prevlocal = open(prevfile, O_RDONLY);
472 if (prevlocal != -1) {
473 do {
474 prev_read = read(prevlocal, prev_buf, PREV_BUF_SIZE);
475 if (prev_read>0) {
476 if (fwrite_sha1_file(prev_buf,
478 prev_read,
479 request) == prev_read) {
480 prev_posn += prev_read;
481 } else {
482 prev_read = -1;
485 } while (prev_read > 0);
486 close(prevlocal);
488 unlink(prevfile);
490 /* Reset inflate/SHA1 if there was an error reading the previous temp
491 file; also rewind to the beginning of the local file. */
492 if (prev_read == -1) {
493 memset(&request->stream, 0, sizeof(request->stream));
494 inflateInit(&request->stream);
495 SHA1_Init(&request->c);
496 if (prev_posn>0) {
497 prev_posn = 0;
498 lseek(request->local, SEEK_SET, 0);
499 ftruncate(request->local, 0);
503 slot = get_active_slot();
504 curl_easy_setopt(slot->curl, CURLOPT_FILE, request);
505 curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite_sha1_file);
506 curl_easy_setopt(slot->curl, CURLOPT_ERRORBUFFER, request->errorstr);
507 curl_easy_setopt(slot->curl, CURLOPT_URL, url);
508 curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, no_pragma_header);
510 /* If we have successfully processed data from a previous fetch
511 attempt, only fetch the data we don't already have. */
512 if (prev_posn>0) {
513 if (get_verbosely)
514 fprintf(stderr,
515 "Resuming fetch of object %s at byte %ld\n",
516 hex, prev_posn);
517 sprintf(range, "Range: bytes=%ld-", prev_posn);
518 range_header = curl_slist_append(range_header, range);
519 curl_easy_setopt(slot->curl,
520 CURLOPT_HTTPHEADER, range_header);
523 /* Try to get the request started, abort the request on error */
524 if (!start_active_slot(slot)) {
525 request->state = ABORTED;
526 close(request->local);
527 free(request->url);
528 return;
531 request->slot = slot;
532 request->state = ACTIVE;
535 static void finish_request(struct transfer_request *request)
537 struct stat st;
539 fchmod(request->local, 0444);
540 close(request->local);
542 if (request->http_code == 416) {
543 fprintf(stderr, "Warning: requested range invalid; we may already have all the data.\n");
544 } else if (request->curl_result != CURLE_OK) {
545 if (stat(request->tmpfile, &st) == 0)
546 if (st.st_size == 0)
547 unlink(request->tmpfile);
548 return;
551 inflateEnd(&request->stream);
552 SHA1_Final(request->real_sha1, &request->c);
553 if (request->zret != Z_STREAM_END) {
554 unlink(request->tmpfile);
555 return;
557 if (memcmp(request->sha1, request->real_sha1, 20)) {
558 unlink(request->tmpfile);
559 return;
561 request->rename =
562 move_temp_to_file(request->tmpfile, request->filename);
564 if (request->rename == 0)
565 pull_say("got %s\n", sha1_to_hex(request->sha1));
568 static void release_request(struct transfer_request *request)
570 struct transfer_request *entry = request_queue_head;
572 if (request == request_queue_head) {
573 request_queue_head = request->next;
574 } else {
575 while (entry->next != NULL && entry->next != request)
576 entry = entry->next;
577 if (entry->next == request)
578 entry->next = entry->next->next;
581 free(request->url);
582 free(request);
585 #ifdef USE_CURL_MULTI
586 static void process_curl_messages(void)
588 int num_messages;
589 struct active_request_slot *slot;
590 struct transfer_request *request = NULL;
591 CURLMsg *curl_message = curl_multi_info_read(curlm, &num_messages);
593 while (curl_message != NULL) {
594 if (curl_message->msg == CURLMSG_DONE) {
595 int curl_result = curl_message->data.result;
596 slot = active_queue_head;
597 while (slot != NULL &&
598 slot->curl != curl_message->easy_handle)
599 slot = slot->next;
600 if (slot != NULL) {
601 curl_multi_remove_handle(curlm, slot->curl);
602 active_requests--;
603 slot->done = 1;
604 slot->in_use = 0;
605 slot->curl_result = curl_result;
606 curl_easy_getinfo(slot->curl,
607 CURLINFO_HTTP_CODE,
608 &slot->http_code);
609 request = request_queue_head;
610 while (request != NULL &&
611 request->slot != slot)
612 request = request->next;
613 } else {
614 fprintf(stderr, "Received DONE message for unknown request!\n");
617 /* Process slot callback if appropriate */
618 if (slot->callback_func != NULL) {
619 slot->callback_func(slot->callback_data);
622 if (request != NULL) {
623 request->curl_result = curl_result;
624 request->http_code = slot->http_code;
625 request->slot = NULL;
626 request->state = COMPLETE;
628 /* Use alternates if necessary */
629 if (request->http_code == 404) {
630 fetch_alternates(alt->base);
631 if (request->repo->next != NULL) {
632 request->repo =
633 request->repo->next;
634 start_request(request);
636 } else {
637 finish_request(request);
640 } else {
641 fprintf(stderr, "Unknown CURL message received: %d\n",
642 (int)curl_message->msg);
644 curl_message = curl_multi_info_read(curlm, &num_messages);
648 static void process_request_queue(void)
650 struct transfer_request *request = request_queue_head;
651 struct active_request_slot *slot = active_queue_head;
652 int num_transfers;
654 while (active_requests < max_requests && request != NULL) {
655 if (request->state == WAITING) {
656 if (has_sha1_file(request->sha1))
657 release_request(request);
658 else
659 start_request(request);
660 curl_multi_perform(curlm, &num_transfers);
662 request = request->next;
665 while (slot != NULL) {
666 if (!slot->in_use && slot->curl != NULL) {
667 curl_easy_cleanup(slot->curl);
668 slot->curl = NULL;
670 slot = slot->next;
673 #endif
675 void prefetch(unsigned char *sha1)
677 struct transfer_request *newreq;
678 struct transfer_request *tail;
679 char *filename = sha1_file_name(sha1);
681 newreq = xmalloc(sizeof(*newreq));
682 memcpy(newreq->sha1, sha1, 20);
683 newreq->repo = alt;
684 newreq->url = NULL;
685 newreq->local = -1;
686 newreq->state = WAITING;
687 snprintf(newreq->filename, sizeof(newreq->filename), "%s", filename);
688 snprintf(newreq->tmpfile, sizeof(newreq->tmpfile),
689 "%s.temp", filename);
690 newreq->next = NULL;
692 if (request_queue_head == NULL) {
693 request_queue_head = newreq;
694 } else {
695 tail = request_queue_head;
696 while (tail->next != NULL) {
697 tail = tail->next;
699 tail->next = newreq;
701 #ifdef USE_CURL_MULTI
702 process_request_queue();
703 process_curl_messages();
704 #endif
707 static int fetch_index(struct alt_base *repo, unsigned char *sha1)
709 char *hex = sha1_to_hex(sha1);
710 char *filename;
711 char *url;
712 char tmpfile[PATH_MAX];
713 long prev_posn = 0;
714 char range[RANGE_HEADER_SIZE];
715 struct curl_slist *range_header = NULL;
717 FILE *indexfile;
718 struct active_request_slot *slot;
720 if (has_pack_index(sha1))
721 return 0;
723 if (get_verbosely)
724 fprintf(stderr, "Getting index for pack %s\n", hex);
726 url = xmalloc(strlen(repo->base) + 64);
727 sprintf(url, "%s/objects/pack/pack-%s.idx", repo->base, hex);
729 filename = sha1_pack_index_name(sha1);
730 snprintf(tmpfile, sizeof(tmpfile), "%s.temp", filename);
731 indexfile = fopen(tmpfile, "a");
732 if (!indexfile)
733 return error("Unable to open local file %s for pack index",
734 filename);
736 slot = get_active_slot();
737 curl_easy_setopt(slot->curl, CURLOPT_FILE, indexfile);
738 curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite);
739 curl_easy_setopt(slot->curl, CURLOPT_URL, url);
740 curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, no_pragma_header);
741 slot->local = indexfile;
743 /* If there is data present from a previous transfer attempt,
744 resume where it left off */
745 prev_posn = ftell(indexfile);
746 if (prev_posn>0) {
747 if (get_verbosely)
748 fprintf(stderr,
749 "Resuming fetch of index for pack %s at byte %ld\n",
750 hex, prev_posn);
751 sprintf(range, "Range: bytes=%ld-", prev_posn);
752 range_header = curl_slist_append(range_header, range);
753 curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, range_header);
756 if (start_active_slot(slot)) {
757 run_active_slot(slot);
758 if (slot->curl_result != CURLE_OK) {
759 fclose(indexfile);
760 return error("Unable to get pack index %s\n%s", url,
761 curl_errorstr);
763 } else {
764 return error("Unable to start request");
767 fclose(indexfile);
769 return move_temp_to_file(tmpfile, filename);
772 static int setup_index(struct alt_base *repo, unsigned char *sha1)
774 struct packed_git *new_pack;
775 if (has_pack_file(sha1))
776 return 0; // don't list this as something we can get
778 if (fetch_index(repo, sha1))
779 return -1;
781 new_pack = parse_pack_index(sha1);
782 new_pack->next = repo->packs;
783 repo->packs = new_pack;
784 return 0;
787 static void process_alternates(void *callback_data)
789 struct alt_request *alt_req = (struct alt_request *)callback_data;
790 struct active_request_slot *slot = alt_req->slot;
791 struct alt_base *tail = alt;
792 char *base = alt_req->base;
793 static const char null_byte = '\0';
794 char *data;
795 int i = 0;
797 if (alt_req->http_specific) {
798 if (slot->curl_result != CURLE_OK ||
799 !alt_req->buffer->posn) {
801 /* Try reusing the slot to get non-http alternates */
802 alt_req->http_specific = 0;
803 sprintf(alt_req->url, "%s/objects/info/alternates",
804 base);
805 curl_easy_setopt(slot->curl, CURLOPT_URL,
806 alt_req->url);
807 active_requests++;
808 slot->in_use = 1;
809 slot->done = 0;
810 if (start_active_slot(slot)) {
811 return;
812 } else {
813 got_alternates = -1;
814 slot->done = 1;
815 return;
818 } else if (slot->curl_result != CURLE_OK) {
819 if (slot->http_code != 404) {
820 got_alternates = -1;
821 return;
825 fwrite_buffer_dynamic(&null_byte, 1, 1, alt_req->buffer);
826 alt_req->buffer->posn--;
827 data = alt_req->buffer->buffer;
829 while (i < alt_req->buffer->posn) {
830 int posn = i;
831 while (posn < alt_req->buffer->posn && data[posn] != '\n')
832 posn++;
833 if (data[posn] == '\n') {
834 int okay = 0;
835 int serverlen = 0;
836 struct alt_base *newalt;
837 char *target = NULL;
838 if (data[i] == '/') {
839 serverlen = strchr(base + 8, '/') - base;
840 okay = 1;
841 } else if (!memcmp(data + i, "../", 3)) {
842 i += 3;
843 serverlen = strlen(base);
844 while (i + 2 < posn &&
845 !memcmp(data + i, "../", 3)) {
846 do {
847 serverlen--;
848 } while (serverlen &&
849 base[serverlen - 1] != '/');
850 i += 3;
852 // If the server got removed, give up.
853 okay = strchr(base, ':') - base + 3 <
854 serverlen;
855 } else if (alt_req->http_specific) {
856 char *colon = strchr(data + i, ':');
857 char *slash = strchr(data + i, '/');
858 if (colon && slash && colon < data + posn &&
859 slash < data + posn && colon < slash) {
860 okay = 1;
863 // skip 'objects' at end
864 if (okay) {
865 target = xmalloc(serverlen + posn - i - 6);
866 strncpy(target, base, serverlen);
867 strncpy(target + serverlen, data + i,
868 posn - i - 7);
869 target[serverlen + posn - i - 7] = '\0';
870 if (get_verbosely)
871 fprintf(stderr,
872 "Also look at %s\n", target);
873 newalt = xmalloc(sizeof(*newalt));
874 newalt->next = NULL;
875 newalt->base = target;
876 newalt->got_indices = 0;
877 newalt->packs = NULL;
878 while (tail->next != NULL)
879 tail = tail->next;
880 tail->next = newalt;
883 i = posn + 1;
886 got_alternates = 1;
889 static void fetch_alternates(char *base)
891 struct buffer buffer;
892 char *url;
893 char *data;
894 struct active_request_slot *slot;
895 static struct alt_request alt_req;
896 int num_transfers;
898 /* If another request has already started fetching alternates,
899 wait for them to arrive and return to processing this request's
900 curl message */
901 while (got_alternates == 0) {
902 curl_multi_perform(curlm, &num_transfers);
903 process_curl_messages();
904 process_request_queue();
907 /* Nothing to do if they've already been fetched */
908 if (got_alternates == 1)
909 return;
911 /* Start the fetch */
912 got_alternates = 0;
914 data = xmalloc(4096);
915 buffer.size = 4096;
916 buffer.posn = 0;
917 buffer.buffer = data;
919 if (get_verbosely)
920 fprintf(stderr, "Getting alternates list for %s\n", base);
922 url = xmalloc(strlen(base) + 31);
923 sprintf(url, "%s/objects/info/http-alternates", base);
925 /* Use a callback to process the result, since another request
926 may fail and need to have alternates loaded before continuing */
927 slot = get_active_slot();
928 slot->callback_func = process_alternates;
929 slot->callback_data = &alt_req;
931 curl_easy_setopt(slot->curl, CURLOPT_FILE, &buffer);
932 curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION,
933 fwrite_buffer_dynamic);
934 curl_easy_setopt(slot->curl, CURLOPT_URL, url);
936 alt_req.base = base;
937 alt_req.url = url;
938 alt_req.buffer = &buffer;
939 alt_req.http_specific = 1;
940 alt_req.slot = slot;
942 if (start_active_slot(slot))
943 run_active_slot(slot);
944 else
945 got_alternates = -1;
947 free(data);
948 free(url);
951 static int fetch_indices(struct alt_base *repo)
953 unsigned char sha1[20];
954 char *url;
955 struct buffer buffer;
956 char *data;
957 int i = 0;
959 struct active_request_slot *slot;
961 if (repo->got_indices)
962 return 0;
964 data = xmalloc(4096);
965 buffer.size = 4096;
966 buffer.posn = 0;
967 buffer.buffer = data;
969 if (get_verbosely)
970 fprintf(stderr, "Getting pack list for %s\n", repo->base);
972 url = xmalloc(strlen(repo->base) + 21);
973 sprintf(url, "%s/objects/info/packs", repo->base);
975 slot = get_active_slot();
976 curl_easy_setopt(slot->curl, CURLOPT_FILE, &buffer);
977 curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION,
978 fwrite_buffer_dynamic);
979 curl_easy_setopt(slot->curl, CURLOPT_URL, url);
980 curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, NULL);
981 if (start_active_slot(slot)) {
982 run_active_slot(slot);
983 if (slot->curl_result != CURLE_OK) {
984 free(buffer.buffer);
985 return error("%s", curl_errorstr);
987 } else {
988 free(buffer.buffer);
989 return error("Unable to start request");
992 data = buffer.buffer;
993 while (i < buffer.posn) {
994 switch (data[i]) {
995 case 'P':
996 i++;
997 if (i + 52 < buffer.posn &&
998 !strncmp(data + i, " pack-", 6) &&
999 !strncmp(data + i + 46, ".pack\n", 6)) {
1000 get_sha1_hex(data + i + 6, sha1);
1001 setup_index(repo, sha1);
1002 i += 51;
1003 break;
1005 default:
1006 while (data[i] != '\n')
1007 i++;
1009 i++;
1012 free(buffer.buffer);
1013 repo->got_indices = 1;
1014 return 0;
1017 static int fetch_pack(struct alt_base *repo, unsigned char *sha1)
1019 char *url;
1020 struct packed_git *target;
1021 struct packed_git **lst;
1022 FILE *packfile;
1023 char *filename;
1024 char tmpfile[PATH_MAX];
1025 int ret;
1026 long prev_posn = 0;
1027 char range[RANGE_HEADER_SIZE];
1028 struct curl_slist *range_header = NULL;
1030 struct active_request_slot *slot;
1032 if (fetch_indices(repo))
1033 return -1;
1034 target = find_sha1_pack(sha1, repo->packs);
1035 if (!target)
1036 return -1;
1038 if (get_verbosely) {
1039 fprintf(stderr, "Getting pack %s\n",
1040 sha1_to_hex(target->sha1));
1041 fprintf(stderr, " which contains %s\n",
1042 sha1_to_hex(sha1));
1045 url = xmalloc(strlen(repo->base) + 65);
1046 sprintf(url, "%s/objects/pack/pack-%s.pack",
1047 repo->base, sha1_to_hex(target->sha1));
1049 filename = sha1_pack_name(target->sha1);
1050 snprintf(tmpfile, sizeof(tmpfile), "%s.temp", filename);
1051 packfile = fopen(tmpfile, "a");
1052 if (!packfile)
1053 return error("Unable to open local file %s for pack",
1054 filename);
1056 slot = get_active_slot();
1057 curl_easy_setopt(slot->curl, CURLOPT_FILE, packfile);
1058 curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite);
1059 curl_easy_setopt(slot->curl, CURLOPT_URL, url);
1060 curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, no_pragma_header);
1061 slot->local = packfile;
1063 /* If there is data present from a previous transfer attempt,
1064 resume where it left off */
1065 prev_posn = ftell(packfile);
1066 if (prev_posn>0) {
1067 if (get_verbosely)
1068 fprintf(stderr,
1069 "Resuming fetch of pack %s at byte %ld\n",
1070 sha1_to_hex(target->sha1), prev_posn);
1071 sprintf(range, "Range: bytes=%ld-", prev_posn);
1072 range_header = curl_slist_append(range_header, range);
1073 curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, range_header);
1076 if (start_active_slot(slot)) {
1077 run_active_slot(slot);
1078 if (slot->curl_result != CURLE_OK) {
1079 fclose(packfile);
1080 return error("Unable to get pack file %s\n%s", url,
1081 curl_errorstr);
1083 } else {
1084 return error("Unable to start request");
1087 fclose(packfile);
1089 ret = move_temp_to_file(tmpfile, filename);
1090 if (ret)
1091 return ret;
1093 lst = &repo->packs;
1094 while (*lst != target)
1095 lst = &((*lst)->next);
1096 *lst = (*lst)->next;
1098 if (verify_pack(target, 0))
1099 return -1;
1100 install_packed_git(target);
1102 return 0;
1105 static int fetch_object(struct alt_base *repo, unsigned char *sha1)
1107 char *hex = sha1_to_hex(sha1);
1108 int ret;
1109 struct transfer_request *request = request_queue_head;
1111 while (request != NULL && memcmp(request->sha1, sha1, 20))
1112 request = request->next;
1113 if (request == NULL)
1114 return error("Couldn't find request for %s in the queue", hex);
1116 if (has_sha1_file(request->sha1)) {
1117 release_request(request);
1118 return 0;
1121 #ifdef USE_CURL_MULTI
1122 while (request->state == WAITING) {
1123 int num_transfers;
1124 curl_multi_perform(curlm, &num_transfers);
1125 if (num_transfers < active_requests) {
1126 process_curl_messages();
1127 process_request_queue();
1130 #else
1131 start_request(request);
1132 #endif
1134 while (request->state == ACTIVE) {
1135 run_active_slot(request->slot);
1136 #ifndef USE_CURL_MULTI
1137 request->curl_result = request->slot->curl_result;
1138 request->http_code = request->slot->http_code;
1139 request->slot = NULL;
1141 /* Use alternates if necessary */
1142 if (request->http_code == 404) {
1143 fetch_alternates(alt->base);
1144 if (request->repo->next != NULL) {
1145 request->repo = request->repo->next;
1146 start_request(request);
1148 } else {
1149 finish_request(request);
1150 request->state = COMPLETE;
1152 #endif
1155 if (request->state == ABORTED) {
1156 release_request(request);
1157 return error("Request for %s aborted", hex);
1160 if (request->curl_result != CURLE_OK && request->http_code != 416) {
1161 if (request->http_code == 404)
1162 ret = -1; /* Be silent, it is probably in a pack. */
1163 else
1164 ret = error("%s (curl_result = %d, http_code = %ld, sha1 = %s)",
1165 request->errorstr, request->curl_result,
1166 request->http_code, hex);
1167 release_request(request);
1168 return ret;
1171 if (request->zret != Z_STREAM_END) {
1172 ret = error("File %s (%s) corrupt\n", hex, request->url);
1173 release_request(request);
1174 return ret;
1177 if (memcmp(request->sha1, request->real_sha1, 20)) {
1178 release_request(request);
1179 return error("File %s has bad hash\n", hex);
1182 if (request->rename < 0) {
1183 ret = error("unable to write sha1 filename %s: %s",
1184 request->filename,
1185 strerror(request->rename));
1186 release_request(request);
1187 return ret;
1190 release_request(request);
1191 return 0;
1194 int fetch(unsigned char *sha1)
1196 struct alt_base *altbase = alt;
1198 if (!fetch_object(altbase, sha1))
1199 return 0;
1200 while (altbase) {
1201 if (!fetch_pack(altbase, sha1))
1202 return 0;
1203 fetch_alternates(alt->base);
1204 altbase = altbase->next;
1206 return error("Unable to find %s under %s\n", sha1_to_hex(sha1),
1207 alt->base);
1210 static inline int needs_quote(int ch)
1212 switch (ch) {
1213 case '/': case '-': case '.':
1214 case 'A'...'Z': case 'a'...'z': case '0'...'9':
1215 return 0;
1216 default:
1217 return 1;
1221 static inline int hex(int v)
1223 if (v < 10) return '0' + v;
1224 else return 'A' + v - 10;
1227 static char *quote_ref_url(const char *base, const char *ref)
1229 const char *cp;
1230 char *dp, *qref;
1231 int len, baselen, ch;
1233 baselen = strlen(base);
1234 len = baselen + 6; /* "refs/" + NUL */
1235 for (cp = ref; (ch = *cp) != 0; cp++, len++)
1236 if (needs_quote(ch))
1237 len += 2; /* extra two hex plus replacement % */
1238 qref = xmalloc(len);
1239 memcpy(qref, base, baselen);
1240 memcpy(qref + baselen, "refs/", 5);
1241 for (cp = ref, dp = qref + baselen + 5; (ch = *cp) != 0; cp++) {
1242 if (needs_quote(ch)) {
1243 *dp++ = '%';
1244 *dp++ = hex((ch >> 4) & 0xF);
1245 *dp++ = hex(ch & 0xF);
1247 else
1248 *dp++ = ch;
1250 *dp = 0;
1252 return qref;
1255 int fetch_ref(char *ref, unsigned char *sha1)
1257 char *url;
1258 char hex[42];
1259 struct buffer buffer;
1260 char *base = alt->base;
1261 struct active_request_slot *slot;
1262 buffer.size = 41;
1263 buffer.posn = 0;
1264 buffer.buffer = hex;
1265 hex[41] = '\0';
1267 url = quote_ref_url(base, ref);
1268 slot = get_active_slot();
1269 curl_easy_setopt(slot->curl, CURLOPT_FILE, &buffer);
1270 curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite_buffer);
1271 curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, NULL);
1272 curl_easy_setopt(slot->curl, CURLOPT_URL, url);
1273 if (start_active_slot(slot)) {
1274 run_active_slot(slot);
1275 if (slot->curl_result != CURLE_OK)
1276 return error("Couldn't get %s for %s\n%s",
1277 url, ref, curl_errorstr);
1278 } else {
1279 return error("Unable to start request");
1282 hex[40] = '\0';
1283 get_sha1_hex(hex, sha1);
1284 return 0;
1287 int main(int argc, char **argv)
1289 char *commit_id;
1290 char *url;
1291 int arg = 1;
1292 struct active_request_slot *slot;
1293 char *low_speed_limit;
1294 char *low_speed_time;
1295 char *wait_url;
1296 int rc = 0;
1298 while (arg < argc && argv[arg][0] == '-') {
1299 if (argv[arg][1] == 't') {
1300 get_tree = 1;
1301 } else if (argv[arg][1] == 'c') {
1302 get_history = 1;
1303 } else if (argv[arg][1] == 'a') {
1304 get_all = 1;
1305 get_tree = 1;
1306 get_history = 1;
1307 } else if (argv[arg][1] == 'v') {
1308 get_verbosely = 1;
1309 } else if (argv[arg][1] == 'w') {
1310 write_ref = argv[arg + 1];
1311 arg++;
1312 } else if (!strcmp(argv[arg], "--recover")) {
1313 get_recover = 1;
1315 arg++;
1317 if (argc < arg + 2) {
1318 usage("git-http-fetch [-c] [-t] [-a] [-d] [-v] [--recover] [-w ref] commit-id url");
1319 return 1;
1321 commit_id = argv[arg];
1322 url = argv[arg + 1];
1324 curl_global_init(CURL_GLOBAL_ALL);
1326 #ifdef USE_CURL_MULTI
1328 char *http_max_requests = getenv("GIT_HTTP_MAX_REQUESTS");
1329 if (http_max_requests != NULL)
1330 max_requests = atoi(http_max_requests);
1333 curlm = curl_multi_init();
1334 if (curlm == NULL) {
1335 fprintf(stderr, "Error creating curl multi handle.\n");
1336 return 1;
1338 #endif
1340 if (getenv("GIT_SSL_NO_VERIFY"))
1341 curl_ssl_verify = 0;
1343 ssl_cert = getenv("GIT_SSL_CERT");
1344 #if LIBCURL_VERSION_NUM >= 0x070902
1345 ssl_key = getenv("GIT_SSL_KEY");
1346 #endif
1347 #if LIBCURL_VERSION_NUM >= 0x070908
1348 ssl_capath = getenv("GIT_SSL_CAPATH");
1349 #endif
1350 ssl_cainfo = getenv("GIT_SSL_CAINFO");
1352 low_speed_limit = getenv("GIT_HTTP_LOW_SPEED_LIMIT");
1353 if (low_speed_limit != NULL)
1354 curl_low_speed_limit = strtol(low_speed_limit, NULL, 10);
1355 low_speed_time = getenv("GIT_HTTP_LOW_SPEED_TIME");
1356 if (low_speed_time != NULL)
1357 curl_low_speed_time = strtol(low_speed_time, NULL, 10);
1359 git_config(http_options);
1361 if (curl_ssl_verify == -1)
1362 curl_ssl_verify = 1;
1364 #ifdef USE_CURL_MULTI
1365 if (max_requests < 1)
1366 max_requests = DEFAULT_MAX_REQUESTS;
1367 #endif
1369 pragma_header = curl_slist_append(pragma_header, "Pragma: no-cache");
1370 no_pragma_header = curl_slist_append(no_pragma_header, "Pragma:");
1371 no_range_header = curl_slist_append(no_range_header, "Range:");
1373 #ifndef NO_CURL_EASY_DUPHANDLE
1374 curl_default = get_curl_handle();
1375 #endif
1377 alt = xmalloc(sizeof(*alt));
1378 alt->base = url;
1379 alt->got_indices = 0;
1380 alt->packs = NULL;
1381 alt->next = NULL;
1383 if (pull(commit_id))
1384 rc = 1;
1386 curl_slist_free_all(pragma_header);
1387 curl_slist_free_all(no_pragma_header);
1388 curl_slist_free_all(no_range_header);
1389 #ifndef NO_CURL_EASY_DUPHANDLE
1390 curl_easy_cleanup(curl_default);
1391 #endif
1392 slot = active_queue_head;
1393 while (slot != NULL) {
1394 if (slot->in_use) {
1395 if (get_verbosely) {
1396 curl_easy_getinfo(slot->curl,
1397 CURLINFO_EFFECTIVE_URL,
1398 &wait_url);
1399 fprintf(stderr, "Waiting for %s\n", wait_url);
1401 run_active_slot(slot);
1403 if (slot->curl != NULL)
1404 curl_easy_cleanup(slot->curl);
1405 slot = slot->next;
1407 #ifdef USE_CURL_MULTI
1408 curl_multi_cleanup(curlm);
1409 #endif
1410 curl_global_cleanup();
1411 return rc;