Support for HTTP transfer timeouts based on transfer speed
[git/dscho.git] / http-fetch.c
bloba7dc2cc3bdbcda8eee8cdadb706aba51782e68da
1 #include "cache.h"
2 #include "commit.h"
3 #include "pack.h"
4 #include "fetch.h"
6 #include <curl/curl.h>
7 #include <curl/easy.h>
9 #if LIBCURL_VERSION_NUM >= 0x070908
10 #define USE_CURL_MULTI
11 #define DEFAULT_MAX_REQUESTS 5
12 #endif
14 #if LIBCURL_VERSION_NUM < 0x070704
15 #define curl_global_cleanup() do { /* nothing */ } while(0)
16 #endif
17 #if LIBCURL_VERSION_NUM < 0x070800
18 #define curl_global_init(a) do { /* nothing */ } while(0)
19 #endif
21 #if LIBCURL_VERSION_NUM < 0x070c04
22 #define NO_CURL_EASY_DUPHANDLE
23 #endif
25 #define PREV_BUF_SIZE 4096
26 #define RANGE_HEADER_SIZE 30
28 static int active_requests = 0;
29 static int data_received;
31 #ifdef USE_CURL_MULTI
32 static int max_requests = -1;
33 static CURLM *curlm;
34 #endif
35 #ifndef NO_CURL_EASY_DUPHANDLE
36 static CURL *curl_default;
37 #endif
38 static struct curl_slist *pragma_header;
39 static struct curl_slist *no_pragma_header;
40 static struct curl_slist *no_range_header;
41 static char curl_errorstr[CURL_ERROR_SIZE];
43 struct alt_base
45 char *base;
46 int got_indices;
47 struct packed_git *packs;
48 struct alt_base *next;
51 static struct alt_base *alt = NULL;
53 enum transfer_state {
54 WAITING,
55 ABORTED,
56 ACTIVE,
57 COMPLETE,
60 struct transfer_request
62 unsigned char sha1[20];
63 struct alt_base *repo;
64 char *url;
65 char filename[PATH_MAX];
66 char tmpfile[PATH_MAX];
67 int local;
68 enum transfer_state state;
69 CURLcode curl_result;
70 char errorstr[CURL_ERROR_SIZE];
71 long http_code;
72 unsigned char real_sha1[20];
73 SHA_CTX c;
74 z_stream stream;
75 int zret;
76 int rename;
77 struct active_request_slot *slot;
78 struct transfer_request *next;
81 struct active_request_slot
83 CURL *curl;
84 FILE *local;
85 int in_use;
86 int done;
87 CURLcode curl_result;
88 struct active_request_slot *next;
91 static struct transfer_request *request_queue_head = NULL;
92 static struct active_request_slot *active_queue_head = NULL;
94 static int curl_ssl_verify = -1;
95 static char *ssl_cert = NULL;
96 #if LIBCURL_VERSION_NUM >= 0x070902
97 static char *ssl_key = NULL;
98 #endif
99 #if LIBCURL_VERSION_NUM >= 0x070908
100 static char *ssl_capath = NULL;
101 #endif
102 static char *ssl_cainfo = NULL;
103 static long curl_low_speed_limit = -1;
104 static long curl_low_speed_time = -1;
106 struct buffer
108 size_t posn;
109 size_t size;
110 void *buffer;
113 static int http_options(const char *var, const char *value)
115 if (!strcmp("http.sslverify", var)) {
116 if (curl_ssl_verify == -1) {
117 curl_ssl_verify = git_config_bool(var, value);
119 return 0;
122 if (!strcmp("http.sslcert", var)) {
123 if (ssl_cert == NULL) {
124 ssl_cert = xmalloc(strlen(value)+1);
125 strcpy(ssl_cert, value);
127 return 0;
129 #if LIBCURL_VERSION_NUM >= 0x070902
130 if (!strcmp("http.sslkey", var)) {
131 if (ssl_key == NULL) {
132 ssl_key = xmalloc(strlen(value)+1);
133 strcpy(ssl_key, value);
135 return 0;
137 #endif
138 #if LIBCURL_VERSION_NUM >= 0x070908
139 if (!strcmp("http.sslcapath", var)) {
140 if (ssl_capath == NULL) {
141 ssl_capath = xmalloc(strlen(value)+1);
142 strcpy(ssl_capath, value);
144 return 0;
146 #endif
147 if (!strcmp("http.sslcainfo", var)) {
148 if (ssl_cainfo == NULL) {
149 ssl_cainfo = xmalloc(strlen(value)+1);
150 strcpy(ssl_cainfo, value);
152 return 0;
155 #ifdef USE_CURL_MULTI
156 if (!strcmp("http.maxrequests", var)) {
157 if (max_requests == -1)
158 max_requests = git_config_int(var, value);
159 return 0;
161 #endif
163 if (!strcmp("http.lowspeedlimit", var)) {
164 if (curl_low_speed_limit == -1)
165 curl_low_speed_limit = (long)git_config_int(var, value);
166 return 0;
168 if (!strcmp("http.lowspeedtime", var)) {
169 if (curl_low_speed_time == -1)
170 curl_low_speed_time = (long)git_config_int(var, value);
171 return 0;
174 /* Fall back on the default ones */
175 return git_default_config(var, value);
178 static size_t fwrite_buffer(void *ptr, size_t eltsize, size_t nmemb,
179 struct buffer *buffer)
181 size_t size = eltsize * nmemb;
182 if (size > buffer->size - buffer->posn)
183 size = buffer->size - buffer->posn;
184 memcpy(buffer->buffer + buffer->posn, ptr, size);
185 buffer->posn += size;
186 data_received++;
187 return size;
190 static size_t fwrite_buffer_dynamic(const void *ptr, size_t eltsize,
191 size_t nmemb, struct buffer *buffer)
193 size_t size = eltsize * nmemb;
194 if (size > buffer->size - buffer->posn) {
195 buffer->size = buffer->size * 3 / 2;
196 if (buffer->size < buffer->posn + size)
197 buffer->size = buffer->posn + size;
198 buffer->buffer = xrealloc(buffer->buffer, buffer->size);
200 memcpy(buffer->buffer + buffer->posn, ptr, size);
201 buffer->posn += size;
202 data_received++;
203 return size;
206 static size_t fwrite_sha1_file(void *ptr, size_t eltsize, size_t nmemb,
207 void *data)
209 unsigned char expn[4096];
210 size_t size = eltsize * nmemb;
211 int posn = 0;
212 struct transfer_request *request = (struct transfer_request *)data;
213 do {
214 ssize_t retval = write(request->local,
215 ptr + posn, size - posn);
216 if (retval < 0)
217 return posn;
218 posn += retval;
219 } while (posn < size);
221 request->stream.avail_in = size;
222 request->stream.next_in = ptr;
223 do {
224 request->stream.next_out = expn;
225 request->stream.avail_out = sizeof(expn);
226 request->zret = inflate(&request->stream, Z_SYNC_FLUSH);
227 SHA1_Update(&request->c, expn,
228 sizeof(expn) - request->stream.avail_out);
229 } while (request->stream.avail_in && request->zret == Z_OK);
230 data_received++;
231 return size;
234 #ifdef USE_CURL_MULTI
235 static void process_curl_messages(void);
236 static void process_request_queue(void);
237 #endif
239 static CURL* get_curl_handle(void)
241 CURL* result = curl_easy_init();
243 curl_easy_setopt(result, CURLOPT_SSL_VERIFYPEER, curl_ssl_verify);
244 #if LIBCURL_VERSION_NUM >= 0x070907
245 curl_easy_setopt(result, CURLOPT_NETRC, CURL_NETRC_OPTIONAL);
246 #endif
248 if (ssl_cert != NULL)
249 curl_easy_setopt(result, CURLOPT_SSLCERT, ssl_cert);
250 #if LIBCURL_VERSION_NUM >= 0x070902
251 if (ssl_key != NULL)
252 curl_easy_setopt(result, CURLOPT_SSLKEY, ssl_key);
253 #endif
254 #if LIBCURL_VERSION_NUM >= 0x070908
255 if (ssl_capath != NULL)
256 curl_easy_setopt(result, CURLOPT_CAPATH, ssl_capath);
257 #endif
258 if (ssl_cainfo != NULL)
259 curl_easy_setopt(result, CURLOPT_CAINFO, ssl_cainfo);
260 curl_easy_setopt(result, CURLOPT_FAILONERROR, 1);
262 if (curl_low_speed_limit > 0 && curl_low_speed_time > 0) {
263 curl_easy_setopt(result, CURLOPT_LOW_SPEED_LIMIT,
264 curl_low_speed_limit);
265 curl_easy_setopt(result, CURLOPT_LOW_SPEED_TIME,
266 curl_low_speed_time);
269 return result;
272 static struct active_request_slot *get_active_slot(void)
274 struct active_request_slot *slot = active_queue_head;
275 struct active_request_slot *newslot;
277 #ifdef USE_CURL_MULTI
278 int num_transfers;
280 /* Wait for a slot to open up if the queue is full */
281 while (active_requests >= max_requests) {
282 curl_multi_perform(curlm, &num_transfers);
283 if (num_transfers < active_requests) {
284 process_curl_messages();
287 #endif
289 while (slot != NULL && slot->in_use) {
290 slot = slot->next;
292 if (slot == NULL) {
293 newslot = xmalloc(sizeof(*newslot));
294 #ifdef NO_CURL_EASY_DUPHANDLE
295 newslot->curl = get_curl_handle();
296 #else
297 newslot->curl = curl_easy_duphandle(curl_default);
298 #endif
299 newslot->in_use = 0;
300 newslot->next = NULL;
302 slot = active_queue_head;
303 if (slot == NULL) {
304 active_queue_head = newslot;
305 } else {
306 while (slot->next != NULL) {
307 slot = slot->next;
309 slot->next = newslot;
311 slot = newslot;
314 active_requests++;
315 slot->in_use = 1;
316 slot->done = 0;
317 slot->local = NULL;
318 curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, pragma_header);
319 curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, no_range_header);
320 curl_easy_setopt(slot->curl, CURLOPT_ERRORBUFFER, curl_errorstr);
322 return slot;
325 static int start_active_slot(struct active_request_slot *slot)
327 #ifdef USE_CURL_MULTI
328 CURLMcode curlm_result = curl_multi_add_handle(curlm, slot->curl);
330 if (curlm_result != CURLM_OK &&
331 curlm_result != CURLM_CALL_MULTI_PERFORM) {
332 active_requests--;
333 slot->in_use = 0;
334 return 0;
336 #endif
337 return 1;
340 static void run_active_slot(struct active_request_slot *slot)
342 #ifdef USE_CURL_MULTI
343 int num_transfers;
344 long last_pos = 0;
345 long current_pos;
346 fd_set readfds;
347 fd_set writefds;
348 fd_set excfds;
349 int max_fd;
350 struct timeval select_timeout;
351 CURLMcode curlm_result;
353 while (!slot->done) {
354 data_received = 0;
355 do {
356 curlm_result = curl_multi_perform(curlm,
357 &num_transfers);
358 } while (curlm_result == CURLM_CALL_MULTI_PERFORM);
359 if (num_transfers < active_requests) {
360 process_curl_messages();
361 process_request_queue();
364 if (!data_received && slot->local != NULL) {
365 current_pos = ftell(slot->local);
366 if (current_pos > last_pos)
367 data_received++;
368 last_pos = current_pos;
371 if (!slot->done && !data_received) {
372 max_fd = 0;
373 FD_ZERO(&readfds);
374 FD_ZERO(&writefds);
375 FD_ZERO(&excfds);
376 select_timeout.tv_sec = 0;
377 select_timeout.tv_usec = 50000;
378 select(max_fd, &readfds, &writefds,
379 &excfds, &select_timeout);
382 #else
383 slot->curl_result = curl_easy_perform(slot->curl);
384 active_requests--;
385 #endif
388 static void start_request(struct transfer_request *request)
390 char *hex = sha1_to_hex(request->sha1);
391 char prevfile[PATH_MAX];
392 char *url;
393 char *posn;
394 int prevlocal;
395 unsigned char prev_buf[PREV_BUF_SIZE];
396 ssize_t prev_read = 0;
397 long prev_posn = 0;
398 char range[RANGE_HEADER_SIZE];
399 struct curl_slist *range_header = NULL;
400 struct active_request_slot *slot;
402 snprintf(prevfile, sizeof(prevfile), "%s.prev", request->filename);
403 unlink(prevfile);
404 rename(request->tmpfile, prevfile);
405 unlink(request->tmpfile);
407 request->local = open(request->tmpfile,
408 O_WRONLY | O_CREAT | O_EXCL, 0666);
409 /* This could have failed due to the "lazy directory creation";
410 * try to mkdir the last path component.
412 if (request->local < 0 && errno == ENOENT) {
413 char *dir = strrchr(request->tmpfile, '/');
414 if (dir) {
415 *dir = 0;
416 mkdir(request->tmpfile, 0777);
417 *dir = '/';
419 request->local = open(request->tmpfile,
420 O_WRONLY | O_CREAT | O_EXCL, 0666);
423 if (request->local < 0) {
424 request->state = ABORTED;
425 error("Couldn't create temporary file %s for %s: %s\n",
426 request->tmpfile, request->filename, strerror(errno));
427 return;
430 memset(&request->stream, 0, sizeof(request->stream));
432 inflateInit(&request->stream);
434 SHA1_Init(&request->c);
436 url = xmalloc(strlen(request->repo->base) + 50);
437 request->url = xmalloc(strlen(request->repo->base) + 50);
438 strcpy(url, request->repo->base);
439 posn = url + strlen(request->repo->base);
440 strcpy(posn, "objects/");
441 posn += 8;
442 memcpy(posn, hex, 2);
443 posn += 2;
444 *(posn++) = '/';
445 strcpy(posn, hex + 2);
446 strcpy(request->url, url);
448 /* If a previous temp file is present, process what was already
449 fetched. */
450 prevlocal = open(prevfile, O_RDONLY);
451 if (prevlocal != -1) {
452 do {
453 prev_read = read(prevlocal, prev_buf, PREV_BUF_SIZE);
454 if (prev_read>0) {
455 if (fwrite_sha1_file(prev_buf,
457 prev_read,
458 request) == prev_read) {
459 prev_posn += prev_read;
460 } else {
461 prev_read = -1;
464 } while (prev_read > 0);
465 close(prevlocal);
467 unlink(prevfile);
469 /* Reset inflate/SHA1 if there was an error reading the previous temp
470 file; also rewind to the beginning of the local file. */
471 if (prev_read == -1) {
472 memset(&request->stream, 0, sizeof(request->stream));
473 inflateInit(&request->stream);
474 SHA1_Init(&request->c);
475 if (prev_posn>0) {
476 prev_posn = 0;
477 lseek(request->local, SEEK_SET, 0);
478 ftruncate(request->local, 0);
482 slot = get_active_slot();
483 curl_easy_setopt(slot->curl, CURLOPT_FILE, request);
484 curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite_sha1_file);
485 curl_easy_setopt(slot->curl, CURLOPT_ERRORBUFFER, request->errorstr);
486 curl_easy_setopt(slot->curl, CURLOPT_URL, url);
487 curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, no_pragma_header);
489 /* If we have successfully processed data from a previous fetch
490 attempt, only fetch the data we don't already have. */
491 if (prev_posn>0) {
492 if (get_verbosely)
493 fprintf(stderr,
494 "Resuming fetch of object %s at byte %ld\n",
495 hex, prev_posn);
496 sprintf(range, "Range: bytes=%ld-", prev_posn);
497 range_header = curl_slist_append(range_header, range);
498 curl_easy_setopt(slot->curl,
499 CURLOPT_HTTPHEADER, range_header);
502 /* Try to get the request started, abort the request on error */
503 if (!start_active_slot(slot)) {
504 request->state = ABORTED;
505 close(request->local);
506 free(request->url);
507 return;
510 request->slot = slot;
511 request->state = ACTIVE;
514 static void finish_request(struct transfer_request *request)
516 fchmod(request->local, 0444);
517 close(request->local);
519 if (request->http_code == 416) {
520 fprintf(stderr, "Warning: requested range invalid; we may already have all the data.\n");
521 } else if (request->curl_result != CURLE_OK) {
522 return;
525 inflateEnd(&request->stream);
526 SHA1_Final(request->real_sha1, &request->c);
527 if (request->zret != Z_STREAM_END) {
528 unlink(request->tmpfile);
529 return;
531 if (memcmp(request->sha1, request->real_sha1, 20)) {
532 unlink(request->tmpfile);
533 return;
535 request->rename =
536 move_temp_to_file(request->tmpfile, request->filename);
538 if (request->rename == 0)
539 pull_say("got %s\n", sha1_to_hex(request->sha1));
542 static void release_request(struct transfer_request *request)
544 struct transfer_request *entry = request_queue_head;
546 if (request == request_queue_head) {
547 request_queue_head = request->next;
548 } else {
549 while (entry->next != NULL && entry->next != request)
550 entry = entry->next;
551 if (entry->next == request)
552 entry->next = entry->next->next;
555 free(request->url);
556 free(request);
559 #ifdef USE_CURL_MULTI
560 void process_curl_messages(void)
562 int num_messages;
563 struct active_request_slot *slot;
564 struct transfer_request *request = NULL;
565 CURLMsg *curl_message = curl_multi_info_read(curlm, &num_messages);
567 while (curl_message != NULL) {
568 if (curl_message->msg == CURLMSG_DONE) {
569 slot = active_queue_head;
570 while (slot != NULL &&
571 slot->curl != curl_message->easy_handle)
572 slot = slot->next;
573 if (slot != NULL) {
574 curl_multi_remove_handle(curlm, slot->curl);
575 active_requests--;
576 slot->done = 1;
577 slot->in_use = 0;
578 slot->curl_result = curl_message->data.result;
579 request = request_queue_head;
580 while (request != NULL &&
581 request->slot != slot)
582 request = request->next;
583 } else {
584 fprintf(stderr, "Received DONE message for unknown request!\n");
586 if (request != NULL) {
587 request->curl_result =
588 curl_message->data.result;
589 curl_easy_getinfo(slot->curl,
590 CURLINFO_HTTP_CODE,
591 &request->http_code);
592 request->slot = NULL;
594 /* Use alternates if necessary */
595 if (request->http_code == 404 &&
596 request->repo->next != NULL) {
597 request->repo = request->repo->next;
598 start_request(request);
599 } else {
600 finish_request(request);
601 request->state = COMPLETE;
604 } else {
605 fprintf(stderr, "Unknown CURL message received: %d\n",
606 (int)curl_message->msg);
608 curl_message = curl_multi_info_read(curlm, &num_messages);
612 void process_request_queue(void)
614 struct transfer_request *request = request_queue_head;
615 int num_transfers;
617 while (active_requests < max_requests && request != NULL) {
618 if (request->state == WAITING) {
619 if (has_sha1_file(request->sha1))
620 release_request(request);
621 else
622 start_request(request);
623 curl_multi_perform(curlm, &num_transfers);
625 request = request->next;
628 #endif
630 void prefetch(unsigned char *sha1)
632 struct transfer_request *newreq;
633 struct transfer_request *tail;
634 char *filename = sha1_file_name(sha1);
636 newreq = xmalloc(sizeof(*newreq));
637 memcpy(newreq->sha1, sha1, 20);
638 newreq->repo = alt;
639 newreq->url = NULL;
640 newreq->local = -1;
641 newreq->state = WAITING;
642 snprintf(newreq->filename, sizeof(newreq->filename), "%s", filename);
643 snprintf(newreq->tmpfile, sizeof(newreq->tmpfile),
644 "%s.temp", filename);
645 newreq->next = NULL;
647 if (request_queue_head == NULL) {
648 request_queue_head = newreq;
649 } else {
650 tail = request_queue_head;
651 while (tail->next != NULL) {
652 tail = tail->next;
654 tail->next = newreq;
656 #ifdef USE_CURL_MULTI
657 process_request_queue();
658 process_curl_messages();
659 #endif
662 static int fetch_index(struct alt_base *repo, unsigned char *sha1)
664 char *hex = sha1_to_hex(sha1);
665 char *filename;
666 char *url;
667 char tmpfile[PATH_MAX];
668 long prev_posn = 0;
669 char range[RANGE_HEADER_SIZE];
670 struct curl_slist *range_header = NULL;
672 FILE *indexfile;
673 struct active_request_slot *slot;
675 if (has_pack_index(sha1))
676 return 0;
678 if (get_verbosely)
679 fprintf(stderr, "Getting index for pack %s\n", hex);
681 url = xmalloc(strlen(repo->base) + 64);
682 sprintf(url, "%s/objects/pack/pack-%s.idx", repo->base, hex);
684 filename = sha1_pack_index_name(sha1);
685 snprintf(tmpfile, sizeof(tmpfile), "%s.temp", filename);
686 indexfile = fopen(tmpfile, "a");
687 if (!indexfile)
688 return error("Unable to open local file %s for pack index",
689 filename);
691 slot = get_active_slot();
692 curl_easy_setopt(slot->curl, CURLOPT_FILE, indexfile);
693 curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite);
694 curl_easy_setopt(slot->curl, CURLOPT_URL, url);
695 curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, no_pragma_header);
696 slot->local = indexfile;
698 /* If there is data present from a previous transfer attempt,
699 resume where it left off */
700 prev_posn = ftell(indexfile);
701 if (prev_posn>0) {
702 if (get_verbosely)
703 fprintf(stderr,
704 "Resuming fetch of index for pack %s at byte %ld\n",
705 hex, prev_posn);
706 sprintf(range, "Range: bytes=%ld-", prev_posn);
707 range_header = curl_slist_append(range_header, range);
708 curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, range_header);
711 if (start_active_slot(slot)) {
712 run_active_slot(slot);
713 if (slot->curl_result != CURLE_OK) {
714 fclose(indexfile);
715 return error("Unable to get pack index %s\n%s", url,
716 curl_errorstr);
718 } else {
719 return error("Unable to start request");
722 fclose(indexfile);
724 return move_temp_to_file(tmpfile, filename);
727 static int setup_index(struct alt_base *repo, unsigned char *sha1)
729 struct packed_git *new_pack;
730 if (has_pack_file(sha1))
731 return 0; // don't list this as something we can get
733 if (fetch_index(repo, sha1))
734 return -1;
736 new_pack = parse_pack_index(sha1);
737 new_pack->next = repo->packs;
738 repo->packs = new_pack;
739 return 0;
742 static int fetch_alternates(char *base)
744 int ret = 0;
745 struct buffer buffer;
746 char *url;
747 char *data;
748 int i = 0;
749 int http_specific = 1;
750 struct alt_base *tail = alt;
751 static const char null_byte = '\0';
753 struct active_request_slot *slot;
755 data = xmalloc(4096);
756 buffer.size = 4096;
757 buffer.posn = 0;
758 buffer.buffer = data;
760 if (get_verbosely)
761 fprintf(stderr, "Getting alternates list\n");
763 url = xmalloc(strlen(base) + 31);
764 sprintf(url, "%s/objects/info/http-alternates", base);
766 slot = get_active_slot();
767 curl_easy_setopt(slot->curl, CURLOPT_FILE, &buffer);
768 curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION,
769 fwrite_buffer_dynamic);
770 curl_easy_setopt(slot->curl, CURLOPT_URL, url);
771 if (start_active_slot(slot)) {
772 run_active_slot(slot);
773 if (slot->curl_result != CURLE_OK || !buffer.posn) {
774 http_specific = 0;
776 sprintf(url, "%s/objects/info/alternates", base);
778 slot = get_active_slot();
779 curl_easy_setopt(slot->curl, CURLOPT_FILE, &buffer);
780 curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION,
781 fwrite_buffer_dynamic);
782 curl_easy_setopt(slot->curl, CURLOPT_URL, url);
783 if (start_active_slot(slot)) {
784 run_active_slot(slot);
785 if (slot->curl_result != CURLE_OK) {
786 free(buffer.buffer);
787 return 0;
791 } else {
792 free(buffer.buffer);
793 return 0;
796 fwrite_buffer_dynamic(&null_byte, 1, 1, &buffer);
797 buffer.posn--;
798 data = buffer.buffer;
800 while (i < buffer.posn) {
801 int posn = i;
802 while (posn < buffer.posn && data[posn] != '\n')
803 posn++;
804 if (data[posn] == '\n') {
805 int okay = 0;
806 int serverlen = 0;
807 struct alt_base *newalt;
808 char *target = NULL;
809 if (data[i] == '/') {
810 serverlen = strchr(base + 8, '/') - base;
811 okay = 1;
812 } else if (!memcmp(data + i, "../", 3)) {
813 i += 3;
814 serverlen = strlen(base);
815 while (i + 2 < posn &&
816 !memcmp(data + i, "../", 3)) {
817 do {
818 serverlen--;
819 } while (serverlen &&
820 base[serverlen - 1] != '/');
821 i += 3;
823 // If the server got removed, give up.
824 okay = strchr(base, ':') - base + 3 <
825 serverlen;
826 } else if (http_specific) {
827 char *colon = strchr(data + i, ':');
828 char *slash = strchr(data + i, '/');
829 if (colon && slash && colon < data + posn &&
830 slash < data + posn && colon < slash) {
831 okay = 1;
834 // skip 'objects' at end
835 if (okay) {
836 target = xmalloc(serverlen + posn - i - 6);
837 strncpy(target, base, serverlen);
838 strncpy(target + serverlen, data + i,
839 posn - i - 7);
840 target[serverlen + posn - i - 7] = '\0';
841 if (get_verbosely)
842 fprintf(stderr,
843 "Also look at %s\n", target);
844 newalt = xmalloc(sizeof(*newalt));
845 newalt->next = NULL;
846 newalt->base = target;
847 newalt->got_indices = 0;
848 newalt->packs = NULL;
849 while (tail->next != NULL)
850 tail = tail->next;
851 tail->next = newalt;
852 ret++;
855 i = posn + 1;
858 free(buffer.buffer);
859 return ret;
862 static int fetch_indices(struct alt_base *repo)
864 unsigned char sha1[20];
865 char *url;
866 struct buffer buffer;
867 char *data;
868 int i = 0;
870 struct active_request_slot *slot;
872 if (repo->got_indices)
873 return 0;
875 data = xmalloc(4096);
876 buffer.size = 4096;
877 buffer.posn = 0;
878 buffer.buffer = data;
880 if (get_verbosely)
881 fprintf(stderr, "Getting pack list\n");
883 url = xmalloc(strlen(repo->base) + 21);
884 sprintf(url, "%s/objects/info/packs", repo->base);
886 slot = get_active_slot();
887 curl_easy_setopt(slot->curl, CURLOPT_FILE, &buffer);
888 curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION,
889 fwrite_buffer_dynamic);
890 curl_easy_setopt(slot->curl, CURLOPT_URL, url);
891 curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, NULL);
892 if (start_active_slot(slot)) {
893 run_active_slot(slot);
894 if (slot->curl_result != CURLE_OK) {
895 free(buffer.buffer);
896 return error("%s", curl_errorstr);
898 } else {
899 free(buffer.buffer);
900 return error("Unable to start request");
903 data = buffer.buffer;
904 while (i < buffer.posn) {
905 switch (data[i]) {
906 case 'P':
907 i++;
908 if (i + 52 < buffer.posn &&
909 !strncmp(data + i, " pack-", 6) &&
910 !strncmp(data + i + 46, ".pack\n", 6)) {
911 get_sha1_hex(data + i + 6, sha1);
912 setup_index(repo, sha1);
913 i += 51;
914 break;
916 default:
917 while (data[i] != '\n')
918 i++;
920 i++;
923 free(buffer.buffer);
924 repo->got_indices = 1;
925 return 0;
928 static int fetch_pack(struct alt_base *repo, unsigned char *sha1)
930 char *url;
931 struct packed_git *target;
932 struct packed_git **lst;
933 FILE *packfile;
934 char *filename;
935 char tmpfile[PATH_MAX];
936 int ret;
937 long prev_posn = 0;
938 char range[RANGE_HEADER_SIZE];
939 struct curl_slist *range_header = NULL;
941 struct active_request_slot *slot;
943 if (fetch_indices(repo))
944 return -1;
945 target = find_sha1_pack(sha1, repo->packs);
946 if (!target)
947 return -1;
949 if (get_verbosely) {
950 fprintf(stderr, "Getting pack %s\n",
951 sha1_to_hex(target->sha1));
952 fprintf(stderr, " which contains %s\n",
953 sha1_to_hex(sha1));
956 url = xmalloc(strlen(repo->base) + 65);
957 sprintf(url, "%s/objects/pack/pack-%s.pack",
958 repo->base, sha1_to_hex(target->sha1));
960 filename = sha1_pack_name(target->sha1);
961 snprintf(tmpfile, sizeof(tmpfile), "%s.temp", filename);
962 packfile = fopen(tmpfile, "a");
963 if (!packfile)
964 return error("Unable to open local file %s for pack",
965 filename);
967 slot = get_active_slot();
968 curl_easy_setopt(slot->curl, CURLOPT_FILE, packfile);
969 curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite);
970 curl_easy_setopt(slot->curl, CURLOPT_URL, url);
971 curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, no_pragma_header);
972 slot->local = packfile;
974 /* If there is data present from a previous transfer attempt,
975 resume where it left off */
976 prev_posn = ftell(packfile);
977 if (prev_posn>0) {
978 if (get_verbosely)
979 fprintf(stderr,
980 "Resuming fetch of pack %s at byte %ld\n",
981 sha1_to_hex(target->sha1), prev_posn);
982 sprintf(range, "Range: bytes=%ld-", prev_posn);
983 range_header = curl_slist_append(range_header, range);
984 curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, range_header);
987 if (start_active_slot(slot)) {
988 run_active_slot(slot);
989 if (slot->curl_result != CURLE_OK) {
990 fclose(packfile);
991 return error("Unable to get pack file %s\n%s", url,
992 curl_errorstr);
994 } else {
995 return error("Unable to start request");
998 fclose(packfile);
1000 ret = move_temp_to_file(tmpfile, filename);
1001 if (ret)
1002 return ret;
1004 lst = &repo->packs;
1005 while (*lst != target)
1006 lst = &((*lst)->next);
1007 *lst = (*lst)->next;
1009 if (verify_pack(target, 0))
1010 return -1;
1011 install_packed_git(target);
1013 return 0;
1016 static int fetch_object(struct alt_base *repo, unsigned char *sha1)
1018 char *hex = sha1_to_hex(sha1);
1019 int ret;
1020 struct transfer_request *request = request_queue_head;
1022 while (request != NULL && memcmp(request->sha1, sha1, 20))
1023 request = request->next;
1024 if (request == NULL)
1025 return error("Couldn't find request for %s in the queue", hex);
1027 if (has_sha1_file(request->sha1)) {
1028 release_request(request);
1029 return 0;
1032 #ifdef USE_CURL_MULTI
1033 while (request->state == WAITING) {
1034 int num_transfers;
1035 curl_multi_perform(curlm, &num_transfers);
1036 if (num_transfers < active_requests) {
1037 process_curl_messages();
1038 process_request_queue();
1041 #else
1042 start_request(request);
1043 #endif
1045 while (request->state == ACTIVE) {
1046 run_active_slot(request->slot);
1047 #ifndef USE_CURL_MULTI
1048 request->curl_result = request->slot->curl_result;
1049 curl_easy_getinfo(request->slot->curl,
1050 CURLINFO_HTTP_CODE,
1051 &request->http_code);
1052 request->slot = NULL;
1054 /* Use alternates if necessary */
1055 if (request->http_code == 404 &&
1056 request->repo->next != NULL) {
1057 request->repo = request->repo->next;
1058 start_request(request);
1059 } else {
1060 finish_request(request);
1061 request->state = COMPLETE;
1063 #endif
1066 if (request->state == ABORTED) {
1067 release_request(request);
1068 return error("Request for %s aborted", hex);
1071 if (request->curl_result != CURLE_OK && request->http_code != 416) {
1072 ret = error("%s (curl_result = %d, http_code = %ld, sha1 = %s)",
1073 request->errorstr, request->curl_result,
1074 request->http_code, hex);
1075 release_request(request);
1076 return ret;
1079 if (request->zret != Z_STREAM_END) {
1080 ret = error("File %s (%s) corrupt\n", hex, request->url);
1081 release_request(request);
1082 return ret;
1085 if (memcmp(request->sha1, request->real_sha1, 20)) {
1086 release_request(request);
1087 return error("File %s has bad hash\n", hex);
1090 if (request->rename < 0) {
1091 ret = error("unable to write sha1 filename %s: %s",
1092 request->filename,
1093 strerror(request->rename));
1094 release_request(request);
1095 return ret;
1098 release_request(request);
1099 return 0;
1102 int fetch(unsigned char *sha1)
1104 struct alt_base *altbase = alt;
1106 if (!fetch_object(altbase, sha1))
1107 return 0;
1108 while (altbase) {
1109 if (!fetch_pack(altbase, sha1))
1110 return 0;
1111 altbase = altbase->next;
1113 return error("Unable to find %s under %s\n", sha1_to_hex(sha1),
1114 alt->base);
1117 static inline int needs_quote(int ch)
1119 switch (ch) {
1120 case '/': case '-': case '.':
1121 case 'A'...'Z': case 'a'...'z': case '0'...'9':
1122 return 0;
1123 default:
1124 return 1;
1128 static inline int hex(int v)
1130 if (v < 10) return '0' + v;
1131 else return 'A' + v - 10;
1134 static char *quote_ref_url(const char *base, const char *ref)
1136 const char *cp;
1137 char *dp, *qref;
1138 int len, baselen, ch;
1140 baselen = strlen(base);
1141 len = baselen + 6; /* "refs/" + NUL */
1142 for (cp = ref; (ch = *cp) != 0; cp++, len++)
1143 if (needs_quote(ch))
1144 len += 2; /* extra two hex plus replacement % */
1145 qref = xmalloc(len);
1146 memcpy(qref, base, baselen);
1147 memcpy(qref + baselen, "refs/", 5);
1148 for (cp = ref, dp = qref + baselen + 5; (ch = *cp) != 0; cp++) {
1149 if (needs_quote(ch)) {
1150 *dp++ = '%';
1151 *dp++ = hex((ch >> 4) & 0xF);
1152 *dp++ = hex(ch & 0xF);
1154 else
1155 *dp++ = ch;
1157 *dp = 0;
1159 return qref;
1162 int fetch_ref(char *ref, unsigned char *sha1)
1164 char *url;
1165 char hex[42];
1166 struct buffer buffer;
1167 char *base = alt->base;
1168 struct active_request_slot *slot;
1169 buffer.size = 41;
1170 buffer.posn = 0;
1171 buffer.buffer = hex;
1172 hex[41] = '\0';
1174 url = quote_ref_url(base, ref);
1175 slot = get_active_slot();
1176 curl_easy_setopt(slot->curl, CURLOPT_FILE, &buffer);
1177 curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite_buffer);
1178 curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, NULL);
1179 curl_easy_setopt(slot->curl, CURLOPT_URL, url);
1180 if (start_active_slot(slot)) {
1181 run_active_slot(slot);
1182 if (slot->curl_result != CURLE_OK)
1183 return error("Couldn't get %s for %s\n%s",
1184 url, ref, curl_errorstr);
1185 } else {
1186 return error("Unable to start request");
1189 hex[40] = '\0';
1190 get_sha1_hex(hex, sha1);
1191 return 0;
1194 int main(int argc, char **argv)
1196 char *commit_id;
1197 char *url;
1198 int arg = 1;
1199 struct active_request_slot *slot;
1200 char *low_speed_limit;
1201 char *low_speed_time;
1203 while (arg < argc && argv[arg][0] == '-') {
1204 if (argv[arg][1] == 't') {
1205 get_tree = 1;
1206 } else if (argv[arg][1] == 'c') {
1207 get_history = 1;
1208 } else if (argv[arg][1] == 'a') {
1209 get_all = 1;
1210 get_tree = 1;
1211 get_history = 1;
1212 } else if (argv[arg][1] == 'v') {
1213 get_verbosely = 1;
1214 } else if (argv[arg][1] == 'w') {
1215 write_ref = argv[arg + 1];
1216 arg++;
1217 } else if (!strcmp(argv[arg], "--recover")) {
1218 get_recover = 1;
1220 arg++;
1222 if (argc < arg + 2) {
1223 usage("git-http-fetch [-c] [-t] [-a] [-d] [-v] [--recover] [-w ref] commit-id url");
1224 return 1;
1226 commit_id = argv[arg];
1227 url = argv[arg + 1];
1229 curl_global_init(CURL_GLOBAL_ALL);
1231 #ifdef USE_CURL_MULTI
1233 char *http_max_requests = getenv("GIT_HTTP_MAX_REQUESTS");
1234 if (http_max_requests != NULL)
1235 max_requests = atoi(http_max_requests);
1238 curlm = curl_multi_init();
1239 if (curlm == NULL) {
1240 fprintf(stderr, "Error creating curl multi handle.\n");
1241 return 1;
1243 #endif
1245 if (getenv("GIT_SSL_NO_VERIFY"))
1246 curl_ssl_verify = 0;
1248 ssl_cert = getenv("GIT_SSL_CERT");
1249 #if LIBCURL_VERSION_NUM >= 0x070902
1250 ssl_key = getenv("GIT_SSL_KEY");
1251 #endif
1252 #if LIBCURL_VERSION_NUM >= 0x070908
1253 ssl_capath = getenv("GIT_SSL_CAPATH");
1254 #endif
1255 ssl_cainfo = getenv("GIT_SSL_CAINFO");
1257 low_speed_limit = getenv("GIT_HTTP_LOW_SPEED_LIMIT");
1258 if (low_speed_limit != NULL)
1259 curl_low_speed_limit = strtol(low_speed_limit, NULL, 10);
1260 low_speed_time = getenv("GIT_HTTP_LOW_SPEED_TIME");
1261 if (low_speed_time != NULL)
1262 curl_low_speed_time = strtol(low_speed_time, NULL, 10);
1264 git_config(http_options);
1266 if (curl_ssl_verify == -1)
1267 curl_ssl_verify = 1;
1269 #ifdef USE_CURL_MULTI
1270 if (max_requests < 1)
1271 max_requests = DEFAULT_MAX_REQUESTS;
1272 #endif
1274 pragma_header = curl_slist_append(pragma_header, "Pragma: no-cache");
1275 no_pragma_header = curl_slist_append(no_pragma_header, "Pragma:");
1276 no_range_header = curl_slist_append(no_range_header, "Range:");
1278 #ifndef NO_CURL_EASY_DUPHANDLE
1279 curl_default = get_curl_handle();
1280 #endif
1282 alt = xmalloc(sizeof(*alt));
1283 alt->base = url;
1284 alt->got_indices = 0;
1285 alt->packs = NULL;
1286 alt->next = NULL;
1287 fetch_alternates(alt->base);
1289 if (pull(commit_id))
1290 return 1;
1292 curl_slist_free_all(pragma_header);
1293 curl_slist_free_all(no_pragma_header);
1294 curl_slist_free_all(no_range_header);
1295 #ifndef NO_CURL_EASY_DUPHANDLE
1296 curl_easy_cleanup(curl_default);
1297 #endif
1298 slot = active_queue_head;
1299 while (slot != NULL) {
1300 curl_easy_cleanup(slot->curl);
1301 slot = slot->next;
1303 #ifdef USE_CURL_MULTI
1304 curl_multi_cleanup(curlm);
1305 #endif
1306 curl_global_cleanup();
1307 return 0;