?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
get rid of num_packed_objects()
[git.git] / http-fetch.c
blob09baedc18ae320a2f09fa61d9065bedb69c02a41
1 #include "cache.h"
2 #include "commit.h"
3 #include "pack.h"
4 #include "fetch.h"
5 #include "http.h"
6
7 #define PREV_BUF_SIZE 4096
8 #define RANGE_HEADER_SIZE 30
9
10 static int commits_on_stdin;
11
12 static int got_alternates = -1;
13 static int corrupt_object_found;
14
15 static struct curl_slist *no_pragma_header;
16
17 struct alt_base
18 {
19 char *base;
20 int got_indices;
21 struct packed_git *packs;
22 struct alt_base *next;
23 };
24
25 static struct alt_base *alt;
26
27 enum object_request_state {
28 WAITING,
29 ABORTED,
30 ACTIVE,
31 COMPLETE,
32 };
33
34 struct object_request
35 {
36 unsigned char sha1[20];
37 struct alt_base *repo;
38 char *url;
39 char filename[PATH_MAX];
40 char tmpfile[PATH_MAX];
41 int local;
42 enum object_request_state state;
43 CURLcode curl_result;
44 char errorstr[CURL_ERROR_SIZE];
45 long http_code;
46 unsigned char real_sha1[20];
47 SHA_CTX c;
48 z_stream stream;
49 int zret;
50 int rename;
51 struct active_request_slot *slot;
52 struct object_request *next;
53 };
54
55 struct alternates_request {
56 const char *base;
57 char *url;
58 struct buffer *buffer;
59 struct active_request_slot *slot;
60 int http_specific;
61 };
62
63 static struct object_request *object_queue_head;
64
65 static size_t fwrite_sha1_file(void *ptr, size_t eltsize, size_t nmemb,
66 void *data)
67 {
68 unsigned char expn[4096];
69 size_t size = eltsize * nmemb;
70 int posn = 0;
71 struct object_request *obj_req = (struct object_request *)data;
72 do {
73 ssize_t retval = xwrite(obj_req->local,
74 (char *) ptr + posn, size - posn);
75 if (retval < 0)
76 return posn;
77 posn += retval;
78 } while (posn < size);
79
80 obj_req->stream.avail_in = size;
81 obj_req->stream.next_in = ptr;
82 do {
83 obj_req->stream.next_out = expn;
84 obj_req->stream.avail_out = sizeof(expn);
85 obj_req->zret = inflate(&obj_req->stream, Z_SYNC_FLUSH);
86 SHA1_Update(&obj_req->c, expn,
87 sizeof(expn) - obj_req->stream.avail_out);
88 } while (obj_req->stream.avail_in && obj_req->zret == Z_OK);
89 data_received++;
90 return size;
91 }
92
93 static int missing__target(int code, int result)
94 {
95 return /* file:// URL -- do we ever use one??? */
96 (result == CURLE_FILE_COULDNT_READ_FILE) ||
97 /* http:// and https:// URL */
98 (code == 404 && result == CURLE_HTTP_RETURNED_ERROR) ||
99 /* ftp:// URL */
100 (code == 550 && result == CURLE_FTP_COULDNT_RETR_FILE)
101 ;
102 }
103
104 #define missing_target(a) missing__target((a)->http_code, (a)->curl_result)
105
106 static void fetch_alternates(const char *base);
107
108 static void process_object_response(void *callback_data);
109
110 static void start_object_request(struct object_request *obj_req)
111 {
112 char *hex = sha1_to_hex(obj_req->sha1);
113 char prevfile[PATH_MAX];
114 char *url;
115 char *posn;
116 int prevlocal;
117 unsigned char prev_buf[PREV_BUF_SIZE];
118 ssize_t prev_read = 0;
119 long prev_posn = 0;
120 char range[RANGE_HEADER_SIZE];
121 struct curl_slist *range_header = NULL;
122 struct active_request_slot *slot;
123
124 snprintf(prevfile, sizeof(prevfile), "%s.prev", obj_req->filename);
125 unlink(prevfile);
126 rename(obj_req->tmpfile, prevfile);
127 unlink(obj_req->tmpfile);
128
129 if (obj_req->local != -1)
130 error("fd leakage in start: %d", obj_req->local);
131 obj_req->local = open(obj_req->tmpfile,
132 O_WRONLY | O_CREAT | O_EXCL, 0666);
133 /* This could have failed due to the "lazy directory creation";
134 * try to mkdir the last path component.
135 */
136 if (obj_req->local < 0 && errno == ENOENT) {
137 char *dir = strrchr(obj_req->tmpfile, '/');
138 if (dir) {
139 *dir = 0;
140 mkdir(obj_req->tmpfile, 0777);
141 *dir = '/';
142 }
143 obj_req->local = open(obj_req->tmpfile,
144 O_WRONLY | O_CREAT | O_EXCL, 0666);
145 }
146
147 if (obj_req->local < 0) {
148 obj_req->state = ABORTED;
149 error("Couldn't create temporary file %s for %s: %s",
150 obj_req->tmpfile, obj_req->filename, strerror(errno));
151 return;
152 }
153
154 memset(&obj_req->stream, 0, sizeof(obj_req->stream));
155
156 inflateInit(&obj_req->stream);
157
158 SHA1_Init(&obj_req->c);
159
160 url = xmalloc(strlen(obj_req->repo->base) + 51);
161 obj_req->url = xmalloc(strlen(obj_req->repo->base) + 51);
162 strcpy(url, obj_req->repo->base);
163 posn = url + strlen(obj_req->repo->base);
164 strcpy(posn, "/objects/");
165 posn += 9;
166 memcpy(posn, hex, 2);
167 posn += 2;
168 *(posn++) = '/';
169 strcpy(posn, hex + 2);
170 strcpy(obj_req->url, url);
171
172 /* If a previous temp file is present, process what was already
173 fetched. */
174 prevlocal = open(prevfile, O_RDONLY);
175 if (prevlocal != -1) {
176 do {
177 prev_read = xread(prevlocal, prev_buf, PREV_BUF_SIZE);
178 if (prev_read>0) {
179 if (fwrite_sha1_file(prev_buf,
180 1,
181 prev_read,
182 obj_req) == prev_read) {
183 prev_posn += prev_read;
184 } else {
185 prev_read = -1;
186 }
187 }
188 } while (prev_read > 0);
189 close(prevlocal);
190 }
191 unlink(prevfile);
192
193 /* Reset inflate/SHA1 if there was an error reading the previous temp
194 file; also rewind to the beginning of the local file. */
195 if (prev_read == -1) {
196 memset(&obj_req->stream, 0, sizeof(obj_req->stream));
197 inflateInit(&obj_req->stream);
198 SHA1_Init(&obj_req->c);
199 if (prev_posn>0) {
200 prev_posn = 0;
201 lseek(obj_req->local, 0, SEEK_SET);
202 ftruncate(obj_req->local, 0);
203 }
204 }
205
206 slot = get_active_slot();
207 slot->callback_func = process_object_response;
208 slot->callback_data = obj_req;
209 obj_req->slot = slot;
210
211 curl_easy_setopt(slot->curl, CURLOPT_FILE, obj_req);
212 curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite_sha1_file);
213 curl_easy_setopt(slot->curl, CURLOPT_ERRORBUFFER, obj_req->errorstr);
214 curl_easy_setopt(slot->curl, CURLOPT_URL, url);
215 curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, no_pragma_header);
216
217 /* If we have successfully processed data from a previous fetch
218 attempt, only fetch the data we don't already have. */
219 if (prev_posn>0) {
220 if (get_verbosely)
221 fprintf(stderr,
222 "Resuming fetch of object %s at byte %ld\n",
223 hex, prev_posn);
224 sprintf(range, "Range: bytes=%ld-", prev_posn);
225 range_header = curl_slist_append(range_header, range);
226 curl_easy_setopt(slot->curl,
227 CURLOPT_HTTPHEADER, range_header);
228 }
229
230 /* Try to get the request started, abort the request on error */
231 obj_req->state = ACTIVE;
232 if (!start_active_slot(slot)) {
233 obj_req->state = ABORTED;
234 obj_req->slot = NULL;
235 close(obj_req->local); obj_req->local = -1;
236 free(obj_req->url);
237 return;
238 }
239 }
240
241 static void finish_object_request(struct object_request *obj_req)
242 {
243 struct stat st;
244
245 fchmod(obj_req->local, 0444);
246 close(obj_req->local); obj_req->local = -1;
247
248 if (obj_req->http_code == 416) {
249 fprintf(stderr, "Warning: requested range invalid; we may already have all the data.\n");
250 } else if (obj_req->curl_result != CURLE_OK) {
251 if (stat(obj_req->tmpfile, &st) == 0)
252 if (st.st_size == 0)
253 unlink(obj_req->tmpfile);
254 return;
255 }
256
257 inflateEnd(&obj_req->stream);
258 SHA1_Final(obj_req->real_sha1, &obj_req->c);
259 if (obj_req->zret != Z_STREAM_END) {
260 unlink(obj_req->tmpfile);
261 return;
262 }
263 if (hashcmp(obj_req->sha1, obj_req->real_sha1)) {
264 unlink(obj_req->tmpfile);
265 return;
266 }
267 obj_req->rename =
268 move_temp_to_file(obj_req->tmpfile, obj_req->filename);
269
270 if (obj_req->rename == 0)
271 pull_say("got %s\n", sha1_to_hex(obj_req->sha1));
272 }
273
274 static void process_object_response(void *callback_data)
275 {
276 struct object_request *obj_req =
277 (struct object_request *)callback_data;
278
279 obj_req->curl_result = obj_req->slot->curl_result;
280 obj_req->http_code = obj_req->slot->http_code;
281 obj_req->slot = NULL;
282 obj_req->state = COMPLETE;
283
284 /* Use alternates if necessary */
285 if (missing_target(obj_req)) {
286 fetch_alternates(alt->base);
287 if (obj_req->repo->next != NULL) {
288 obj_req->repo =
289 obj_req->repo->next;
290 close(obj_req->local);
291 obj_req->local = -1;
292 start_object_request(obj_req);
293 return;
294 }
295 }
296
297 finish_object_request(obj_req);
298 }
299
300 static void release_object_request(struct object_request *obj_req)
301 {
302 struct object_request *entry = object_queue_head;
303
304 if (obj_req->local != -1)
305 error("fd leakage in release: %d", obj_req->local);
306 if (obj_req == object_queue_head) {
307 object_queue_head = obj_req->next;
308 } else {
309 while (entry->next != NULL && entry->next != obj_req)
310 entry = entry->next;
311 if (entry->next == obj_req)
312 entry->next = entry->next->next;
313 }
314
315 free(obj_req->url);
316 free(obj_req);
317 }
318
319 #ifdef USE_CURL_MULTI
320 void fill_active_slots(void)
321 {
322 struct object_request *obj_req = object_queue_head;
323 struct active_request_slot *slot = active_queue_head;
324 int num_transfers;
325
326 while (active_requests < max_requests && obj_req != NULL) {
327 if (obj_req->state == WAITING) {
328 if (has_sha1_file(obj_req->sha1))
329 obj_req->state = COMPLETE;
330 else
331 start_object_request(obj_req);
332 curl_multi_perform(curlm, &num_transfers);
333 }
334 obj_req = obj_req->next;
335 }
336
337 while (slot != NULL) {
338 if (!slot->in_use && slot->curl != NULL) {
339 curl_easy_cleanup(slot->curl);
340 slot->curl = NULL;
341 }
342 slot = slot->next;
343 }
344 }
345 #endif
346
347 void prefetch(unsigned char *sha1)
348 {
349 struct object_request *newreq;
350 struct object_request *tail;
351 char *filename = sha1_file_name(sha1);
352
353 newreq = xmalloc(sizeof(*newreq));
354 hashcpy(newreq->sha1, sha1);
355 newreq->repo = alt;
356 newreq->url = NULL;
357 newreq->local = -1;
358 newreq->state = WAITING;
359 snprintf(newreq->filename, sizeof(newreq->filename), "%s", filename);
360 snprintf(newreq->tmpfile, sizeof(newreq->tmpfile),
361 "%s.temp", filename);
362 newreq->slot = NULL;
363 newreq->next = NULL;
364
365 if (object_queue_head == NULL) {
366 object_queue_head = newreq;
367 } else {
368 tail = object_queue_head;
369 while (tail->next != NULL) {
370 tail = tail->next;
371 }
372 tail->next = newreq;
373 }
374
375 #ifdef USE_CURL_MULTI
376 fill_active_slots();
377 step_active_slots();
378 #endif
379 }
380
381 static int fetch_index(struct alt_base *repo, unsigned char *sha1)
382 {
383 char *hex = sha1_to_hex(sha1);
384 char *filename;
385 char *url;
386 char tmpfile[PATH_MAX];
387 long prev_posn = 0;
388 char range[RANGE_HEADER_SIZE];
389 struct curl_slist *range_header = NULL;
390
391 FILE *indexfile;
392 struct active_request_slot *slot;
393 struct slot_results results;
394
395 if (has_pack_index(sha1))
396 return 0;
397
398 if (get_verbosely)
399 fprintf(stderr, "Getting index for pack %s\n", hex);
400
401 url = xmalloc(strlen(repo->base) + 64);
402 sprintf(url, "%s/objects/pack/pack-%s.idx", repo->base, hex);
403
404 filename = sha1_pack_index_name(sha1);
405 snprintf(tmpfile, sizeof(tmpfile), "%s.temp", filename);
406 indexfile = fopen(tmpfile, "a");
407 if (!indexfile)
408 return error("Unable to open local file %s for pack index",
409 filename);
410
411 slot = get_active_slot();
412 slot->results = &results;
413 curl_easy_setopt(slot->curl, CURLOPT_FILE, indexfile);
414 curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite);
415 curl_easy_setopt(slot->curl, CURLOPT_URL, url);
416 curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, no_pragma_header);
417 slot->local = indexfile;
418
419 /* If there is data present from a previous transfer attempt,
420 resume where it left off */
421 prev_posn = ftell(indexfile);
422 if (prev_posn>0) {
423 if (get_verbosely)
424 fprintf(stderr,
425 "Resuming fetch of index for pack %s at byte %ld\n",
426 hex, prev_posn);
427 sprintf(range, "Range: bytes=%ld-", prev_posn);
428 range_header = curl_slist_append(range_header, range);
429 curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, range_header);
430 }
431
432 if (start_active_slot(slot)) {
433 run_active_slot(slot);
434 if (results.curl_result != CURLE_OK) {
435 fclose(indexfile);
436 return error("Unable to get pack index %s\n%s", url,
437 curl_errorstr);
438 }
439 } else {
440 fclose(indexfile);
441 return error("Unable to start request");
442 }
443
444 fclose(indexfile);
445
446 return move_temp_to_file(tmpfile, filename);
447 }
448
449 static int setup_index(struct alt_base *repo, unsigned char *sha1)
450 {
451 struct packed_git *new_pack;
452 if (has_pack_file(sha1))
453 return 0; /* don't list this as something we can get */
454
455 if (fetch_index(repo, sha1))
456 return -1;
457
458 new_pack = parse_pack_index(sha1);
459 new_pack->next = repo->packs;
460 repo->packs = new_pack;
461 return 0;
462 }
463
464 static void process_alternates_response(void *callback_data)
465 {
466 struct alternates_request *alt_req =
467 (struct alternates_request *)callback_data;
468 struct active_request_slot *slot = alt_req->slot;
469 struct alt_base *tail = alt;
470 const char *base = alt_req->base;
471 static const char null_byte = '\0';
472 char *data;
473 int i = 0;
474
475 if (alt_req->http_specific) {
476 if (slot->curl_result != CURLE_OK ||
477 !alt_req->buffer->posn) {
478
479 /* Try reusing the slot to get non-http alternates */
480 alt_req->http_specific = 0;
481 sprintf(alt_req->url, "%s/objects/info/alternates",
482 base);
483 curl_easy_setopt(slot->curl, CURLOPT_URL,
484 alt_req->url);
485 active_requests++;
486 slot->in_use = 1;
487 if (slot->finished != NULL)
488 (*slot->finished) = 0;
489 if (!start_active_slot(slot)) {
490 got_alternates = -1;
491 slot->in_use = 0;
492 if (slot->finished != NULL)
493 (*slot->finished) = 1;
494 }
495 return;
496 }
497 } else if (slot->curl_result != CURLE_OK) {
498 if (!missing_target(slot)) {
499 got_alternates = -1;
500 return;
501 }
502 }
503
504 fwrite_buffer(&null_byte, 1, 1, alt_req->buffer);
505 alt_req->buffer->posn--;
506 data = alt_req->buffer->buffer;
507
508 while (i < alt_req->buffer->posn) {
509 int posn = i;
510 while (posn < alt_req->buffer->posn && data[posn] != '\n')
511 posn++;
512 if (data[posn] == '\n') {
513 int okay = 0;
514 int serverlen = 0;
515 struct alt_base *newalt;
516 char *target = NULL;
517 if (data[i] == '/') {
518 /* This counts
519 * http://git.host/pub/scm/linux.git/
520 * -----------here^
521 * so memcpy(dst, base, serverlen) will
522 * copy up to "...git.host".
523 */
524 const char *colon_ss = strstr(base,"://");
525 if (colon_ss) {
526 serverlen = (strchr(colon_ss + 3, '/')
527 - base);
528 okay = 1;
529 }
530 } else if (!memcmp(data + i, "../", 3)) {
531 /* Relative URL; chop the corresponding
532 * number of subpath from base (and ../
533 * from data), and concatenate the result.
534 *
535 * The code first drops ../ from data, and
536 * then drops one ../ from data and one path
537 * from base. IOW, one extra ../ is dropped
538 * from data than path is dropped from base.
539 *
540 * This is not wrong. The alternate in
541 * http://git.host/pub/scm/linux.git/
542 * to borrow from
543 * http://git.host/pub/scm/linus.git/
544 * is ../../linus.git/objects/. You need
545 * two ../../ to borrow from your direct
546 * neighbour.
547 */
548 i += 3;
549 serverlen = strlen(base);
550 while (i + 2 < posn &&
551 !memcmp(data + i, "../", 3)) {
552 do {
553 serverlen--;
554 } while (serverlen &&
555 base[serverlen - 1] != '/');
556 i += 3;
557 }
558 /* If the server got removed, give up. */
559 okay = strchr(base, ':') - base + 3 <
560 serverlen;
561 } else if (alt_req->http_specific) {
562 char *colon = strchr(data + i, ':');
563 char *slash = strchr(data + i, '/');
564 if (colon && slash && colon < data + posn &&
565 slash < data + posn && colon < slash) {
566 okay = 1;
567 }
568 }
569 /* skip "objects\n" at end */
570 if (okay) {
571 target = xmalloc(serverlen + posn - i - 6);
572 memcpy(target, base, serverlen);
573 memcpy(target + serverlen, data + i,
574 posn - i - 7);
575 target[serverlen + posn - i - 7] = 0;
576 if (get_verbosely)
577 fprintf(stderr,
578 "Also look at %s\n", target);
579 newalt = xmalloc(sizeof(*newalt));
580 newalt->next = NULL;
581 newalt->base = target;
582 newalt->got_indices = 0;
583 newalt->packs = NULL;
584
585 while (tail->next != NULL)
586 tail = tail->next;
587 tail->next = newalt;
588 }
589 }
590 i = posn + 1;
591 }
592
593 got_alternates = 1;
594 }
595
596 static void fetch_alternates(const char *base)
597 {
598 struct buffer buffer;
599 char *url;
600 char *data;
601 struct active_request_slot *slot;
602 struct alternates_request alt_req;
603
604 /* If another request has already started fetching alternates,
605 wait for them to arrive and return to processing this request's
606 curl message */
607 #ifdef USE_CURL_MULTI
608 while (got_alternates == 0) {
609 step_active_slots();
610 }
611 #endif
612
613 /* Nothing to do if they've already been fetched */
614 if (got_alternates == 1)
615 return;
616
617 /* Start the fetch */
618 got_alternates = 0;
619
620 data = xmalloc(4096);
621 buffer.size = 4096;
622 buffer.posn = 0;
623 buffer.buffer = data;
624
625 if (get_verbosely)
626 fprintf(stderr, "Getting alternates list for %s\n", base);
627
628 url = xmalloc(strlen(base) + 31);
629 sprintf(url, "%s/objects/info/http-alternates", base);
630
631 /* Use a callback to process the result, since another request
632 may fail and need to have alternates loaded before continuing */
633 slot = get_active_slot();
634 slot->callback_func = process_alternates_response;
635 slot->callback_data = &alt_req;
636
637 curl_easy_setopt(slot->curl, CURLOPT_FILE, &buffer);
638 curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite_buffer);
639 curl_easy_setopt(slot->curl, CURLOPT_URL, url);
640
641 alt_req.base = base;
642 alt_req.url = url;
643 alt_req.buffer = &buffer;
644 alt_req.http_specific = 1;
645 alt_req.slot = slot;
646
647 if (start_active_slot(slot))
648 run_active_slot(slot);
649 else
650 got_alternates = -1;
651
652 free(data);
653 free(url);
654 }
655
656 static int fetch_indices(struct alt_base *repo)
657 {
658 unsigned char sha1[20];
659 char *url;
660 struct buffer buffer;
661 char *data;
662 int i = 0;
663
664 struct active_request_slot *slot;
665 struct slot_results results;
666
667 if (repo->got_indices)
668 return 0;
669
670 data = xmalloc(4096);
671 buffer.size = 4096;
672 buffer.posn = 0;
673 buffer.buffer = data;
674
675 if (get_verbosely)
676 fprintf(stderr, "Getting pack list for %s\n", repo->base);
677
678 url = xmalloc(strlen(repo->base) + 21);
679 sprintf(url, "%s/objects/info/packs", repo->base);
680
681 slot = get_active_slot();
682 slot->results = &results;
683 curl_easy_setopt(slot->curl, CURLOPT_FILE, &buffer);
684 curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite_buffer);
685 curl_easy_setopt(slot->curl, CURLOPT_URL, url);
686 curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, NULL);
687 if (start_active_slot(slot)) {
688 run_active_slot(slot);
689 if (results.curl_result != CURLE_OK) {
690 if (missing_target(&results)) {
691 repo->got_indices = 1;
692 free(buffer.buffer);
693 return 0;
694 } else {
695 repo->got_indices = 0;
696 free(buffer.buffer);
697 return error("%s", curl_errorstr);
698 }
699 }
700 } else {
701 repo->got_indices = 0;
702 free(buffer.buffer);
703 return error("Unable to start request");
704 }
705
706 data = buffer.buffer;
707 while (i < buffer.posn) {
708 switch (data[i]) {
709 case 'P':
710 i++;
711 if (i + 52 <= buffer.posn &&
712 !prefixcmp(data + i, " pack-") &&
713 !prefixcmp(data + i + 46, ".pack\n")) {
714 get_sha1_hex(data + i + 6, sha1);
715 setup_index(repo, sha1);
716 i += 51;
717 break;
718 }
719 default:
720 while (i < buffer.posn && data[i] != '\n')
721 i++;
722 }
723 i++;
724 }
725
726 free(buffer.buffer);
727 repo->got_indices = 1;
728 return 0;
729 }
730
731 static int fetch_pack(struct alt_base *repo, unsigned char *sha1)
732 {
733 char *url;
734 struct packed_git *target;
735 struct packed_git **lst;
736 FILE *packfile;
737 char *filename;
738 char tmpfile[PATH_MAX];
739 int ret;
740 long prev_posn = 0;
741 char range[RANGE_HEADER_SIZE];
742 struct curl_slist *range_header = NULL;
743
744 struct active_request_slot *slot;
745 struct slot_results results;
746
747 if (fetch_indices(repo))
748 return -1;
749 target = find_sha1_pack(sha1, repo->packs);
750 if (!target)
751 return -1;
752
753 if (get_verbosely) {
754 fprintf(stderr, "Getting pack %s\n",
755 sha1_to_hex(target->sha1));
756 fprintf(stderr, " which contains %s\n",
757 sha1_to_hex(sha1));
758 }
759
760 url = xmalloc(strlen(repo->base) + 65);
761 sprintf(url, "%s/objects/pack/pack-%s.pack",
762 repo->base, sha1_to_hex(target->sha1));
763
764 filename = sha1_pack_name(target->sha1);
765 snprintf(tmpfile, sizeof(tmpfile), "%s.temp", filename);
766 packfile = fopen(tmpfile, "a");
767 if (!packfile)
768 return error("Unable to open local file %s for pack",
769 filename);
770
771 slot = get_active_slot();
772 slot->results = &results;
773 curl_easy_setopt(slot->curl, CURLOPT_FILE, packfile);
774 curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite);
775 curl_easy_setopt(slot->curl, CURLOPT_URL, url);
776 curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, no_pragma_header);
777 slot->local = packfile;
778
779 /* If there is data present from a previous transfer attempt,
780 resume where it left off */
781 prev_posn = ftell(packfile);
782 if (prev_posn>0) {
783 if (get_verbosely)
784 fprintf(stderr,
785 "Resuming fetch of pack %s at byte %ld\n",
786 sha1_to_hex(target->sha1), prev_posn);
787 sprintf(range, "Range: bytes=%ld-", prev_posn);
788 range_header = curl_slist_append(range_header, range);
789 curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, range_header);
790 }
791
792 if (start_active_slot(slot)) {
793 run_active_slot(slot);
794 if (results.curl_result != CURLE_OK) {
795 fclose(packfile);
796 return error("Unable to get pack file %s\n%s", url,
797 curl_errorstr);
798 }
799 } else {
800 fclose(packfile);
801 return error("Unable to start request");
802 }
803
804 target->pack_size = ftell(packfile);
805 fclose(packfile);
806
807 ret = move_temp_to_file(tmpfile, filename);
808 if (ret)
809 return ret;
810
811 lst = &repo->packs;
812 while (*lst != target)
813 lst = &((*lst)->next);
814 *lst = (*lst)->next;
815
816 if (verify_pack(target, 0))
817 return -1;
818 install_packed_git(target);
819
820 return 0;
821 }
822
823 static void abort_object_request(struct object_request *obj_req)
824 {
825 if (obj_req->local >= 0) {
826 close(obj_req->local);
827 obj_req->local = -1;
828 }
829 unlink(obj_req->tmpfile);
830 if (obj_req->slot) {
831 release_active_slot(obj_req->slot);
832 obj_req->slot = NULL;
833 }
834 release_object_request(obj_req);
835 }
836
837 static int fetch_object(struct alt_base *repo, unsigned char *sha1)
838 {
839 char *hex = sha1_to_hex(sha1);
840 int ret = 0;
841 struct object_request *obj_req = object_queue_head;
842
843 while (obj_req != NULL && hashcmp(obj_req->sha1, sha1))
844 obj_req = obj_req->next;
845 if (obj_req == NULL)
846 return error("Couldn't find request for %s in the queue", hex);
847
848 if (has_sha1_file(obj_req->sha1)) {
849 abort_object_request(obj_req);
850 return 0;
851 }
852
853 #ifdef USE_CURL_MULTI
854 while (obj_req->state == WAITING) {
855 step_active_slots();
856 }
857 #else
858 start_object_request(obj_req);
859 #endif
860
861 while (obj_req->state == ACTIVE) {
862 run_active_slot(obj_req->slot);
863 }
864 if (obj_req->local != -1) {
865 close(obj_req->local); obj_req->local = -1;
866 }
867
868 if (obj_req->state == ABORTED) {
869 ret = error("Request for %s aborted", hex);
870 } else if (obj_req->curl_result != CURLE_OK &&
871 obj_req->http_code != 416) {
872 if (missing_target(obj_req))
873 ret = -1; /* Be silent, it is probably in a pack. */
874 else
875 ret = error("%s (curl_result = %d, http_code = %ld, sha1 = %s)",
876 obj_req->errorstr, obj_req->curl_result,
877 obj_req->http_code, hex);
878 } else if (obj_req->zret != Z_STREAM_END) {
879 corrupt_object_found++;
880 ret = error("File %s (%s) corrupt", hex, obj_req->url);
881 } else if (hashcmp(obj_req->sha1, obj_req->real_sha1)) {
882 ret = error("File %s has bad hash", hex);
883 } else if (obj_req->rename < 0) {
884 ret = error("unable to write sha1 filename %s",
885 obj_req->filename);
886 }
887
888 release_object_request(obj_req);
889 return ret;
890 }
891
892 int fetch(unsigned char *sha1)
893 {
894 struct alt_base *altbase = alt;
895
896 if (!fetch_object(altbase, sha1))
897 return 0;
898 while (altbase) {
899 if (!fetch_pack(altbase, sha1))
900 return 0;
901 fetch_alternates(alt->base);
902 altbase = altbase->next;
903 }
904 return error("Unable to find %s under %s", sha1_to_hex(sha1),
905 alt->base);
906 }
907
908 static inline int needs_quote(int ch)
909 {
910 if (((ch >= 'A') && (ch <= 'Z'))
911 || ((ch >= 'a') && (ch <= 'z'))
912 || ((ch >= '0') && (ch <= '9'))
913 || (ch == '/')
914 || (ch == '-')
915 || (ch == '.'))
916 return 0;
917 return 1;
918 }
919
920 static inline int hex(int v)
921 {
922 if (v < 10) return '0' + v;
923 else return 'A' + v - 10;
924 }
925
926 static char *quote_ref_url(const char *base, const char *ref)
927 {
928 const char *cp;
929 char *dp, *qref;
930 int len, baselen, ch;
931
932 baselen = strlen(base);
933 len = baselen + 7; /* "/refs/" + NUL */
934 for (cp = ref; (ch = *cp) != 0; cp++, len++)
935 if (needs_quote(ch))
936 len += 2; /* extra two hex plus replacement % */
937 qref = xmalloc(len);
938 memcpy(qref, base, baselen);
939 memcpy(qref + baselen, "/refs/", 6);
940 for (cp = ref, dp = qref + baselen + 6; (ch = *cp) != 0; cp++) {
941 if (needs_quote(ch)) {
942 *dp++ = '%';
943 *dp++ = hex((ch >> 4) & 0xF);
944 *dp++ = hex(ch & 0xF);
945 }
946 else
947 *dp++ = ch;
948 }
949 *dp = 0;
950
951 return qref;
952 }
953
954 int fetch_ref(char *ref, unsigned char *sha1)
955 {
956 char *url;
957 char hex[42];
958 struct buffer buffer;
959 const char *base = alt->base;
960 struct active_request_slot *slot;
961 struct slot_results results;
962 buffer.size = 41;
963 buffer.posn = 0;
964 buffer.buffer = hex;
965 hex[41] = '\0';
966
967 url = quote_ref_url(base, ref);
968 slot = get_active_slot();
969 slot->results = &results;
970 curl_easy_setopt(slot->curl, CURLOPT_FILE, &buffer);
971 curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite_buffer);
972 curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, NULL);
973 curl_easy_setopt(slot->curl, CURLOPT_URL, url);
974 if (start_active_slot(slot)) {
975 run_active_slot(slot);
976 if (results.curl_result != CURLE_OK)
977 return error("Couldn't get %s for %s\n%s",
978 url, ref, curl_errorstr);
979 } else {
980 return error("Unable to start request");
981 }
982
983 hex[40] = '\0';
984 get_sha1_hex(hex, sha1);
985 return 0;
986 }
987
988 int main(int argc, const char **argv)
989 {
990 int commits;
991 const char **write_ref = NULL;
992 char **commit_id;
993 const char *url;
994 char *s;
995 int arg = 1;
996 int rc = 0;
997
998 setup_git_directory();
999 git_config(git_default_config);
1000
1001 while (arg < argc && argv[arg][0] == '-') {
1002 if (argv[arg][1] == 't') {
1003 get_tree = 1;
1004 } else if (argv[arg][1] == 'c') {
1005 get_history = 1;
1006 } else if (argv[arg][1] == 'a') {
1007 get_all = 1;
1008 get_tree = 1;
1009 get_history = 1;
1010 } else if (argv[arg][1] == 'v') {
1011 get_verbosely = 1;
1012 } else if (argv[arg][1] == 'w') {
1013 write_ref = &argv[arg + 1];
1014 arg++;
1015 } else if (!strcmp(argv[arg], "--recover")) {
1016 get_recover = 1;
1017 } else if (!strcmp(argv[arg], "--stdin")) {
1018 commits_on_stdin = 1;
1019 }
1020 arg++;
1021 }
1022 if (argc < arg + 2 - commits_on_stdin) {
1023 usage("git-http-fetch [-c] [-t] [-a] [-v] [--recover] [-w ref] [--stdin] commit-id url");
1024 return 1;
1025 }
1026 if (commits_on_stdin) {
1027 commits = pull_targets_stdin(&commit_id, &write_ref);
1028 } else {
1029 commit_id = (char **) &argv[arg++];
1030 commits = 1;
1031 }
1032 url = argv[arg];
1033
1034 http_init();
1035
1036 no_pragma_header = curl_slist_append(no_pragma_header, "Pragma:");
1037
1038 alt = xmalloc(sizeof(*alt));
1039 alt->base = xmalloc(strlen(url) + 1);
1040 strcpy(alt->base, url);
1041 for (s = alt->base + strlen(alt->base) - 1; *s == '/'; --s)
1042 *s = 0;
1043 alt->got_indices = 0;
1044 alt->packs = NULL;
1045 alt->next = NULL;
1046
1047 if (pull(commits, commit_id, write_ref, url))
1048 rc = 1;
1049
1050 http_cleanup();
1051
1052 curl_slist_free_all(no_pragma_header);
1053
1054 if (commits_on_stdin)
1055 pull_targets_free(commits, commit_id, write_ref);
1056
1057 if (corrupt_object_found) {
1058 fprintf(stderr,
1059 "Some loose object were found to be corrupt, but they might be just\n"
1060 "a false '404 Not Found' error message sent with incorrect HTTP\n"
1061 "status code. Suggest running git-fsck.\n");
1062 }
1063 return rc;
1064 }
The Best VCS Around