git-svn: simplify the (multi-)init methods of fetching
[git/dscho.git] / http-fetch.c
blobe6cd11db737e1ed79c3b60d33674c6e0ce38af55
1 #include "cache.h"
2 #include "commit.h"
3 #include "pack.h"
4 #include "fetch.h"
5 #include "http.h"
7 #define PREV_BUF_SIZE 4096
8 #define RANGE_HEADER_SIZE 30
10 static int commits_on_stdin;
12 static int got_alternates = -1;
13 static int corrupt_object_found;
15 static struct curl_slist *no_pragma_header;
17 struct alt_base
19         const char *base;
20         int path_len;
21         int got_indices;
22         struct packed_git *packs;
23         struct alt_base *next;
26 static struct alt_base *alt;
28 enum object_request_state {
29         WAITING,
30         ABORTED,
31         ACTIVE,
32         COMPLETE,
35 struct object_request
37         unsigned char sha1[20];
38         struct alt_base *repo;
39         char *url;
40         char filename[PATH_MAX];
41         char tmpfile[PATH_MAX];
42         int local;
43         enum object_request_state state;
44         CURLcode curl_result;
45         char errorstr[CURL_ERROR_SIZE];
46         long http_code;
47         unsigned char real_sha1[20];
48         SHA_CTX c;
49         z_stream stream;
50         int zret;
51         int rename;
52         struct active_request_slot *slot;
53         struct object_request *next;
56 struct alternates_request {
57         const char *base;
58         char *url;
59         struct buffer *buffer;
60         struct active_request_slot *slot;
61         int http_specific;
64 static struct object_request *object_queue_head;
66 static size_t fwrite_sha1_file(void *ptr, size_t eltsize, size_t nmemb,
67                                void *data)
69         unsigned char expn[4096];
70         size_t size = eltsize * nmemb;
71         int posn = 0;
72         struct object_request *obj_req = (struct object_request *)data;
73         do {
74                 ssize_t retval = xwrite(obj_req->local,
75                                        (char *) ptr + posn, size - posn);
76                 if (retval < 0)
77                         return posn;
78                 posn += retval;
79         } while (posn < size);
81         obj_req->stream.avail_in = size;
82         obj_req->stream.next_in = ptr;
83         do {
84                 obj_req->stream.next_out = expn;
85                 obj_req->stream.avail_out = sizeof(expn);
86                 obj_req->zret = inflate(&obj_req->stream, Z_SYNC_FLUSH);
87                 SHA1_Update(&obj_req->c, expn,
88                             sizeof(expn) - obj_req->stream.avail_out);
89         } while (obj_req->stream.avail_in && obj_req->zret == Z_OK);
90         data_received++;
91         return size;
94 static int missing__target(int code, int result)
96         return  /* file:// URL -- do we ever use one??? */
97                 (result == CURLE_FILE_COULDNT_READ_FILE) ||
98                 /* http:// and https:// URL */
99                 (code == 404 && result == CURLE_HTTP_RETURNED_ERROR) ||
100                 /* ftp:// URL */
101                 (code == 550 && result == CURLE_FTP_COULDNT_RETR_FILE)
102                 ;
105 #define missing_target(a) missing__target((a)->http_code, (a)->curl_result)
107 static void fetch_alternates(const char *base);
109 static void process_object_response(void *callback_data);
111 static void start_object_request(struct object_request *obj_req)
113         char *hex = sha1_to_hex(obj_req->sha1);
114         char prevfile[PATH_MAX];
115         char *url;
116         char *posn;
117         int prevlocal;
118         unsigned char prev_buf[PREV_BUF_SIZE];
119         ssize_t prev_read = 0;
120         long prev_posn = 0;
121         char range[RANGE_HEADER_SIZE];
122         struct curl_slist *range_header = NULL;
123         struct active_request_slot *slot;
125         snprintf(prevfile, sizeof(prevfile), "%s.prev", obj_req->filename);
126         unlink(prevfile);
127         rename(obj_req->tmpfile, prevfile);
128         unlink(obj_req->tmpfile);
130         if (obj_req->local != -1)
131                 error("fd leakage in start: %d", obj_req->local);
132         obj_req->local = open(obj_req->tmpfile,
133                               O_WRONLY | O_CREAT | O_EXCL, 0666);
134         /* This could have failed due to the "lazy directory creation";
135          * try to mkdir the last path component.
136          */
137         if (obj_req->local < 0 && errno == ENOENT) {
138                 char *dir = strrchr(obj_req->tmpfile, '/');
139                 if (dir) {
140                         *dir = 0;
141                         mkdir(obj_req->tmpfile, 0777);
142                         *dir = '/';
143                 }
144                 obj_req->local = open(obj_req->tmpfile,
145                                       O_WRONLY | O_CREAT | O_EXCL, 0666);
146         }
148         if (obj_req->local < 0) {
149                 obj_req->state = ABORTED;
150                 error("Couldn't create temporary file %s for %s: %s",
151                       obj_req->tmpfile, obj_req->filename, strerror(errno));
152                 return;
153         }
155         memset(&obj_req->stream, 0, sizeof(obj_req->stream));
157         inflateInit(&obj_req->stream);
159         SHA1_Init(&obj_req->c);
161         url = xmalloc(strlen(obj_req->repo->base) + 50);
162         obj_req->url = xmalloc(strlen(obj_req->repo->base) + 50);
163         strcpy(url, obj_req->repo->base);
164         posn = url + strlen(obj_req->repo->base);
165         strcpy(posn, "objects/");
166         posn += 8;
167         memcpy(posn, hex, 2);
168         posn += 2;
169         *(posn++) = '/';
170         strcpy(posn, hex + 2);
171         strcpy(obj_req->url, url);
173         /* If a previous temp file is present, process what was already
174            fetched. */
175         prevlocal = open(prevfile, O_RDONLY);
176         if (prevlocal != -1) {
177                 do {
178                         prev_read = xread(prevlocal, prev_buf, PREV_BUF_SIZE);
179                         if (prev_read>0) {
180                                 if (fwrite_sha1_file(prev_buf,
181                                                      1,
182                                                      prev_read,
183                                                      obj_req) == prev_read) {
184                                         prev_posn += prev_read;
185                                 } else {
186                                         prev_read = -1;
187                                 }
188                         }
189                 } while (prev_read > 0);
190                 close(prevlocal);
191         }
192         unlink(prevfile);
194         /* Reset inflate/SHA1 if there was an error reading the previous temp
195            file; also rewind to the beginning of the local file. */
196         if (prev_read == -1) {
197                 memset(&obj_req->stream, 0, sizeof(obj_req->stream));
198                 inflateInit(&obj_req->stream);
199                 SHA1_Init(&obj_req->c);
200                 if (prev_posn>0) {
201                         prev_posn = 0;
202                         lseek(obj_req->local, SEEK_SET, 0);
203                         ftruncate(obj_req->local, 0);
204                 }
205         }
207         slot = get_active_slot();
208         slot->callback_func = process_object_response;
209         slot->callback_data = obj_req;
210         obj_req->slot = slot;
212         curl_easy_setopt(slot->curl, CURLOPT_FILE, obj_req);
213         curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite_sha1_file);
214         curl_easy_setopt(slot->curl, CURLOPT_ERRORBUFFER, obj_req->errorstr);
215         curl_easy_setopt(slot->curl, CURLOPT_URL, url);
216         curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, no_pragma_header);
218         /* If we have successfully processed data from a previous fetch
219            attempt, only fetch the data we don't already have. */
220         if (prev_posn>0) {
221                 if (get_verbosely)
222                         fprintf(stderr,
223                                 "Resuming fetch of object %s at byte %ld\n",
224                                 hex, prev_posn);
225                 sprintf(range, "Range: bytes=%ld-", prev_posn);
226                 range_header = curl_slist_append(range_header, range);
227                 curl_easy_setopt(slot->curl,
228                                  CURLOPT_HTTPHEADER, range_header);
229         }
231         /* Try to get the request started, abort the request on error */
232         obj_req->state = ACTIVE;
233         if (!start_active_slot(slot)) {
234                 obj_req->state = ABORTED;
235                 obj_req->slot = NULL;
236                 close(obj_req->local); obj_req->local = -1;
237                 free(obj_req->url);
238                 return;
239         }
242 static void finish_object_request(struct object_request *obj_req)
244         struct stat st;
246         fchmod(obj_req->local, 0444);
247         close(obj_req->local); obj_req->local = -1;
249         if (obj_req->http_code == 416) {
250                 fprintf(stderr, "Warning: requested range invalid; we may already have all the data.\n");
251         } else if (obj_req->curl_result != CURLE_OK) {
252                 if (stat(obj_req->tmpfile, &st) == 0)
253                         if (st.st_size == 0)
254                                 unlink(obj_req->tmpfile);
255                 return;
256         }
258         inflateEnd(&obj_req->stream);
259         SHA1_Final(obj_req->real_sha1, &obj_req->c);
260         if (obj_req->zret != Z_STREAM_END) {
261                 unlink(obj_req->tmpfile);
262                 return;
263         }
264         if (hashcmp(obj_req->sha1, obj_req->real_sha1)) {
265                 unlink(obj_req->tmpfile);
266                 return;
267         }
268         obj_req->rename =
269                 move_temp_to_file(obj_req->tmpfile, obj_req->filename);
271         if (obj_req->rename == 0)
272                 pull_say("got %s\n", sha1_to_hex(obj_req->sha1));
275 static void process_object_response(void *callback_data)
277         struct object_request *obj_req =
278                 (struct object_request *)callback_data;
280         obj_req->curl_result = obj_req->slot->curl_result;
281         obj_req->http_code = obj_req->slot->http_code;
282         obj_req->slot = NULL;
283         obj_req->state = COMPLETE;
285         /* Use alternates if necessary */
286         if (missing_target(obj_req)) {
287                 fetch_alternates(alt->base);
288                 if (obj_req->repo->next != NULL) {
289                         obj_req->repo =
290                                 obj_req->repo->next;
291                         close(obj_req->local);
292                         obj_req->local = -1;
293                         start_object_request(obj_req);
294                         return;
295                 }
296         }
298         finish_object_request(obj_req);
301 static void release_object_request(struct object_request *obj_req)
303         struct object_request *entry = object_queue_head;
305         if (obj_req->local != -1)
306                 error("fd leakage in release: %d", obj_req->local);
307         if (obj_req == object_queue_head) {
308                 object_queue_head = obj_req->next;
309         } else {
310                 while (entry->next != NULL && entry->next != obj_req)
311                         entry = entry->next;
312                 if (entry->next == obj_req)
313                         entry->next = entry->next->next;
314         }
316         free(obj_req->url);
317         free(obj_req);
320 #ifdef USE_CURL_MULTI
321 void fill_active_slots(void)
323         struct object_request *obj_req = object_queue_head;
324         struct active_request_slot *slot = active_queue_head;
325         int num_transfers;
327         while (active_requests < max_requests && obj_req != NULL) {
328                 if (obj_req->state == WAITING) {
329                         if (has_sha1_file(obj_req->sha1))
330                                 obj_req->state = COMPLETE;
331                         else
332                                 start_object_request(obj_req);
333                         curl_multi_perform(curlm, &num_transfers);
334                 }
335                 obj_req = obj_req->next;
336         }
338         while (slot != NULL) {
339                 if (!slot->in_use && slot->curl != NULL) {
340                         curl_easy_cleanup(slot->curl);
341                         slot->curl = NULL;
342                 }
343                 slot = slot->next;
344         }
346 #endif
348 void prefetch(unsigned char *sha1)
350         struct object_request *newreq;
351         struct object_request *tail;
352         char *filename = sha1_file_name(sha1);
354         newreq = xmalloc(sizeof(*newreq));
355         hashcpy(newreq->sha1, sha1);
356         newreq->repo = alt;
357         newreq->url = NULL;
358         newreq->local = -1;
359         newreq->state = WAITING;
360         snprintf(newreq->filename, sizeof(newreq->filename), "%s", filename);
361         snprintf(newreq->tmpfile, sizeof(newreq->tmpfile),
362                  "%s.temp", filename);
363         newreq->slot = NULL;
364         newreq->next = NULL;
366         if (object_queue_head == NULL) {
367                 object_queue_head = newreq;
368         } else {
369                 tail = object_queue_head;
370                 while (tail->next != NULL) {
371                         tail = tail->next;
372                 }
373                 tail->next = newreq;
374         }
376 #ifdef USE_CURL_MULTI
377         fill_active_slots();
378         step_active_slots();
379 #endif
382 static int fetch_index(struct alt_base *repo, unsigned char *sha1)
384         char *hex = sha1_to_hex(sha1);
385         char *filename;
386         char *url;
387         char tmpfile[PATH_MAX];
388         long prev_posn = 0;
389         char range[RANGE_HEADER_SIZE];
390         struct curl_slist *range_header = NULL;
392         FILE *indexfile;
393         struct active_request_slot *slot;
394         struct slot_results results;
396         if (has_pack_index(sha1))
397                 return 0;
399         if (get_verbosely)
400                 fprintf(stderr, "Getting index for pack %s\n", hex);
402         url = xmalloc(strlen(repo->base) + 64);
403         sprintf(url, "%s/objects/pack/pack-%s.idx", repo->base, hex);
405         filename = sha1_pack_index_name(sha1);
406         snprintf(tmpfile, sizeof(tmpfile), "%s.temp", filename);
407         indexfile = fopen(tmpfile, "a");
408         if (!indexfile)
409                 return error("Unable to open local file %s for pack index",
410                              filename);
412         slot = get_active_slot();
413         slot->results = &results;
414         curl_easy_setopt(slot->curl, CURLOPT_FILE, indexfile);
415         curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite);
416         curl_easy_setopt(slot->curl, CURLOPT_URL, url);
417         curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, no_pragma_header);
418         slot->local = indexfile;
420         /* If there is data present from a previous transfer attempt,
421            resume where it left off */
422         prev_posn = ftell(indexfile);
423         if (prev_posn>0) {
424                 if (get_verbosely)
425                         fprintf(stderr,
426                                 "Resuming fetch of index for pack %s at byte %ld\n",
427                                 hex, prev_posn);
428                 sprintf(range, "Range: bytes=%ld-", prev_posn);
429                 range_header = curl_slist_append(range_header, range);
430                 curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, range_header);
431         }
433         if (start_active_slot(slot)) {
434                 run_active_slot(slot);
435                 if (results.curl_result != CURLE_OK) {
436                         fclose(indexfile);
437                         return error("Unable to get pack index %s\n%s", url,
438                                      curl_errorstr);
439                 }
440         } else {
441                 fclose(indexfile);
442                 return error("Unable to start request");
443         }
445         fclose(indexfile);
447         return move_temp_to_file(tmpfile, filename);
450 static int setup_index(struct alt_base *repo, unsigned char *sha1)
452         struct packed_git *new_pack;
453         if (has_pack_file(sha1))
454                 return 0; /* don't list this as something we can get */
456         if (fetch_index(repo, sha1))
457                 return -1;
459         new_pack = parse_pack_index(sha1);
460         new_pack->next = repo->packs;
461         repo->packs = new_pack;
462         return 0;
465 static void process_alternates_response(void *callback_data)
467         struct alternates_request *alt_req =
468                 (struct alternates_request *)callback_data;
469         struct active_request_slot *slot = alt_req->slot;
470         struct alt_base *tail = alt;
471         const char *base = alt_req->base;
472         static const char null_byte = '\0';
473         char *data;
474         int i = 0;
476         if (alt_req->http_specific) {
477                 if (slot->curl_result != CURLE_OK ||
478                     !alt_req->buffer->posn) {
480                         /* Try reusing the slot to get non-http alternates */
481                         alt_req->http_specific = 0;
482                         sprintf(alt_req->url, "%s/objects/info/alternates",
483                                 base);
484                         curl_easy_setopt(slot->curl, CURLOPT_URL,
485                                          alt_req->url);
486                         active_requests++;
487                         slot->in_use = 1;
488                         if (slot->finished != NULL)
489                                 (*slot->finished) = 0;
490                         if (!start_active_slot(slot)) {
491                                 got_alternates = -1;
492                                 slot->in_use = 0;
493                                 if (slot->finished != NULL)
494                                         (*slot->finished) = 1;
495                         }
496                         return;
497                 }
498         } else if (slot->curl_result != CURLE_OK) {
499                 if (!missing_target(slot)) {
500                         got_alternates = -1;
501                         return;
502                 }
503         }
505         fwrite_buffer(&null_byte, 1, 1, alt_req->buffer);
506         alt_req->buffer->posn--;
507         data = alt_req->buffer->buffer;
509         while (i < alt_req->buffer->posn) {
510                 int posn = i;
511                 while (posn < alt_req->buffer->posn && data[posn] != '\n')
512                         posn++;
513                 if (data[posn] == '\n') {
514                         int okay = 0;
515                         int serverlen = 0;
516                         struct alt_base *newalt;
517                         char *target = NULL;
518                         char *path;
519                         if (data[i] == '/') {
520                                 /* This counts
521                                  * http://git.host/pub/scm/linux.git/
522                                  * -----------here^
523                                  * so memcpy(dst, base, serverlen) will
524                                  * copy up to "...git.host".
525                                  */
526                                 const char *colon_ss = strstr(base,"://");
527                                 if (colon_ss) {
528                                         serverlen = (strchr(colon_ss + 3, '/')
529                                                      - base);
530                                         okay = 1;
531                                 }
532                         } else if (!memcmp(data + i, "../", 3)) {
533                                 /* Relative URL; chop the corresponding
534                                  * number of subpath from base (and ../
535                                  * from data), and concatenate the result.
536                                  *
537                                  * The code first drops ../ from data, and
538                                  * then drops one ../ from data and one path
539                                  * from base.  IOW, one extra ../ is dropped
540                                  * from data than path is dropped from base.
541                                  *
542                                  * This is not wrong.  The alternate in
543                                  *     http://git.host/pub/scm/linux.git/
544                                  * to borrow from
545                                  *     http://git.host/pub/scm/linus.git/
546                                  * is ../../linus.git/objects/.  You need
547                                  * two ../../ to borrow from your direct
548                                  * neighbour.
549                                  */
550                                 i += 3;
551                                 serverlen = strlen(base);
552                                 while (i + 2 < posn &&
553                                        !memcmp(data + i, "../", 3)) {
554                                         do {
555                                                 serverlen--;
556                                         } while (serverlen &&
557                                                  base[serverlen - 1] != '/');
558                                         i += 3;
559                                 }
560                                 /* If the server got removed, give up. */
561                                 okay = strchr(base, ':') - base + 3 <
562                                         serverlen;
563                         } else if (alt_req->http_specific) {
564                                 char *colon = strchr(data + i, ':');
565                                 char *slash = strchr(data + i, '/');
566                                 if (colon && slash && colon < data + posn &&
567                                     slash < data + posn && colon < slash) {
568                                         okay = 1;
569                                 }
570                         }
571                         /* skip "objects\n" at end */
572                         if (okay) {
573                                 target = xmalloc(serverlen + posn - i - 6);
574                                 memcpy(target, base, serverlen);
575                                 memcpy(target + serverlen, data + i,
576                                        posn - i - 7);
577                                 target[serverlen + posn - i - 7] = 0;
578                                 if (get_verbosely)
579                                         fprintf(stderr,
580                                                 "Also look at %s\n", target);
581                                 newalt = xmalloc(sizeof(*newalt));
582                                 newalt->next = NULL;
583                                 newalt->base = target;
584                                 newalt->got_indices = 0;
585                                 newalt->packs = NULL;
586                                 path = strstr(target, "//");
587                                 if (path) {
588                                         path = strchr(path+2, '/');
589                                         if (path)
590                                                 newalt->path_len = strlen(path);
591                                 }
593                                 while (tail->next != NULL)
594                                         tail = tail->next;
595                                 tail->next = newalt;
596                         }
597                 }
598                 i = posn + 1;
599         }
601         got_alternates = 1;
604 static void fetch_alternates(const char *base)
606         struct buffer buffer;
607         char *url;
608         char *data;
609         struct active_request_slot *slot;
610         struct alternates_request alt_req;
612         /* If another request has already started fetching alternates,
613            wait for them to arrive and return to processing this request's
614            curl message */
615 #ifdef USE_CURL_MULTI
616         while (got_alternates == 0) {
617                 step_active_slots();
618         }
619 #endif
621         /* Nothing to do if they've already been fetched */
622         if (got_alternates == 1)
623                 return;
625         /* Start the fetch */
626         got_alternates = 0;
628         data = xmalloc(4096);
629         buffer.size = 4096;
630         buffer.posn = 0;
631         buffer.buffer = data;
633         if (get_verbosely)
634                 fprintf(stderr, "Getting alternates list for %s\n", base);
636         url = xmalloc(strlen(base) + 31);
637         sprintf(url, "%s/objects/info/http-alternates", base);
639         /* Use a callback to process the result, since another request
640            may fail and need to have alternates loaded before continuing */
641         slot = get_active_slot();
642         slot->callback_func = process_alternates_response;
643         slot->callback_data = &alt_req;
645         curl_easy_setopt(slot->curl, CURLOPT_FILE, &buffer);
646         curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite_buffer);
647         curl_easy_setopt(slot->curl, CURLOPT_URL, url);
649         alt_req.base = base;
650         alt_req.url = url;
651         alt_req.buffer = &buffer;
652         alt_req.http_specific = 1;
653         alt_req.slot = slot;
655         if (start_active_slot(slot))
656                 run_active_slot(slot);
657         else
658                 got_alternates = -1;
660         free(data);
661         free(url);
664 static int fetch_indices(struct alt_base *repo)
666         unsigned char sha1[20];
667         char *url;
668         struct buffer buffer;
669         char *data;
670         int i = 0;
672         struct active_request_slot *slot;
673         struct slot_results results;
675         if (repo->got_indices)
676                 return 0;
678         data = xmalloc(4096);
679         buffer.size = 4096;
680         buffer.posn = 0;
681         buffer.buffer = data;
683         if (get_verbosely)
684                 fprintf(stderr, "Getting pack list for %s\n", repo->base);
686         url = xmalloc(strlen(repo->base) + 21);
687         sprintf(url, "%s/objects/info/packs", repo->base);
689         slot = get_active_slot();
690         slot->results = &results;
691         curl_easy_setopt(slot->curl, CURLOPT_FILE, &buffer);
692         curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite_buffer);
693         curl_easy_setopt(slot->curl, CURLOPT_URL, url);
694         curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, NULL);
695         if (start_active_slot(slot)) {
696                 run_active_slot(slot);
697                 if (results.curl_result != CURLE_OK) {
698                         if (missing_target(&results)) {
699                                 repo->got_indices = 1;
700                                 free(buffer.buffer);
701                                 return 0;
702                         } else {
703                                 repo->got_indices = 0;
704                                 free(buffer.buffer);
705                                 return error("%s", curl_errorstr);
706                         }
707                 }
708         } else {
709                 repo->got_indices = 0;
710                 free(buffer.buffer);
711                 return error("Unable to start request");
712         }
714         data = buffer.buffer;
715         while (i < buffer.posn) {
716                 switch (data[i]) {
717                 case 'P':
718                         i++;
719                         if (i + 52 <= buffer.posn &&
720                             !prefixcmp(data + i, " pack-") &&
721                             !prefixcmp(data + i + 46, ".pack\n")) {
722                                 get_sha1_hex(data + i + 6, sha1);
723                                 setup_index(repo, sha1);
724                                 i += 51;
725                                 break;
726                         }
727                 default:
728                         while (i < buffer.posn && data[i] != '\n')
729                                 i++;
730                 }
731                 i++;
732         }
734         free(buffer.buffer);
735         repo->got_indices = 1;
736         return 0;
739 static int fetch_pack(struct alt_base *repo, unsigned char *sha1)
741         char *url;
742         struct packed_git *target;
743         struct packed_git **lst;
744         FILE *packfile;
745         char *filename;
746         char tmpfile[PATH_MAX];
747         int ret;
748         long prev_posn = 0;
749         char range[RANGE_HEADER_SIZE];
750         struct curl_slist *range_header = NULL;
752         struct active_request_slot *slot;
753         struct slot_results results;
755         if (fetch_indices(repo))
756                 return -1;
757         target = find_sha1_pack(sha1, repo->packs);
758         if (!target)
759                 return -1;
761         if (get_verbosely) {
762                 fprintf(stderr, "Getting pack %s\n",
763                         sha1_to_hex(target->sha1));
764                 fprintf(stderr, " which contains %s\n",
765                         sha1_to_hex(sha1));
766         }
768         url = xmalloc(strlen(repo->base) + 65);
769         sprintf(url, "%s/objects/pack/pack-%s.pack",
770                 repo->base, sha1_to_hex(target->sha1));
772         filename = sha1_pack_name(target->sha1);
773         snprintf(tmpfile, sizeof(tmpfile), "%s.temp", filename);
774         packfile = fopen(tmpfile, "a");
775         if (!packfile)
776                 return error("Unable to open local file %s for pack",
777                              filename);
779         slot = get_active_slot();
780         slot->results = &results;
781         curl_easy_setopt(slot->curl, CURLOPT_FILE, packfile);
782         curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite);
783         curl_easy_setopt(slot->curl, CURLOPT_URL, url);
784         curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, no_pragma_header);
785         slot->local = packfile;
787         /* If there is data present from a previous transfer attempt,
788            resume where it left off */
789         prev_posn = ftell(packfile);
790         if (prev_posn>0) {
791                 if (get_verbosely)
792                         fprintf(stderr,
793                                 "Resuming fetch of pack %s at byte %ld\n",
794                                 sha1_to_hex(target->sha1), prev_posn);
795                 sprintf(range, "Range: bytes=%ld-", prev_posn);
796                 range_header = curl_slist_append(range_header, range);
797                 curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, range_header);
798         }
800         if (start_active_slot(slot)) {
801                 run_active_slot(slot);
802                 if (results.curl_result != CURLE_OK) {
803                         fclose(packfile);
804                         return error("Unable to get pack file %s\n%s", url,
805                                      curl_errorstr);
806                 }
807         } else {
808                 fclose(packfile);
809                 return error("Unable to start request");
810         }
812         target->pack_size = ftell(packfile);
813         fclose(packfile);
815         ret = move_temp_to_file(tmpfile, filename);
816         if (ret)
817                 return ret;
819         lst = &repo->packs;
820         while (*lst != target)
821                 lst = &((*lst)->next);
822         *lst = (*lst)->next;
824         if (verify_pack(target, 0))
825                 return -1;
826         install_packed_git(target);
828         return 0;
831 static void abort_object_request(struct object_request *obj_req)
833         if (obj_req->local >= 0) {
834                 close(obj_req->local);
835                 obj_req->local = -1;
836         }
837         unlink(obj_req->tmpfile);
838         if (obj_req->slot) {
839                 release_active_slot(obj_req->slot);
840                 obj_req->slot = NULL;
841         }
842         release_object_request(obj_req);
845 static int fetch_object(struct alt_base *repo, unsigned char *sha1)
847         char *hex = sha1_to_hex(sha1);
848         int ret = 0;
849         struct object_request *obj_req = object_queue_head;
851         while (obj_req != NULL && hashcmp(obj_req->sha1, sha1))
852                 obj_req = obj_req->next;
853         if (obj_req == NULL)
854                 return error("Couldn't find request for %s in the queue", hex);
856         if (has_sha1_file(obj_req->sha1)) {
857                 abort_object_request(obj_req);
858                 return 0;
859         }
861 #ifdef USE_CURL_MULTI
862         while (obj_req->state == WAITING) {
863                 step_active_slots();
864         }
865 #else
866         start_object_request(obj_req);
867 #endif
869         while (obj_req->state == ACTIVE) {
870                 run_active_slot(obj_req->slot);
871         }
872         if (obj_req->local != -1) {
873                 close(obj_req->local); obj_req->local = -1;
874         }
876         if (obj_req->state == ABORTED) {
877                 ret = error("Request for %s aborted", hex);
878         } else if (obj_req->curl_result != CURLE_OK &&
879                    obj_req->http_code != 416) {
880                 if (missing_target(obj_req))
881                         ret = -1; /* Be silent, it is probably in a pack. */
882                 else
883                         ret = error("%s (curl_result = %d, http_code = %ld, sha1 = %s)",
884                                     obj_req->errorstr, obj_req->curl_result,
885                                     obj_req->http_code, hex);
886         } else if (obj_req->zret != Z_STREAM_END) {
887                 corrupt_object_found++;
888                 ret = error("File %s (%s) corrupt", hex, obj_req->url);
889         } else if (hashcmp(obj_req->sha1, obj_req->real_sha1)) {
890                 ret = error("File %s has bad hash", hex);
891         } else if (obj_req->rename < 0) {
892                 ret = error("unable to write sha1 filename %s",
893                             obj_req->filename);
894         }
896         release_object_request(obj_req);
897         return ret;
900 int fetch(unsigned char *sha1)
902         struct alt_base *altbase = alt;
904         if (!fetch_object(altbase, sha1))
905                 return 0;
906         while (altbase) {
907                 if (!fetch_pack(altbase, sha1))
908                         return 0;
909                 fetch_alternates(alt->base);
910                 altbase = altbase->next;
911         }
912         return error("Unable to find %s under %s", sha1_to_hex(sha1),
913                      alt->base);
916 static inline int needs_quote(int ch)
918         if (((ch >= 'A') && (ch <= 'Z'))
919                         || ((ch >= 'a') && (ch <= 'z'))
920                         || ((ch >= '0') && (ch <= '9'))
921                         || (ch == '/')
922                         || (ch == '-')
923                         || (ch == '.'))
924                 return 0;
925         return 1;
928 static inline int hex(int v)
930         if (v < 10) return '0' + v;
931         else return 'A' + v - 10;
934 static char *quote_ref_url(const char *base, const char *ref)
936         const char *cp;
937         char *dp, *qref;
938         int len, baselen, ch;
940         baselen = strlen(base);
941         len = baselen + 6; /* "refs/" + NUL */
942         for (cp = ref; (ch = *cp) != 0; cp++, len++)
943                 if (needs_quote(ch))
944                         len += 2; /* extra two hex plus replacement % */
945         qref = xmalloc(len);
946         memcpy(qref, base, baselen);
947         memcpy(qref + baselen, "refs/", 5);
948         for (cp = ref, dp = qref + baselen + 5; (ch = *cp) != 0; cp++) {
949                 if (needs_quote(ch)) {
950                         *dp++ = '%';
951                         *dp++ = hex((ch >> 4) & 0xF);
952                         *dp++ = hex(ch & 0xF);
953                 }
954                 else
955                         *dp++ = ch;
956         }
957         *dp = 0;
959         return qref;
962 int fetch_ref(char *ref, unsigned char *sha1)
964         char *url;
965         char hex[42];
966         struct buffer buffer;
967         const char *base = alt->base;
968         struct active_request_slot *slot;
969         struct slot_results results;
970         buffer.size = 41;
971         buffer.posn = 0;
972         buffer.buffer = hex;
973         hex[41] = '\0';
975         url = quote_ref_url(base, ref);
976         slot = get_active_slot();
977         slot->results = &results;
978         curl_easy_setopt(slot->curl, CURLOPT_FILE, &buffer);
979         curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite_buffer);
980         curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, NULL);
981         curl_easy_setopt(slot->curl, CURLOPT_URL, url);
982         if (start_active_slot(slot)) {
983                 run_active_slot(slot);
984                 if (results.curl_result != CURLE_OK)
985                         return error("Couldn't get %s for %s\n%s",
986                                      url, ref, curl_errorstr);
987         } else {
988                 return error("Unable to start request");
989         }
991         hex[40] = '\0';
992         get_sha1_hex(hex, sha1);
993         return 0;
996 int main(int argc, const char **argv)
998         int commits;
999         const char **write_ref = NULL;
1000         char **commit_id;
1001         const char *url;
1002         char *path;
1003         int arg = 1;
1004         int rc = 0;
1006         setup_git_directory();
1007         git_config(git_default_config);
1009         while (arg < argc && argv[arg][0] == '-') {
1010                 if (argv[arg][1] == 't') {
1011                         get_tree = 1;
1012                 } else if (argv[arg][1] == 'c') {
1013                         get_history = 1;
1014                 } else if (argv[arg][1] == 'a') {
1015                         get_all = 1;
1016                         get_tree = 1;
1017                         get_history = 1;
1018                 } else if (argv[arg][1] == 'v') {
1019                         get_verbosely = 1;
1020                 } else if (argv[arg][1] == 'w') {
1021                         write_ref = &argv[arg + 1];
1022                         arg++;
1023                 } else if (!strcmp(argv[arg], "--recover")) {
1024                         get_recover = 1;
1025                 } else if (!strcmp(argv[arg], "--stdin")) {
1026                         commits_on_stdin = 1;
1027                 }
1028                 arg++;
1029         }
1030         if (argc < arg + 2 - commits_on_stdin) {
1031                 usage("git-http-fetch [-c] [-t] [-a] [-v] [--recover] [-w ref] [--stdin] commit-id url");
1032                 return 1;
1033         }
1034         if (commits_on_stdin) {
1035                 commits = pull_targets_stdin(&commit_id, &write_ref);
1036         } else {
1037                 commit_id = (char **) &argv[arg++];
1038                 commits = 1;
1039         }
1040         url = argv[arg];
1042         http_init();
1044         no_pragma_header = curl_slist_append(no_pragma_header, "Pragma:");
1046         alt = xmalloc(sizeof(*alt));
1047         alt->base = url;
1048         alt->got_indices = 0;
1049         alt->packs = NULL;
1050         alt->next = NULL;
1051         path = strstr(url, "//");
1052         if (path) {
1053                 path = strchr(path+2, '/');
1054                 if (path)
1055                         alt->path_len = strlen(path);
1056         }
1058         if (pull(commits, commit_id, write_ref, url))
1059                 rc = 1;
1061         http_cleanup();
1063         curl_slist_free_all(no_pragma_header);
1065         if (commits_on_stdin)
1066                 pull_targets_free(commits, commit_id, write_ref);
1068         if (corrupt_object_found) {
1069                 fprintf(stderr,
1070 "Some loose object were found to be corrupt, but they might be just\n"
1071 "a false '404 Not Found' error message sent with incorrect HTTP\n"
1072 "status code.  Suggest running git-fsck.\n");
1073         }
1074         return rc;