http-fetch.c

   1 #include "cache.h"
   2 #include "commit.h"
   3 #include "pack.h"
   4 #include "fetch.h"
   5 #include "http.h"
   6
   7 #define PREV_BUF_SIZE 4096
   8 #define RANGE_HEADER_SIZE 30
   9
  10 static int commits_on_stdin;
  11
  12 static int got_alternates = -1;
  13 static int corrupt_object_found;
  14
  15 static struct curl_slist *no_pragma_header;
  16
  17 struct alt_base
  18 {
  19         char *base;
  20         int got_indices;
  21         struct packed_git *packs;
  22         struct alt_base *next;
  23 };
  24
  25 static struct alt_base *alt;
  26
  27 enum object_request_state {
  28         WAITING,
  29         ABORTED,
  30         ACTIVE,
  31         COMPLETE,
  32 };
  33
  34 struct object_request
  35 {
  36         unsigned char sha1[20];
  37         struct alt_base *repo;
  38         char *url;
  39         char filename[PATH_MAX];
  40         char tmpfile[PATH_MAX];
  41         int local;
  42         enum object_request_state state;
  43         CURLcode curl_result;
  44         char errorstr[CURL_ERROR_SIZE];
  45         long http_code;
  46         unsigned char real_sha1[20];
  47         SHA_CTX c;
  48         z_stream stream;
  49         int zret;
  50         int rename;
  51         struct active_request_slot *slot;
  52         struct object_request *next;
  53 };
  54
  55 struct alternates_request {
  56         const char *base;
  57         char *url;
  58         struct buffer *buffer;
  59         struct active_request_slot *slot;
  60         int http_specific;
  61 };
  62
  63 static struct object_request *object_queue_head;
  64
  65 static size_t fwrite_sha1_file(void *ptr, size_t eltsize, size_t nmemb,
  66                                void *data)
  67 {
  68         unsigned char expn[4096];
  69         size_t size = eltsize * nmemb;
  70         int posn = 0;
  71         struct object_request *obj_req = (struct object_request *)data;
  72         do {
  73                 ssize_t retval = xwrite(obj_req->local,
  74                                        (char *) ptr + posn, size - posn);
  75                 if (retval < 0)
  76                         return posn;
  77                 posn += retval;
  78         } while (posn < size);
  79
  80         obj_req->stream.avail_in = size;
  81         obj_req->stream.next_in = ptr;
  82         do {
  83                 obj_req->stream.next_out = expn;
  84                 obj_req->stream.avail_out = sizeof(expn);
  85                 obj_req->zret = inflate(&obj_req->stream, Z_SYNC_FLUSH);
  86                 SHA1_Update(&obj_req->c, expn,
  87                             sizeof(expn) - obj_req->stream.avail_out);
  88         } while (obj_req->stream.avail_in && obj_req->zret == Z_OK);
  89         data_received++;
  90         return size;
  91 }
  92
  93 static int missing__target(int code, int result)
  94 {
  95         return  /* file:// URL -- do we ever use one??? */
  96                 (result == CURLE_FILE_COULDNT_READ_FILE) ||
  97                 /* http:// and https:// URL */
  98                 (code == 404 && result == CURLE_HTTP_RETURNED_ERROR) ||
  99                 /* ftp:// URL */
 100                 (code == 550 && result == CURLE_FTP_COULDNT_RETR_FILE)
 101                 ;
 102 }
 103
 104 #define missing_target(a) missing__target((a)->http_code, (a)->curl_result)
 105
 106 static void fetch_alternates(const char *base);
 107
 108 static void process_object_response(void *callback_data);
 109
 110 static void start_object_request(struct object_request *obj_req)
 111 {
 112         char *hex = sha1_to_hex(obj_req->sha1);
 113         char prevfile[PATH_MAX];
 114         char *url;
 115         char *posn;
 116         int prevlocal;
 117         unsigned char prev_buf[PREV_BUF_SIZE];
 118         ssize_t prev_read = 0;
 119         long prev_posn = 0;
 120         char range[RANGE_HEADER_SIZE];
 121         struct curl_slist *range_header = NULL;
 122         struct active_request_slot *slot;
 123
 124         snprintf(prevfile, sizeof(prevfile), "%s.prev", obj_req->filename);
 125         unlink(prevfile);
 126         rename(obj_req->tmpfile, prevfile);
 127         unlink(obj_req->tmpfile);
 128
 129         if (obj_req->local != -1)
 130                 error("fd leakage in start: %d", obj_req->local);
 131         obj_req->local = open(obj_req->tmpfile,
 132                               O_WRONLY | O_CREAT | O_EXCL, 0666);
 133         /* This could have failed due to the "lazy directory creation";
 134          * try to mkdir the last path component.
 135          */
 136         if (obj_req->local < 0 && errno == ENOENT) {
 137                 char *dir = strrchr(obj_req->tmpfile, '/');
 138                 if (dir) {
 139                         *dir = 0;
 140                         mkdir(obj_req->tmpfile, 0777);
 141                         *dir = '/';
 142                 }
 143                 obj_req->local = open(obj_req->tmpfile,
 144                                       O_WRONLY | O_CREAT | O_EXCL, 0666);
 145         }
 146
 147         if (obj_req->local < 0) {
 148                 obj_req->state = ABORTED;
 149                 error("Couldn't create temporary file %s for %s: %s",
 150                       obj_req->tmpfile, obj_req->filename, strerror(errno));
 151                 return;
 152         }
 153
 154         memset(&obj_req->stream, 0, sizeof(obj_req->stream));
 155
 156         inflateInit(&obj_req->stream);
 157
 158         SHA1_Init(&obj_req->c);
 159
 160         url = xmalloc(strlen(obj_req->repo->base) + 51);
 161         obj_req->url = xmalloc(strlen(obj_req->repo->base) + 51);
 162         strcpy(url, obj_req->repo->base);
 163         posn = url + strlen(obj_req->repo->base);
 164         strcpy(posn, "/objects/");
 165         posn += 9;
 166         memcpy(posn, hex, 2);
 167         posn += 2;
 168         *(posn++) = '/';
 169         strcpy(posn, hex + 2);
 170         strcpy(obj_req->url, url);
 171
 172         /* If a previous temp file is present, process what was already
 173            fetched. */
 174         prevlocal = open(prevfile, O_RDONLY);
 175         if (prevlocal != -1) {
 176                 do {
 177                         prev_read = xread(prevlocal, prev_buf, PREV_BUF_SIZE);
 178                         if (prev_read>0) {
 179                                 if (fwrite_sha1_file(prev_buf,
 180                                                      1,
 181                                                      prev_read,
 182                                                      obj_req) == prev_read) {
 183                                         prev_posn += prev_read;
 184                                 } else {
 185                                         prev_read = -1;
 186                                 }
 187                         }
 188                 } while (prev_read > 0);
 189                 close(prevlocal);
 190         }
 191         unlink(prevfile);
 192
 193         /* Reset inflate/SHA1 if there was an error reading the previous temp
 194            file; also rewind to the beginning of the local file. */
 195         if (prev_read == -1) {
 196                 memset(&obj_req->stream, 0, sizeof(obj_req->stream));
 197                 inflateInit(&obj_req->stream);
 198                 SHA1_Init(&obj_req->c);
 199                 if (prev_posn>0) {
 200                         prev_posn = 0;
 201                         lseek(obj_req->local, 0, SEEK_SET);
 202                         ftruncate(obj_req->local, 0);
 203                 }
 204         }
 205
 206         slot = get_active_slot();
 207         slot->callback_func = process_object_response;
 208         slot->callback_data = obj_req;
 209         obj_req->slot = slot;
 210
 211         curl_easy_setopt(slot->curl, CURLOPT_FILE, obj_req);
 212         curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite_sha1_file);
 213         curl_easy_setopt(slot->curl, CURLOPT_ERRORBUFFER, obj_req->errorstr);
 214         curl_easy_setopt(slot->curl, CURLOPT_URL, url);
 215         curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, no_pragma_header);
 216
 217         /* If we have successfully processed data from a previous fetch
 218            attempt, only fetch the data we don't already have. */
 219         if (prev_posn>0) {
 220                 if (get_verbosely)
 221                         fprintf(stderr,
 222                                 "Resuming fetch of object %s at byte %ld\n",
 223                                 hex, prev_posn);
 224                 sprintf(range, "Range: bytes=%ld-", prev_posn);
 225                 range_header = curl_slist_append(range_header, range);
 226                 curl_easy_setopt(slot->curl,
 227                                  CURLOPT_HTTPHEADER, range_header);
 228         }
 229
 230         /* Try to get the request started, abort the request on error */
 231         obj_req->state = ACTIVE;
 232         if (!start_active_slot(slot)) {
 233                 obj_req->state = ABORTED;
 234                 obj_req->slot = NULL;
 235                 close(obj_req->local); obj_req->local = -1;
 236                 free(obj_req->url);
 237                 return;
 238         }
 239 }
 240
 241 static void finish_object_request(struct object_request *obj_req)
 242 {
 243         struct stat st;
 244
 245         fchmod(obj_req->local, 0444);
 246         close(obj_req->local); obj_req->local = -1;
 247
 248         if (obj_req->http_code == 416) {
 249                 fprintf(stderr, "Warning: requested range invalid; we may already have all the data.\n");
 250         } else if (obj_req->curl_result != CURLE_OK) {
 251                 if (stat(obj_req->tmpfile, &st) == 0)
 252                         if (st.st_size == 0)
 253                                 unlink(obj_req->tmpfile);
 254                 return;
 255         }
 256
 257         inflateEnd(&obj_req->stream);
 258         SHA1_Final(obj_req->real_sha1, &obj_req->c);
 259         if (obj_req->zret != Z_STREAM_END) {
 260                 unlink(obj_req->tmpfile);
 261                 return;
 262         }
 263         if (hashcmp(obj_req->sha1, obj_req->real_sha1)) {
 264                 unlink(obj_req->tmpfile);
 265                 return;
 266         }
 267         obj_req->rename =
 268                 move_temp_to_file(obj_req->tmpfile, obj_req->filename);
 269
 270         if (obj_req->rename == 0)
 271                 pull_say("got %s\n", sha1_to_hex(obj_req->sha1));
 272 }
 273
 274 static void process_object_response(void *callback_data)
 275 {
 276         struct object_request *obj_req =
 277                 (struct object_request *)callback_data;
 278
 279         obj_req->curl_result = obj_req->slot->curl_result;
 280         obj_req->http_code = obj_req->slot->http_code;
 281         obj_req->slot = NULL;
 282         obj_req->state = COMPLETE;
 283
 284         /* Use alternates if necessary */
 285         if (missing_target(obj_req)) {
 286                 fetch_alternates(alt->base);
 287                 if (obj_req->repo->next != NULL) {
 288                         obj_req->repo =
 289                                 obj_req->repo->next;
 290                         close(obj_req->local);
 291                         obj_req->local = -1;
 292                         start_object_request(obj_req);
 293                         return;
 294                 }
 295         }
 296
 297         finish_object_request(obj_req);
 298 }
 299
 300 static void release_object_request(struct object_request *obj_req)
 301 {
 302         struct object_request *entry = object_queue_head;
 303
 304         if (obj_req->local != -1)
 305                 error("fd leakage in release: %d", obj_req->local);
 306         if (obj_req == object_queue_head) {
 307                 object_queue_head = obj_req->next;
 308         } else {
 309                 while (entry->next != NULL && entry->next != obj_req)
 310                         entry = entry->next;
 311                 if (entry->next == obj_req)
 312                         entry->next = entry->next->next;
 313         }
 314
 315         free(obj_req->url);
 316         free(obj_req);
 317 }
 318
 319 #ifdef USE_CURL_MULTI
 320 void fill_active_slots(void)
 321 {
 322         struct object_request *obj_req = object_queue_head;
 323         struct active_request_slot *slot = active_queue_head;
 324         int num_transfers;
 325
 326         while (active_requests < max_requests && obj_req != NULL) {
 327                 if (obj_req->state == WAITING) {
 328                         if (has_sha1_file(obj_req->sha1))
 329                                 obj_req->state = COMPLETE;
 330                         else
 331                                 start_object_request(obj_req);
 332                         curl_multi_perform(curlm, &num_transfers);
 333                 }
 334                 obj_req = obj_req->next;
 335         }
 336
 337         while (slot != NULL) {
 338                 if (!slot->in_use && slot->curl != NULL) {
 339                         curl_easy_cleanup(slot->curl);
 340                         slot->curl = NULL;
 341                 }
 342                 slot = slot->next;
 343         }
 344 }
 345 #endif
 346
 347 void prefetch(unsigned char *sha1)
 348 {
 349         struct object_request *newreq;
 350         struct object_request *tail;
 351         char *filename = sha1_file_name(sha1);
 352
 353         newreq = xmalloc(sizeof(*newreq));
 354         hashcpy(newreq->sha1, sha1);
 355         newreq->repo = alt;
 356         newreq->url = NULL;
 357         newreq->local = -1;
 358         newreq->state = WAITING;
 359         snprintf(newreq->filename, sizeof(newreq->filename), "%s", filename);
 360         snprintf(newreq->tmpfile, sizeof(newreq->tmpfile),
 361                  "%s.temp", filename);
 362         newreq->slot = NULL;
 363         newreq->next = NULL;
 364
 365         if (object_queue_head == NULL) {
 366                 object_queue_head = newreq;
 367         } else {
 368                 tail = object_queue_head;
 369                 while (tail->next != NULL) {
 370                         tail = tail->next;
 371                 }
 372                 tail->next = newreq;
 373         }
 374
 375 #ifdef USE_CURL_MULTI
 376         fill_active_slots();
 377         step_active_slots();
 378 #endif
 379 }
 380
 381 static int fetch_index(struct alt_base *repo, unsigned char *sha1)
 382 {
 383         char *hex = sha1_to_hex(sha1);
 384         char *filename;
 385         char *url;
 386         char tmpfile[PATH_MAX];
 387         long prev_posn = 0;
 388         char range[RANGE_HEADER_SIZE];
 389         struct curl_slist *range_header = NULL;
 390
 391         FILE *indexfile;
 392         struct active_request_slot *slot;
 393         struct slot_results results;
 394
 395         if (has_pack_index(sha1))
 396                 return 0;
 397
 398         if (get_verbosely)
 399                 fprintf(stderr, "Getting index for pack %s\n", hex);
 400
 401         url = xmalloc(strlen(repo->base) + 64);
 402         sprintf(url, "%s/objects/pack/pack-%s.idx", repo->base, hex);
 403
 404         filename = sha1_pack_index_name(sha1);
 405         snprintf(tmpfile, sizeof(tmpfile), "%s.temp", filename);
 406         indexfile = fopen(tmpfile, "a");
 407         if (!indexfile)
 408                 return error("Unable to open local file %s for pack index",
 409                              filename);
 410
 411         slot = get_active_slot();
 412         slot->results = &results;
 413         curl_easy_setopt(slot->curl, CURLOPT_FILE, indexfile);
 414         curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite);
 415         curl_easy_setopt(slot->curl, CURLOPT_URL, url);
 416         curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, no_pragma_header);
 417         slot->local = indexfile;
 418
 419         /* If there is data present from a previous transfer attempt,
 420            resume where it left off */
 421         prev_posn = ftell(indexfile);
 422         if (prev_posn>0) {
 423                 if (get_verbosely)
 424                         fprintf(stderr,
 425                                 "Resuming fetch of index for pack %s at byte %ld\n",
 426                                 hex, prev_posn);
 427                 sprintf(range, "Range: bytes=%ld-", prev_posn);
 428                 range_header = curl_slist_append(range_header, range);
 429                 curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, range_header);
 430         }
 431
 432         if (start_active_slot(slot)) {
 433                 run_active_slot(slot);
 434                 if (results.curl_result != CURLE_OK) {
 435                         fclose(indexfile);
 436                         return error("Unable to get pack index %s\n%s", url,
 437                                      curl_errorstr);
 438                 }
 439         } else {
 440                 fclose(indexfile);
 441                 return error("Unable to start request");
 442         }
 443
 444         fclose(indexfile);
 445
 446         return move_temp_to_file(tmpfile, filename);
 447 }
 448
 449 static int setup_index(struct alt_base *repo, unsigned char *sha1)
 450 {
 451         struct packed_git *new_pack;
 452         if (has_pack_file(sha1))
 453                 return 0; /* don't list this as something we can get */
 454
 455         if (fetch_index(repo, sha1))
 456                 return -1;
 457
 458         new_pack = parse_pack_index(sha1);
 459         new_pack->next = repo->packs;
 460         repo->packs = new_pack;
 461         return 0;
 462 }
 463
 464 static void process_alternates_response(void *callback_data)
 465 {
 466         struct alternates_request *alt_req =
 467                 (struct alternates_request *)callback_data;
 468         struct active_request_slot *slot = alt_req->slot;
 469         struct alt_base *tail = alt;
 470         const char *base = alt_req->base;
 471         static const char null_byte = '\0';
 472         char *data;
 473         int i = 0;
 474
 475         if (alt_req->http_specific) {
 476                 if (slot->curl_result != CURLE_OK ||
 477                     !alt_req->buffer->posn) {
 478
 479                         /* Try reusing the slot to get non-http alternates */
 480                         alt_req->http_specific = 0;
 481                         sprintf(alt_req->url, "%s/objects/info/alternates",
 482                                 base);
 483                         curl_easy_setopt(slot->curl, CURLOPT_URL,
 484                                          alt_req->url);
 485                         active_requests++;
 486                         slot->in_use = 1;
 487                         if (slot->finished != NULL)
 488                                 (*slot->finished) = 0;
 489                         if (!start_active_slot(slot)) {
 490                                 got_alternates = -1;
 491                                 slot->in_use = 0;
 492                                 if (slot->finished != NULL)
 493                                         (*slot->finished) = 1;
 494                         }
 495                         return;
 496                 }
 497         } else if (slot->curl_result != CURLE_OK) {
 498                 if (!missing_target(slot)) {
 499                         got_alternates = -1;
 500                         return;
 501                 }
 502         }
 503
 504         fwrite_buffer(&null_byte, 1, 1, alt_req->buffer);
 505         alt_req->buffer->posn--;
 506         data = alt_req->buffer->buffer;
 507
 508         while (i < alt_req->buffer->posn) {
 509                 int posn = i;
 510                 while (posn < alt_req->buffer->posn && data[posn] != '\n')
 511                         posn++;
 512                 if (data[posn] == '\n') {
 513                         int okay = 0;
 514                         int serverlen = 0;
 515                         struct alt_base *newalt;
 516                         char *target = NULL;
 517                         if (data[i] == '/') {
 518                                 /* This counts
 519                                  * http://git.host/pub/scm/linux.git/
 520                                  * -----------here^
 521                                  * so memcpy(dst, base, serverlen) will
 522                                  * copy up to "...git.host".
 523                                  */
 524                                 const char *colon_ss = strstr(base,"://");
 525                                 if (colon_ss) {
 526                                         serverlen = (strchr(colon_ss + 3, '/')
 527                                                      - base);
 528                                         okay = 1;
 529                                 }
 530                         } else if (!memcmp(data + i, "../", 3)) {
 531                                 /* Relative URL; chop the corresponding
 532                                  * number of subpath from base (and ../
 533                                  * from data), and concatenate the result.
 534                                  *
 535                                  * The code first drops ../ from data, and
 536                                  * then drops one ../ from data and one path
 537                                  * from base.  IOW, one extra ../ is dropped
 538                                  * from data than path is dropped from base.
 539                                  *
 540                                  * This is not wrong.  The alternate in
 541                                  *     http://git.host/pub/scm/linux.git/
 542                                  * to borrow from
 543                                  *     http://git.host/pub/scm/linus.git/
 544                                  * is ../../linus.git/objects/.  You need
 545                                  * two ../../ to borrow from your direct
 546                                  * neighbour.
 547                                  */
 548                                 i += 3;
 549                                 serverlen = strlen(base);
 550                                 while (i + 2 < posn &&
 551                                        !memcmp(data + i, "../", 3)) {
 552                                         do {
 553                                                 serverlen--;
 554                                         } while (serverlen &&
 555                                                  base[serverlen - 1] != '/');
 556                                         i += 3;
 557                                 }
 558                                 /* If the server got removed, give up. */
 559                                 okay = strchr(base, ':') - base + 3 <
 560                                         serverlen;
 561                         } else if (alt_req->http_specific) {
 562                                 char *colon = strchr(data + i, ':');
 563                                 char *slash = strchr(data + i, '/');
 564                                 if (colon && slash && colon < data + posn &&
 565                                     slash < data + posn && colon < slash) {
 566                                         okay = 1;
 567                                 }
 568                         }
 569                         /* skip "objects\n" at end */
 570                         if (okay) {
 571                                 target = xmalloc(serverlen + posn - i - 6);
 572                                 memcpy(target, base, serverlen);
 573                                 memcpy(target + serverlen, data + i,
 574                                        posn - i - 7);
 575                                 target[serverlen + posn - i - 7] = 0;
 576                                 if (get_verbosely)
 577                                         fprintf(stderr,
 578                                                 "Also look at %s\n", target);
 579                                 newalt = xmalloc(sizeof(*newalt));
 580                                 newalt->next = NULL;
 581                                 newalt->base = target;
 582                                 newalt->got_indices = 0;
 583                                 newalt->packs = NULL;
 584
 585                                 while (tail->next != NULL)
 586                                         tail = tail->next;
 587                                 tail->next = newalt;
 588                         }
 589                 }
 590                 i = posn + 1;
 591         }
 592
 593         got_alternates = 1;
 594 }
 595
 596 static void fetch_alternates(const char *base)
 597 {
 598         struct buffer buffer;
 599         char *url;
 600         char *data;
 601         struct active_request_slot *slot;
 602         struct alternates_request alt_req;
 603
 604         /* If another request has already started fetching alternates,
 605            wait for them to arrive and return to processing this request's
 606            curl message */
 607 #ifdef USE_CURL_MULTI
 608         while (got_alternates == 0) {
 609                 step_active_slots();
 610         }
 611 #endif
 612
 613         /* Nothing to do if they've already been fetched */
 614         if (got_alternates == 1)
 615                 return;
 616
 617         /* Start the fetch */
 618         got_alternates = 0;
 619
 620         data = xmalloc(4096);
 621         buffer.size = 4096;
 622         buffer.posn = 0;
 623         buffer.buffer = data;
 624
 625         if (get_verbosely)
 626                 fprintf(stderr, "Getting alternates list for %s\n", base);
 627
 628         url = xmalloc(strlen(base) + 31);
 629         sprintf(url, "%s/objects/info/http-alternates", base);
 630
 631         /* Use a callback to process the result, since another request
 632            may fail and need to have alternates loaded before continuing */
 633         slot = get_active_slot();
 634         slot->callback_func = process_alternates_response;
 635         slot->callback_data = &alt_req;
 636
 637         curl_easy_setopt(slot->curl, CURLOPT_FILE, &buffer);
 638         curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite_buffer);
 639         curl_easy_setopt(slot->curl, CURLOPT_URL, url);
 640
 641         alt_req.base = base;
 642         alt_req.url = url;
 643         alt_req.buffer = &buffer;
 644         alt_req.http_specific = 1;
 645         alt_req.slot = slot;
 646
 647         if (start_active_slot(slot))
 648                 run_active_slot(slot);
 649         else
 650                 got_alternates = -1;
 651
 652         free(data);
 653         free(url);
 654 }
 655
 656 static int fetch_indices(struct alt_base *repo)
 657 {
 658         unsigned char sha1[20];
 659         char *url;
 660         struct buffer buffer;
 661         char *data;
 662         int i = 0;
 663
 664         struct active_request_slot *slot;
 665         struct slot_results results;
 666
 667         if (repo->got_indices)
 668                 return 0;
 669
 670         data = xmalloc(4096);
 671         buffer.size = 4096;
 672         buffer.posn = 0;
 673         buffer.buffer = data;
 674
 675         if (get_verbosely)
 676                 fprintf(stderr, "Getting pack list for %s\n", repo->base);
 677
 678         url = xmalloc(strlen(repo->base) + 21);
 679         sprintf(url, "%s/objects/info/packs", repo->base);
 680
 681         slot = get_active_slot();
 682         slot->results = &results;
 683         curl_easy_setopt(slot->curl, CURLOPT_FILE, &buffer);
 684         curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite_buffer);
 685         curl_easy_setopt(slot->curl, CURLOPT_URL, url);
 686         curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, NULL);
 687         if (start_active_slot(slot)) {
 688                 run_active_slot(slot);
 689                 if (results.curl_result != CURLE_OK) {
 690                         if (missing_target(&results)) {
 691                                 repo->got_indices = 1;
 692                                 free(buffer.buffer);
 693                                 return 0;
 694                         } else {
 695                                 repo->got_indices = 0;
 696                                 free(buffer.buffer);
 697                                 return error("%s", curl_errorstr);
 698                         }
 699                 }
 700         } else {
 701                 repo->got_indices = 0;
 702                 free(buffer.buffer);
 703                 return error("Unable to start request");
 704         }
 705
 706         data = buffer.buffer;
 707         while (i < buffer.posn) {
 708                 switch (data[i]) {
 709                 case 'P':
 710                         i++;
 711                         if (i + 52 <= buffer.posn &&
 712                             !prefixcmp(data + i, " pack-") &&
 713                             !prefixcmp(data + i + 46, ".pack\n")) {
 714                                 get_sha1_hex(data + i + 6, sha1);
 715                                 setup_index(repo, sha1);
 716                                 i += 51;
 717                                 break;
 718                         }
 719                 default:
 720                         while (i < buffer.posn && data[i] != '\n')
 721                                 i++;
 722                 }
 723                 i++;
 724         }
 725
 726         free(buffer.buffer);
 727         repo->got_indices = 1;
 728         return 0;
 729 }
 730
 731 static int fetch_pack(struct alt_base *repo, unsigned char *sha1)
 732 {
 733         char *url;
 734         struct packed_git *target;
 735         struct packed_git **lst;
 736         FILE *packfile;
 737         char *filename;
 738         char tmpfile[PATH_MAX];
 739         int ret;
 740         long prev_posn = 0;
 741         char range[RANGE_HEADER_SIZE];
 742         struct curl_slist *range_header = NULL;
 743
 744         struct active_request_slot *slot;
 745         struct slot_results results;
 746
 747         if (fetch_indices(repo))
 748                 return -1;
 749         target = find_sha1_pack(sha1, repo->packs);
 750         if (!target)
 751                 return -1;
 752
 753         if (get_verbosely) {
 754                 fprintf(stderr, "Getting pack %s\n",
 755                         sha1_to_hex(target->sha1));
 756                 fprintf(stderr, " which contains %s\n",
 757                         sha1_to_hex(sha1));
 758         }
 759
 760         url = xmalloc(strlen(repo->base) + 65);
 761         sprintf(url, "%s/objects/pack/pack-%s.pack",
 762                 repo->base, sha1_to_hex(target->sha1));
 763
 764         filename = sha1_pack_name(target->sha1);
 765         snprintf(tmpfile, sizeof(tmpfile), "%s.temp", filename);
 766         packfile = fopen(tmpfile, "a");
 767         if (!packfile)
 768                 return error("Unable to open local file %s for pack",
 769                              filename);
 770
 771         slot = get_active_slot();
 772         slot->results = &results;
 773         curl_easy_setopt(slot->curl, CURLOPT_FILE, packfile);
 774         curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite);
 775         curl_easy_setopt(slot->curl, CURLOPT_URL, url);
 776         curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, no_pragma_header);
 777         slot->local = packfile;
 778
 779         /* If there is data present from a previous transfer attempt,
 780            resume where it left off */
 781         prev_posn = ftell(packfile);
 782         if (prev_posn>0) {
 783                 if (get_verbosely)
 784                         fprintf(stderr,
 785                                 "Resuming fetch of pack %s at byte %ld\n",
 786                                 sha1_to_hex(target->sha1), prev_posn);
 787                 sprintf(range, "Range: bytes=%ld-", prev_posn);
 788                 range_header = curl_slist_append(range_header, range);
 789                 curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, range_header);
 790         }
 791
 792         if (start_active_slot(slot)) {
 793                 run_active_slot(slot);
 794                 if (results.curl_result != CURLE_OK) {
 795                         fclose(packfile);
 796                         return error("Unable to get pack file %s\n%s", url,
 797                                      curl_errorstr);
 798                 }
 799         } else {
 800                 fclose(packfile);
 801                 return error("Unable to start request");
 802         }
 803
 804         target->pack_size = ftell(packfile);
 805         fclose(packfile);
 806
 807         ret = move_temp_to_file(tmpfile, filename);
 808         if (ret)
 809                 return ret;
 810
 811         lst = &repo->packs;
 812         while (*lst != target)
 813                 lst = &((*lst)->next);
 814         *lst = (*lst)->next;
 815
 816         if (verify_pack(target, 0))
 817                 return -1;
 818         install_packed_git(target);
 819
 820         return 0;
 821 }
 822
 823 static void abort_object_request(struct object_request *obj_req)
 824 {
 825         if (obj_req->local >= 0) {
 826                 close(obj_req->local);
 827                 obj_req->local = -1;
 828         }
 829         unlink(obj_req->tmpfile);
 830         if (obj_req->slot) {
 831                 release_active_slot(obj_req->slot);
 832                 obj_req->slot = NULL;
 833         }
 834         release_object_request(obj_req);
 835 }
 836
 837 static int fetch_object(struct alt_base *repo, unsigned char *sha1)
 838 {
 839         char *hex = sha1_to_hex(sha1);
 840         int ret = 0;
 841         struct object_request *obj_req = object_queue_head;
 842
 843         while (obj_req != NULL && hashcmp(obj_req->sha1, sha1))
 844                 obj_req = obj_req->next;
 845         if (obj_req == NULL)
 846                 return error("Couldn't find request for %s in the queue", hex);
 847
 848         if (has_sha1_file(obj_req->sha1)) {
 849                 abort_object_request(obj_req);
 850                 return 0;
 851         }
 852
 853 #ifdef USE_CURL_MULTI
 854         while (obj_req->state == WAITING) {
 855                 step_active_slots();
 856         }
 857 #else
 858         start_object_request(obj_req);
 859 #endif
 860
 861         while (obj_req->state == ACTIVE) {
 862                 run_active_slot(obj_req->slot);
 863         }
 864         if (obj_req->local != -1) {
 865                 close(obj_req->local); obj_req->local = -1;
 866         }
 867
 868         if (obj_req->state == ABORTED) {
 869                 ret = error("Request for %s aborted", hex);
 870         } else if (obj_req->curl_result != CURLE_OK &&
 871                    obj_req->http_code != 416) {
 872                 if (missing_target(obj_req))
 873                         ret = -1; /* Be silent, it is probably in a pack. */
 874                 else
 875                         ret = error("%s (curl_result = %d, http_code = %ld, sha1 = %s)",
 876                                     obj_req->errorstr, obj_req->curl_result,
 877                                     obj_req->http_code, hex);
 878         } else if (obj_req->zret != Z_STREAM_END) {
 879                 corrupt_object_found++;
 880                 ret = error("File %s (%s) corrupt", hex, obj_req->url);
 881         } else if (hashcmp(obj_req->sha1, obj_req->real_sha1)) {
 882                 ret = error("File %s has bad hash", hex);
 883         } else if (obj_req->rename < 0) {
 884                 ret = error("unable to write sha1 filename %s",
 885                             obj_req->filename);
 886         }
 887
 888         release_object_request(obj_req);
 889         return ret;
 890 }
 891
 892 int fetch(unsigned char *sha1)
 893 {
 894         struct alt_base *altbase = alt;
 895
 896         if (!fetch_object(altbase, sha1))
 897                 return 0;
 898         while (altbase) {
 899                 if (!fetch_pack(altbase, sha1))
 900                         return 0;
 901                 fetch_alternates(alt->base);
 902                 altbase = altbase->next;
 903         }
 904         return error("Unable to find %s under %s", sha1_to_hex(sha1),
 905                      alt->base);
 906 }
 907
 908 static inline int needs_quote(int ch)
 909 {
 910         if (((ch >= 'A') && (ch <= 'Z'))
 911                         || ((ch >= 'a') && (ch <= 'z'))
 912                         || ((ch >= '0') && (ch <= '9'))
 913                         || (ch == '/')
 914                         || (ch == '-')
 915                         || (ch == '.'))
 916                 return 0;
 917         return 1;
 918 }
 919
 920 static inline int hex(int v)
 921 {
 922         if (v < 10) return '0' + v;
 923         else return 'A' + v - 10;
 924 }
 925
 926 static char *quote_ref_url(const char *base, const char *ref)
 927 {
 928         const char *cp;
 929         char *dp, *qref;
 930         int len, baselen, ch;
 931
 932         baselen = strlen(base);
 933         len = baselen + 7; /* "/refs/" + NUL */
 934         for (cp = ref; (ch = *cp) != 0; cp++, len++)
 935                 if (needs_quote(ch))
 936                         len += 2; /* extra two hex plus replacement % */
 937         qref = xmalloc(len);
 938         memcpy(qref, base, baselen);
 939         memcpy(qref + baselen, "/refs/", 6);
 940         for (cp = ref, dp = qref + baselen + 6; (ch = *cp) != 0; cp++) {
 941                 if (needs_quote(ch)) {
 942                         *dp++ = '%';
 943                         *dp++ = hex((ch >> 4) & 0xF);
 944                         *dp++ = hex(ch & 0xF);
 945                 }
 946                 else
 947                         *dp++ = ch;
 948         }
 949         *dp = 0;
 950
 951         return qref;
 952 }
 953
 954 int fetch_ref(char *ref, unsigned char *sha1)
 955 {
 956         char *url;
 957         char hex[42];
 958         struct buffer buffer;
 959         const char *base = alt->base;
 960         struct active_request_slot *slot;
 961         struct slot_results results;
 962         buffer.size = 41;
 963         buffer.posn = 0;
 964         buffer.buffer = hex;
 965         hex[41] = '\0';
 966
 967         url = quote_ref_url(base, ref);
 968         slot = get_active_slot();
 969         slot->results = &results;
 970         curl_easy_setopt(slot->curl, CURLOPT_FILE, &buffer);
 971         curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite_buffer);
 972         curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, NULL);
 973         curl_easy_setopt(slot->curl, CURLOPT_URL, url);
 974         if (start_active_slot(slot)) {
 975                 run_active_slot(slot);
 976                 if (results.curl_result != CURLE_OK)
 977                         return error("Couldn't get %s for %s\n%s",
 978                                      url, ref, curl_errorstr);
 979         } else {
 980                 return error("Unable to start request");
 981         }
 982
 983         hex[40] = '\0';
 984         get_sha1_hex(hex, sha1);
 985         return 0;
 986 }
 987
 988 int main(int argc, const char **argv)
 989 {
 990         int commits;
 991         const char **write_ref = NULL;
 992         char **commit_id;
 993         const char *url;
 994         char *s;
 995         int arg = 1;
 996         int rc = 0;
 997
 998         setup_git_directory();
 999         git_config(git_default_config);
1000
1001         while (arg < argc && argv[arg][0] == '-') {
1002                 if (argv[arg][1] == 't') {
1003                         get_tree = 1;
1004                 } else if (argv[arg][1] == 'c') {
1005                         get_history = 1;
1006                 } else if (argv[arg][1] == 'a') {
1007                         get_all = 1;
1008                         get_tree = 1;
1009                         get_history = 1;
1010                 } else if (argv[arg][1] == 'v') {
1011                         get_verbosely = 1;
1012                 } else if (argv[arg][1] == 'w') {
1013                         write_ref = &argv[arg + 1];
1014                         arg++;
1015                 } else if (!strcmp(argv[arg], "--recover")) {
1016                         get_recover = 1;
1017                 } else if (!strcmp(argv[arg], "--stdin")) {
1018                         commits_on_stdin = 1;
1019                 }
1020                 arg++;
1021         }
1022         if (argc < arg + 2 - commits_on_stdin) {
1023                 usage("git-http-fetch [-c] [-t] [-a] [-v] [--recover] [-w ref] [--stdin] commit-id url");
1024                 return 1;
1025         }
1026         if (commits_on_stdin) {
1027                 commits = pull_targets_stdin(&commit_id, &write_ref);
1028         } else {
1029                 commit_id = (char **) &argv[arg++];
1030                 commits = 1;
1031         }
1032         url = argv[arg];
1033
1034         http_init();
1035
1036         no_pragma_header = curl_slist_append(no_pragma_header, "Pragma:");
1037
1038         alt = xmalloc(sizeof(*alt));
1039         alt->base = xmalloc(strlen(url) + 1);
1040         strcpy(alt->base, url);
1041         for (s = alt->base + strlen(alt->base) - 1; *s == '/'; --s)
1042                 *s = 0;
1043         alt->got_indices = 0;
1044         alt->packs = NULL;
1045         alt->next = NULL;
1046
1047         if (pull(commits, commit_id, write_ref, url))
1048                 rc = 1;
1049
1050         http_cleanup();
1051
1052         curl_slist_free_all(no_pragma_header);
1053
1054         if (commits_on_stdin)
1055                 pull_targets_free(commits, commit_id, write_ref);
1056
1057         if (corrupt_object_found) {
1058                 fprintf(stderr,
1059 "Some loose object were found to be corrupt, but they might be just\n"
1060 "a false '404 Not Found' error message sent with incorrect HTTP\n"
1061 "status code.  Suggest running git-fsck.\n");
1062         }
1063         return rc;
1064 }