http-fetch.c

   1 #include "cache.h"
   2 #include "commit.h"
   3 #include "pack.h"
   4 #include "fetch.h"
   5 #include "http.h"
   6
   7 #define PREV_BUF_SIZE 4096
   8 #define RANGE_HEADER_SIZE 30
   9
  10 static int commits_on_stdin;
  11
  12 static int got_alternates = -1;
  13 static int corrupt_object_found;
  14
  15 static struct curl_slist *no_pragma_header;
  16
  17 struct alt_base
  18 {
  19         char *base;
  20         int got_indices;
  21         struct packed_git *packs;
  22         struct alt_base *next;
  23 };
  24
  25 static struct alt_base *alt;
  26
  27 enum object_request_state {
  28         WAITING,
  29         ABORTED,
  30         ACTIVE,
  31         COMPLETE,
  32 };
  33
  34 struct object_request
  35 {
  36         unsigned char sha1[20];
  37         struct alt_base *repo;
  38         char *url;
  39         char filename[PATH_MAX];
  40         char tmpfile[PATH_MAX];
  41         int local;
  42         enum object_request_state state;
  43         CURLcode curl_result;
  44         char errorstr[CURL_ERROR_SIZE];
  45         long http_code;
  46         unsigned char real_sha1[20];
  47         SHA_CTX c;
  48         z_stream stream;
  49         int zret;
  50         int rename;
  51         struct active_request_slot *slot;
  52         struct object_request *next;
  53 };
  54
  55 struct alternates_request {
  56         const char *base;
  57         char *url;
  58         struct buffer *buffer;
  59         struct active_request_slot *slot;
  60         int http_specific;
  61 };
  62
  63 static struct object_request *object_queue_head;
  64
  65 static size_t fwrite_sha1_file(void *ptr, size_t eltsize, size_t nmemb,
  66                                void *data)
  67 {
  68         unsigned char expn[4096];
  69         size_t size = eltsize * nmemb;
  70         int posn = 0;
  71         struct object_request *obj_req = (struct object_request *)data;
  72         do {
  73                 ssize_t retval = xwrite(obj_req->local,
  74                                        (char *) ptr + posn, size - posn);
  75                 if (retval < 0)
  76                         return posn;
  77                 posn += retval;
  78         } while (posn < size);
  79
  80         obj_req->stream.avail_in = size;
  81         obj_req->stream.next_in = ptr;
  82         do {
  83                 obj_req->stream.next_out = expn;
  84                 obj_req->stream.avail_out = sizeof(expn);
  85                 obj_req->zret = inflate(&obj_req->stream, Z_SYNC_FLUSH);
  86                 SHA1_Update(&obj_req->c, expn,
  87                             sizeof(expn) - obj_req->stream.avail_out);
  88         } while (obj_req->stream.avail_in && obj_req->zret == Z_OK);
  89         data_received++;
  90         return size;
  91 }
  92
  93 static int missing__target(int code, int result)
  94 {
  95         return  /* file:// URL -- do we ever use one??? */
  96                 (result == CURLE_FILE_COULDNT_READ_FILE) ||
  97                 /* http:// and https:// URL */
  98                 (code == 404 && result == CURLE_HTTP_RETURNED_ERROR) ||
  99                 /* ftp:// URL */
 100                 (code == 550 && result == CURLE_FTP_COULDNT_RETR_FILE)
 101                 ;
 102 }
 103
 104 #define missing_target(a) missing__target((a)->http_code, (a)->curl_result)
 105
 106 static void fetch_alternates(const char *base);
 107
 108 static void process_object_response(void *callback_data);
 109
 110 static void start_object_request(struct object_request *obj_req)
 111 {
 112         char *hex = sha1_to_hex(obj_req->sha1);
 113         char prevfile[PATH_MAX];
 114         char *url;
 115         char *posn;
 116         int prevlocal;
 117         unsigned char prev_buf[PREV_BUF_SIZE];
 118         ssize_t prev_read = 0;
 119         long prev_posn = 0;
 120         char range[RANGE_HEADER_SIZE];
 121         struct curl_slist *range_header = NULL;
 122         struct active_request_slot *slot;
 123
 124         snprintf(prevfile, sizeof(prevfile), "%s.prev", obj_req->filename);
 125         unlink(prevfile);
 126         rename(obj_req->tmpfile, prevfile);
 127         unlink(obj_req->tmpfile);
 128
 129         if (obj_req->local != -1)
 130                 error("fd leakage in start: %d", obj_req->local);
 131         obj_req->local = open(obj_req->tmpfile,
 132                               O_WRONLY | O_CREAT | O_EXCL, 0666);
 133         /* This could have failed due to the "lazy directory creation";
 134          * try to mkdir the last path component.
 135          */
 136         if (obj_req->local < 0 && errno == ENOENT) {
 137                 char *dir = strrchr(obj_req->tmpfile, '/');
 138                 if (dir) {
 139                         *dir = 0;
 140                         mkdir(obj_req->tmpfile, 0777);
 141                         *dir = '/';
 142                 }
 143                 obj_req->local = open(obj_req->tmpfile,
 144                                       O_WRONLY | O_CREAT | O_EXCL, 0666);
 145         }
 146
 147         if (obj_req->local < 0) {
 148                 obj_req->state = ABORTED;
 149                 error("Couldn't create temporary file %s for %s: %s",
 150                       obj_req->tmpfile, obj_req->filename, strerror(errno));
 151                 return;
 152         }
 153
 154         memset(&obj_req->stream, 0, sizeof(obj_req->stream));
 155
 156         inflateInit(&obj_req->stream);
 157
 158         SHA1_Init(&obj_req->c);
 159
 160         url = xmalloc(strlen(obj_req->repo->base) + 51);
 161         obj_req->url = xmalloc(strlen(obj_req->repo->base) + 51);
 162         strcpy(url, obj_req->repo->base);
 163         posn = url + strlen(obj_req->repo->base);
 164         strcpy(posn, "/objects/");
 165         posn += 9;
 166         memcpy(posn, hex, 2);
 167         posn += 2;
 168         *(posn++) = '/';
 169         strcpy(posn, hex + 2);
 170         strcpy(obj_req->url, url);
 171
 172         /* If a previous temp file is present, process what was already
 173            fetched. */
 174         prevlocal = open(prevfile, O_RDONLY);
 175         if (prevlocal != -1) {
 176                 do {
 177                         prev_read = xread(prevlocal, prev_buf, PREV_BUF_SIZE);
 178                         if (prev_read>0) {
 179                                 if (fwrite_sha1_file(prev_buf,
 180                                                      1,
 181                                                      prev_read,
 182                                                      obj_req) == prev_read) {
 183                                         prev_posn += prev_read;
 184                                 } else {
 185                                         prev_read = -1;
 186                                 }
 187                         }
 188                 } while (prev_read > 0);
 189                 close(prevlocal);
 190         }
 191         unlink(prevfile);
 192
 193         /* Reset inflate/SHA1 if there was an error reading the previous temp
 194            file; also rewind to the beginning of the local file. */
 195         if (prev_read == -1) {
 196                 memset(&obj_req->stream, 0, sizeof(obj_req->stream));
 197                 inflateInit(&obj_req->stream);
 198                 SHA1_Init(&obj_req->c);
 199                 if (prev_posn>0) {
 200                         prev_posn = 0;
 201                         lseek(obj_req->local, 0, SEEK_SET);
 202                         ftruncate(obj_req->local, 0);
 203                 }
 204         }
 205
 206         slot = get_active_slot();
 207         slot->callback_func = process_object_response;
 208         slot->callback_data = obj_req;
 209         obj_req->slot = slot;
 210
 211         curl_easy_setopt(slot->curl, CURLOPT_FILE, obj_req);
 212         curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite_sha1_file);
 213         curl_easy_setopt(slot->curl, CURLOPT_ERRORBUFFER, obj_req->errorstr);
 214         curl_easy_setopt(slot->curl, CURLOPT_URL, url);
 215         curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, no_pragma_header);
 216
 217         /* If we have successfully processed data from a previous fetch
 218            attempt, only fetch the data we don't already have. */
 219         if (prev_posn>0) {
 220                 if (get_verbosely)
 221                         fprintf(stderr,
 222                                 "Resuming fetch of object %s at byte %ld\n",
 223                                 hex, prev_posn);
 224                 sprintf(range, "Range: bytes=%ld-", prev_posn);
 225                 range_header = curl_slist_append(range_header, range);
 226                 curl_easy_setopt(slot->curl,
 227                                  CURLOPT_HTTPHEADER, range_header);
 228         }
 229
 230         /* Try to get the request started, abort the request on error */
 231         obj_req->state = ACTIVE;
 232         if (!start_active_slot(slot)) {
 233                 obj_req->state = ABORTED;
 234                 obj_req->slot = NULL;
 235                 close(obj_req->local); obj_req->local = -1;
 236                 free(obj_req->url);
 237                 return;
 238         }
 239 }
 240
 241 static void finish_object_request(struct object_request *obj_req)
 242 {
 243         struct stat st;
 244
 245         fchmod(obj_req->local, 0444);
 246         close(obj_req->local); obj_req->local = -1;
 247
 248         if (obj_req->http_code == 416) {
 249                 fprintf(stderr, "Warning: requested range invalid; we may already have all the data.\n");
 250         } else if (obj_req->curl_result != CURLE_OK) {
 251                 if (stat(obj_req->tmpfile, &st) == 0)
 252                         if (st.st_size == 0)
 253                                 unlink(obj_req->tmpfile);
 254                 return;
 255         }
 256
 257         inflateEnd(&obj_req->stream);
 258         SHA1_Final(obj_req->real_sha1, &obj_req->c);
 259         if (obj_req->zret != Z_STREAM_END) {
 260                 unlink(obj_req->tmpfile);
 261                 return;
 262         }
 263         if (hashcmp(obj_req->sha1, obj_req->real_sha1)) {
 264                 unlink(obj_req->tmpfile);
 265                 return;
 266         }
 267         obj_req->rename =
 268                 move_temp_to_file(obj_req->tmpfile, obj_req->filename);
 269
 270         if (obj_req->rename == 0)
 271                 pull_say("got %s\n", sha1_to_hex(obj_req->sha1));
 272 }
 273
 274 static void process_object_response(void *callback_data)
 275 {
 276         struct object_request *obj_req =
 277                 (struct object_request *)callback_data;
 278
 279         obj_req->curl_result = obj_req->slot->curl_result;
 280         obj_req->http_code = obj_req->slot->http_code;
 281         obj_req->slot = NULL;
 282         obj_req->state = COMPLETE;
 283
 284         /* Use alternates if necessary */
 285         if (missing_target(obj_req)) {
 286                 fetch_alternates(alt->base);
 287                 if (obj_req->repo->next != NULL) {
 288                         obj_req->repo =
 289                                 obj_req->repo->next;
 290                         close(obj_req->local);
 291                         obj_req->local = -1;
 292                         start_object_request(obj_req);
 293                         return;
 294                 }
 295         }
 296
 297         finish_object_request(obj_req);
 298 }
 299
 300 static void release_object_request(struct object_request *obj_req)
 301 {
 302         struct object_request *entry = object_queue_head;
 303
 304         if (obj_req->local != -1)
 305                 error("fd leakage in release: %d", obj_req->local);
 306         if (obj_req == object_queue_head) {
 307                 object_queue_head = obj_req->next;
 308         } else {
 309                 while (entry->next != NULL && entry->next != obj_req)
 310                         entry = entry->next;
 311                 if (entry->next == obj_req)
 312                         entry->next = entry->next->next;
 313         }
 314
 315         free(obj_req->url);
 316         free(obj_req);
 317 }
 318
 319 #ifdef USE_CURL_MULTI
 320 static int fill_active_slot(void *unused)
 321 {
 322         struct object_request *obj_req;
 323
 324         for (obj_req = object_queue_head; obj_req; obj_req = obj_req->next) {
 325                 if (obj_req->state == WAITING) {
 326                         if (has_sha1_file(obj_req->sha1))
 327                                 obj_req->state = COMPLETE;
 328                         else {
 329                                 start_object_request(obj_req);
 330                                 return 1;
 331                         }
 332                 }
 333         }
 334         return 0;
 335 }
 336 #endif
 337
 338 void prefetch(unsigned char *sha1)
 339 {
 340         struct object_request *newreq;
 341         struct object_request *tail;
 342         char *filename = sha1_file_name(sha1);
 343
 344         newreq = xmalloc(sizeof(*newreq));
 345         hashcpy(newreq->sha1, sha1);
 346         newreq->repo = alt;
 347         newreq->url = NULL;
 348         newreq->local = -1;
 349         newreq->state = WAITING;
 350         snprintf(newreq->filename, sizeof(newreq->filename), "%s", filename);
 351         snprintf(newreq->tmpfile, sizeof(newreq->tmpfile),
 352                  "%s.temp", filename);
 353         newreq->slot = NULL;
 354         newreq->next = NULL;
 355
 356         if (object_queue_head == NULL) {
 357                 object_queue_head = newreq;
 358         } else {
 359                 tail = object_queue_head;
 360                 while (tail->next != NULL) {
 361                         tail = tail->next;
 362                 }
 363                 tail->next = newreq;
 364         }
 365
 366 #ifdef USE_CURL_MULTI
 367         fill_active_slots();
 368         step_active_slots();
 369 #endif
 370 }
 371
 372 static int fetch_index(struct alt_base *repo, unsigned char *sha1)
 373 {
 374         char *hex = sha1_to_hex(sha1);
 375         char *filename;
 376         char *url;
 377         char tmpfile[PATH_MAX];
 378         long prev_posn = 0;
 379         char range[RANGE_HEADER_SIZE];
 380         struct curl_slist *range_header = NULL;
 381
 382         FILE *indexfile;
 383         struct active_request_slot *slot;
 384         struct slot_results results;
 385
 386         if (has_pack_index(sha1))
 387                 return 0;
 388
 389         if (get_verbosely)
 390                 fprintf(stderr, "Getting index for pack %s\n", hex);
 391
 392         url = xmalloc(strlen(repo->base) + 64);
 393         sprintf(url, "%s/objects/pack/pack-%s.idx", repo->base, hex);
 394
 395         filename = sha1_pack_index_name(sha1);
 396         snprintf(tmpfile, sizeof(tmpfile), "%s.temp", filename);
 397         indexfile = fopen(tmpfile, "a");
 398         if (!indexfile)
 399                 return error("Unable to open local file %s for pack index",
 400                              filename);
 401
 402         slot = get_active_slot();
 403         slot->results = &results;
 404         curl_easy_setopt(slot->curl, CURLOPT_FILE, indexfile);
 405         curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite);
 406         curl_easy_setopt(slot->curl, CURLOPT_URL, url);
 407         curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, no_pragma_header);
 408         slot->local = indexfile;
 409
 410         /* If there is data present from a previous transfer attempt,
 411            resume where it left off */
 412         prev_posn = ftell(indexfile);
 413         if (prev_posn>0) {
 414                 if (get_verbosely)
 415                         fprintf(stderr,
 416                                 "Resuming fetch of index for pack %s at byte %ld\n",
 417                                 hex, prev_posn);
 418                 sprintf(range, "Range: bytes=%ld-", prev_posn);
 419                 range_header = curl_slist_append(range_header, range);
 420                 curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, range_header);
 421         }
 422
 423         if (start_active_slot(slot)) {
 424                 run_active_slot(slot);
 425                 if (results.curl_result != CURLE_OK) {
 426                         fclose(indexfile);
 427                         return error("Unable to get pack index %s\n%s", url,
 428                                      curl_errorstr);
 429                 }
 430         } else {
 431                 fclose(indexfile);
 432                 return error("Unable to start request");
 433         }
 434
 435         fclose(indexfile);
 436
 437         return move_temp_to_file(tmpfile, filename);
 438 }
 439
 440 static int setup_index(struct alt_base *repo, unsigned char *sha1)
 441 {
 442         struct packed_git *new_pack;
 443         if (has_pack_file(sha1))
 444                 return 0; /* don't list this as something we can get */
 445
 446         if (fetch_index(repo, sha1))
 447                 return -1;
 448
 449         new_pack = parse_pack_index(sha1);
 450         new_pack->next = repo->packs;
 451         repo->packs = new_pack;
 452         return 0;
 453 }
 454
 455 static void process_alternates_response(void *callback_data)
 456 {
 457         struct alternates_request *alt_req =
 458                 (struct alternates_request *)callback_data;
 459         struct active_request_slot *slot = alt_req->slot;
 460         struct alt_base *tail = alt;
 461         const char *base = alt_req->base;
 462         static const char null_byte = '\0';
 463         char *data;
 464         int i = 0;
 465
 466         if (alt_req->http_specific) {
 467                 if (slot->curl_result != CURLE_OK ||
 468                     !alt_req->buffer->posn) {
 469
 470                         /* Try reusing the slot to get non-http alternates */
 471                         alt_req->http_specific = 0;
 472                         sprintf(alt_req->url, "%s/objects/info/alternates",
 473                                 base);
 474                         curl_easy_setopt(slot->curl, CURLOPT_URL,
 475                                          alt_req->url);
 476                         active_requests++;
 477                         slot->in_use = 1;
 478                         if (slot->finished != NULL)
 479                                 (*slot->finished) = 0;
 480                         if (!start_active_slot(slot)) {
 481                                 got_alternates = -1;
 482                                 slot->in_use = 0;
 483                                 if (slot->finished != NULL)
 484                                         (*slot->finished) = 1;
 485                         }
 486                         return;
 487                 }
 488         } else if (slot->curl_result != CURLE_OK) {
 489                 if (!missing_target(slot)) {
 490                         got_alternates = -1;
 491                         return;
 492                 }
 493         }
 494
 495         fwrite_buffer(&null_byte, 1, 1, alt_req->buffer);
 496         alt_req->buffer->posn--;
 497         data = alt_req->buffer->buffer;
 498
 499         while (i < alt_req->buffer->posn) {
 500                 int posn = i;
 501                 while (posn < alt_req->buffer->posn && data[posn] != '\n')
 502                         posn++;
 503                 if (data[posn] == '\n') {
 504                         int okay = 0;
 505                         int serverlen = 0;
 506                         struct alt_base *newalt;
 507                         char *target = NULL;
 508                         if (data[i] == '/') {
 509                                 /* This counts
 510                                  * http://git.host/pub/scm/linux.git/
 511                                  * -----------here^
 512                                  * so memcpy(dst, base, serverlen) will
 513                                  * copy up to "...git.host".
 514                                  */
 515                                 const char *colon_ss = strstr(base,"://");
 516                                 if (colon_ss) {
 517                                         serverlen = (strchr(colon_ss + 3, '/')
 518                                                      - base);
 519                                         okay = 1;
 520                                 }
 521                         } else if (!memcmp(data + i, "../", 3)) {
 522                                 /* Relative URL; chop the corresponding
 523                                  * number of subpath from base (and ../
 524                                  * from data), and concatenate the result.
 525                                  *
 526                                  * The code first drops ../ from data, and
 527                                  * then drops one ../ from data and one path
 528                                  * from base.  IOW, one extra ../ is dropped
 529                                  * from data than path is dropped from base.
 530                                  *
 531                                  * This is not wrong.  The alternate in
 532                                  *     http://git.host/pub/scm/linux.git/
 533                                  * to borrow from
 534                                  *     http://git.host/pub/scm/linus.git/
 535                                  * is ../../linus.git/objects/.  You need
 536                                  * two ../../ to borrow from your direct
 537                                  * neighbour.
 538                                  */
 539                                 i += 3;
 540                                 serverlen = strlen(base);
 541                                 while (i + 2 < posn &&
 542                                        !memcmp(data + i, "../", 3)) {
 543                                         do {
 544                                                 serverlen--;
 545                                         } while (serverlen &&
 546                                                  base[serverlen - 1] != '/');
 547                                         i += 3;
 548                                 }
 549                                 /* If the server got removed, give up. */
 550                                 okay = strchr(base, ':') - base + 3 <
 551                                         serverlen;
 552                         } else if (alt_req->http_specific) {
 553                                 char *colon = strchr(data + i, ':');
 554                                 char *slash = strchr(data + i, '/');
 555                                 if (colon && slash && colon < data + posn &&
 556                                     slash < data + posn && colon < slash) {
 557                                         okay = 1;
 558                                 }
 559                         }
 560                         /* skip "objects\n" at end */
 561                         if (okay) {
 562                                 target = xmalloc(serverlen + posn - i - 6);
 563                                 memcpy(target, base, serverlen);
 564                                 memcpy(target + serverlen, data + i,
 565                                        posn - i - 7);
 566                                 target[serverlen + posn - i - 7] = 0;
 567                                 if (get_verbosely)
 568                                         fprintf(stderr,
 569                                                 "Also look at %s\n", target);
 570                                 newalt = xmalloc(sizeof(*newalt));
 571                                 newalt->next = NULL;
 572                                 newalt->base = target;
 573                                 newalt->got_indices = 0;
 574                                 newalt->packs = NULL;
 575
 576                                 while (tail->next != NULL)
 577                                         tail = tail->next;
 578                                 tail->next = newalt;
 579                         }
 580                 }
 581                 i = posn + 1;
 582         }
 583
 584         got_alternates = 1;
 585 }
 586
 587 static void fetch_alternates(const char *base)
 588 {
 589         struct buffer buffer;
 590         char *url;
 591         char *data;
 592         struct active_request_slot *slot;
 593         struct alternates_request alt_req;
 594
 595         /* If another request has already started fetching alternates,
 596            wait for them to arrive and return to processing this request's
 597            curl message */
 598 #ifdef USE_CURL_MULTI
 599         while (got_alternates == 0) {
 600                 step_active_slots();
 601         }
 602 #endif
 603
 604         /* Nothing to do if they've already been fetched */
 605         if (got_alternates == 1)
 606                 return;
 607
 608         /* Start the fetch */
 609         got_alternates = 0;
 610
 611         data = xmalloc(4096);
 612         buffer.size = 4096;
 613         buffer.posn = 0;
 614         buffer.buffer = data;
 615
 616         if (get_verbosely)
 617                 fprintf(stderr, "Getting alternates list for %s\n", base);
 618
 619         url = xmalloc(strlen(base) + 31);
 620         sprintf(url, "%s/objects/info/http-alternates", base);
 621
 622         /* Use a callback to process the result, since another request
 623            may fail and need to have alternates loaded before continuing */
 624         slot = get_active_slot();
 625         slot->callback_func = process_alternates_response;
 626         slot->callback_data = &alt_req;
 627
 628         curl_easy_setopt(slot->curl, CURLOPT_FILE, &buffer);
 629         curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite_buffer);
 630         curl_easy_setopt(slot->curl, CURLOPT_URL, url);
 631
 632         alt_req.base = base;
 633         alt_req.url = url;
 634         alt_req.buffer = &buffer;
 635         alt_req.http_specific = 1;
 636         alt_req.slot = slot;
 637
 638         if (start_active_slot(slot))
 639                 run_active_slot(slot);
 640         else
 641                 got_alternates = -1;
 642
 643         free(data);
 644         free(url);
 645 }
 646
 647 static int fetch_indices(struct alt_base *repo)
 648 {
 649         unsigned char sha1[20];
 650         char *url;
 651         struct buffer buffer;
 652         char *data;
 653         int i = 0;
 654
 655         struct active_request_slot *slot;
 656         struct slot_results results;
 657
 658         if (repo->got_indices)
 659                 return 0;
 660
 661         data = xmalloc(4096);
 662         buffer.size = 4096;
 663         buffer.posn = 0;
 664         buffer.buffer = data;
 665
 666         if (get_verbosely)
 667                 fprintf(stderr, "Getting pack list for %s\n", repo->base);
 668
 669         url = xmalloc(strlen(repo->base) + 21);
 670         sprintf(url, "%s/objects/info/packs", repo->base);
 671
 672         slot = get_active_slot();
 673         slot->results = &results;
 674         curl_easy_setopt(slot->curl, CURLOPT_FILE, &buffer);
 675         curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite_buffer);
 676         curl_easy_setopt(slot->curl, CURLOPT_URL, url);
 677         curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, NULL);
 678         if (start_active_slot(slot)) {
 679                 run_active_slot(slot);
 680                 if (results.curl_result != CURLE_OK) {
 681                         if (missing_target(&results)) {
 682                                 repo->got_indices = 1;
 683                                 free(buffer.buffer);
 684                                 return 0;
 685                         } else {
 686                                 repo->got_indices = 0;
 687                                 free(buffer.buffer);
 688                                 return error("%s", curl_errorstr);
 689                         }
 690                 }
 691         } else {
 692                 repo->got_indices = 0;
 693                 free(buffer.buffer);
 694                 return error("Unable to start request");
 695         }
 696
 697         data = buffer.buffer;
 698         while (i < buffer.posn) {
 699                 switch (data[i]) {
 700                 case 'P':
 701                         i++;
 702                         if (i + 52 <= buffer.posn &&
 703                             !prefixcmp(data + i, " pack-") &&
 704                             !prefixcmp(data + i + 46, ".pack\n")) {
 705                                 get_sha1_hex(data + i + 6, sha1);
 706                                 setup_index(repo, sha1);
 707                                 i += 51;
 708                                 break;
 709                         }
 710                 default:
 711                         while (i < buffer.posn && data[i] != '\n')
 712                                 i++;
 713                 }
 714                 i++;
 715         }
 716
 717         free(buffer.buffer);
 718         repo->got_indices = 1;
 719         return 0;
 720 }
 721
 722 static int fetch_pack(struct alt_base *repo, unsigned char *sha1)
 723 {
 724         char *url;
 725         struct packed_git *target;
 726         struct packed_git **lst;
 727         FILE *packfile;
 728         char *filename;
 729         char tmpfile[PATH_MAX];
 730         int ret;
 731         long prev_posn = 0;
 732         char range[RANGE_HEADER_SIZE];
 733         struct curl_slist *range_header = NULL;
 734
 735         struct active_request_slot *slot;
 736         struct slot_results results;
 737
 738         if (fetch_indices(repo))
 739                 return -1;
 740         target = find_sha1_pack(sha1, repo->packs);
 741         if (!target)
 742                 return -1;
 743
 744         if (get_verbosely) {
 745                 fprintf(stderr, "Getting pack %s\n",
 746                         sha1_to_hex(target->sha1));
 747                 fprintf(stderr, " which contains %s\n",
 748                         sha1_to_hex(sha1));
 749         }
 750
 751         url = xmalloc(strlen(repo->base) + 65);
 752         sprintf(url, "%s/objects/pack/pack-%s.pack",
 753                 repo->base, sha1_to_hex(target->sha1));
 754
 755         filename = sha1_pack_name(target->sha1);
 756         snprintf(tmpfile, sizeof(tmpfile), "%s.temp", filename);
 757         packfile = fopen(tmpfile, "a");
 758         if (!packfile)
 759                 return error("Unable to open local file %s for pack",
 760                              filename);
 761
 762         slot = get_active_slot();
 763         slot->results = &results;
 764         curl_easy_setopt(slot->curl, CURLOPT_FILE, packfile);
 765         curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite);
 766         curl_easy_setopt(slot->curl, CURLOPT_URL, url);
 767         curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, no_pragma_header);
 768         slot->local = packfile;
 769
 770         /* If there is data present from a previous transfer attempt,
 771            resume where it left off */
 772         prev_posn = ftell(packfile);
 773         if (prev_posn>0) {
 774                 if (get_verbosely)
 775                         fprintf(stderr,
 776                                 "Resuming fetch of pack %s at byte %ld\n",
 777                                 sha1_to_hex(target->sha1), prev_posn);
 778                 sprintf(range, "Range: bytes=%ld-", prev_posn);
 779                 range_header = curl_slist_append(range_header, range);
 780                 curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, range_header);
 781         }
 782
 783         if (start_active_slot(slot)) {
 784                 run_active_slot(slot);
 785                 if (results.curl_result != CURLE_OK) {
 786                         fclose(packfile);
 787                         return error("Unable to get pack file %s\n%s", url,
 788                                      curl_errorstr);
 789                 }
 790         } else {
 791                 fclose(packfile);
 792                 return error("Unable to start request");
 793         }
 794
 795         target->pack_size = ftell(packfile);
 796         fclose(packfile);
 797
 798         ret = move_temp_to_file(tmpfile, filename);
 799         if (ret)
 800                 return ret;
 801
 802         lst = &repo->packs;
 803         while (*lst != target)
 804                 lst = &((*lst)->next);
 805         *lst = (*lst)->next;
 806
 807         if (verify_pack(target, 0))
 808                 return -1;
 809         install_packed_git(target);
 810
 811         return 0;
 812 }
 813
 814 static void abort_object_request(struct object_request *obj_req)
 815 {
 816         if (obj_req->local >= 0) {
 817                 close(obj_req->local);
 818                 obj_req->local = -1;
 819         }
 820         unlink(obj_req->tmpfile);
 821         if (obj_req->slot) {
 822                 release_active_slot(obj_req->slot);
 823                 obj_req->slot = NULL;
 824         }
 825         release_object_request(obj_req);
 826 }
 827
 828 static int fetch_object(struct alt_base *repo, unsigned char *sha1)
 829 {
 830         char *hex = sha1_to_hex(sha1);
 831         int ret = 0;
 832         struct object_request *obj_req = object_queue_head;
 833
 834         while (obj_req != NULL && hashcmp(obj_req->sha1, sha1))
 835                 obj_req = obj_req->next;
 836         if (obj_req == NULL)
 837                 return error("Couldn't find request for %s in the queue", hex);
 838
 839         if (has_sha1_file(obj_req->sha1)) {
 840                 abort_object_request(obj_req);
 841                 return 0;
 842         }
 843
 844 #ifdef USE_CURL_MULTI
 845         while (obj_req->state == WAITING) {
 846                 step_active_slots();
 847         }
 848 #else
 849         start_object_request(obj_req);
 850 #endif
 851
 852         while (obj_req->state == ACTIVE) {
 853                 run_active_slot(obj_req->slot);
 854         }
 855         if (obj_req->local != -1) {
 856                 close(obj_req->local); obj_req->local = -1;
 857         }
 858
 859         if (obj_req->state == ABORTED) {
 860                 ret = error("Request for %s aborted", hex);
 861         } else if (obj_req->curl_result != CURLE_OK &&
 862                    obj_req->http_code != 416) {
 863                 if (missing_target(obj_req))
 864                         ret = -1; /* Be silent, it is probably in a pack. */
 865                 else
 866                         ret = error("%s (curl_result = %d, http_code = %ld, sha1 = %s)",
 867                                     obj_req->errorstr, obj_req->curl_result,
 868                                     obj_req->http_code, hex);
 869         } else if (obj_req->zret != Z_STREAM_END) {
 870                 corrupt_object_found++;
 871                 ret = error("File %s (%s) corrupt", hex, obj_req->url);
 872         } else if (hashcmp(obj_req->sha1, obj_req->real_sha1)) {
 873                 ret = error("File %s has bad hash", hex);
 874         } else if (obj_req->rename < 0) {
 875                 ret = error("unable to write sha1 filename %s",
 876                             obj_req->filename);
 877         }
 878
 879         release_object_request(obj_req);
 880         return ret;
 881 }
 882
 883 int fetch(unsigned char *sha1)
 884 {
 885         struct alt_base *altbase = alt;
 886
 887         if (!fetch_object(altbase, sha1))
 888                 return 0;
 889         while (altbase) {
 890                 if (!fetch_pack(altbase, sha1))
 891                         return 0;
 892                 fetch_alternates(alt->base);
 893                 altbase = altbase->next;
 894         }
 895         return error("Unable to find %s under %s", sha1_to_hex(sha1),
 896                      alt->base);
 897 }
 898
 899 static inline int needs_quote(int ch)
 900 {
 901         if (((ch >= 'A') && (ch <= 'Z'))
 902                         || ((ch >= 'a') && (ch <= 'z'))
 903                         || ((ch >= '0') && (ch <= '9'))
 904                         || (ch == '/')
 905                         || (ch == '-')
 906                         || (ch == '.'))
 907                 return 0;
 908         return 1;
 909 }
 910
 911 static inline int hex(int v)
 912 {
 913         if (v < 10) return '0' + v;
 914         else return 'A' + v - 10;
 915 }
 916
 917 static char *quote_ref_url(const char *base, const char *ref)
 918 {
 919         const char *cp;
 920         char *dp, *qref;
 921         int len, baselen, ch;
 922
 923         baselen = strlen(base);
 924         len = baselen + 7; /* "/refs/" + NUL */
 925         for (cp = ref; (ch = *cp) != 0; cp++, len++)
 926                 if (needs_quote(ch))
 927                         len += 2; /* extra two hex plus replacement % */
 928         qref = xmalloc(len);
 929         memcpy(qref, base, baselen);
 930         memcpy(qref + baselen, "/refs/", 6);
 931         for (cp = ref, dp = qref + baselen + 6; (ch = *cp) != 0; cp++) {
 932                 if (needs_quote(ch)) {
 933                         *dp++ = '%';
 934                         *dp++ = hex((ch >> 4) & 0xF);
 935                         *dp++ = hex(ch & 0xF);
 936                 }
 937                 else
 938                         *dp++ = ch;
 939         }
 940         *dp = 0;
 941
 942         return qref;
 943 }
 944
 945 int fetch_ref(char *ref, unsigned char *sha1)
 946 {
 947         char *url;
 948         char hex[42];
 949         struct buffer buffer;
 950         const char *base = alt->base;
 951         struct active_request_slot *slot;
 952         struct slot_results results;
 953         buffer.size = 41;
 954         buffer.posn = 0;
 955         buffer.buffer = hex;
 956         hex[41] = '\0';
 957
 958         url = quote_ref_url(base, ref);
 959         slot = get_active_slot();
 960         slot->results = &results;
 961         curl_easy_setopt(slot->curl, CURLOPT_FILE, &buffer);
 962         curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite_buffer);
 963         curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, NULL);
 964         curl_easy_setopt(slot->curl, CURLOPT_URL, url);
 965         if (start_active_slot(slot)) {
 966                 run_active_slot(slot);
 967                 if (results.curl_result != CURLE_OK)
 968                         return error("Couldn't get %s for %s\n%s",
 969                                      url, ref, curl_errorstr);
 970         } else {
 971                 return error("Unable to start request");
 972         }
 973
 974         hex[40] = '\0';
 975         get_sha1_hex(hex, sha1);
 976         return 0;
 977 }
 978
 979 int main(int argc, const char **argv)
 980 {
 981         int commits;
 982         const char **write_ref = NULL;
 983         char **commit_id;
 984         const char *url;
 985         char *s;
 986         int arg = 1;
 987         int rc = 0;
 988
 989         setup_git_directory();
 990         git_config(git_default_config);
 991
 992         while (arg < argc && argv[arg][0] == '-') {
 993                 if (argv[arg][1] == 't') {
 994                         get_tree = 1;
 995                 } else if (argv[arg][1] == 'c') {
 996                         get_history = 1;
 997                 } else if (argv[arg][1] == 'a') {
 998                         get_all = 1;
 999                         get_tree = 1;
1000                         get_history = 1;
1001                 } else if (argv[arg][1] == 'v') {
1002                         get_verbosely = 1;
1003                 } else if (argv[arg][1] == 'w') {
1004                         write_ref = &argv[arg + 1];
1005                         arg++;
1006                 } else if (!strcmp(argv[arg], "--recover")) {
1007                         get_recover = 1;
1008                 } else if (!strcmp(argv[arg], "--stdin")) {
1009                         commits_on_stdin = 1;
1010                 }
1011                 arg++;
1012         }
1013         if (argc < arg + 2 - commits_on_stdin) {
1014                 usage("git-http-fetch [-c] [-t] [-a] [-v] [--recover] [-w ref] [--stdin] commit-id url");
1015                 return 1;
1016         }
1017         if (commits_on_stdin) {
1018                 commits = pull_targets_stdin(&commit_id, &write_ref);
1019         } else {
1020                 commit_id = (char **) &argv[arg++];
1021                 commits = 1;
1022         }
1023         url = argv[arg];
1024
1025         http_init();
1026
1027         no_pragma_header = curl_slist_append(no_pragma_header, "Pragma:");
1028
1029         alt = xmalloc(sizeof(*alt));
1030         alt->base = xmalloc(strlen(url) + 1);
1031         strcpy(alt->base, url);
1032         for (s = alt->base + strlen(alt->base) - 1; *s == '/'; --s)
1033                 *s = 0;
1034         alt->got_indices = 0;
1035         alt->packs = NULL;
1036         alt->next = NULL;
1037
1038 #ifdef USE_CURL_MULTI
1039         add_fill_function(NULL, fill_active_slot);
1040 #endif
1041
1042         if (pull(commits, commit_id, write_ref, url))
1043                 rc = 1;
1044
1045         http_cleanup();
1046
1047         curl_slist_free_all(no_pragma_header);
1048
1049         if (commits_on_stdin)
1050                 pull_targets_free(commits, commit_id, write_ref);
1051
1052         if (corrupt_object_found) {
1053                 fprintf(stderr,
1054 "Some loose object were found to be corrupt, but they might be just\n"
1055 "a false '404 Not Found' error message sent with incorrect HTTP\n"
1056 "status code.  Suggest running git-fsck.\n");
1057         }
1058         return rc;
1059 }