http-walker.c

   1 #include "cache.h"
   2 #include "commit.h"
   3 #include "pack.h"
   4 #include "walker.h"
   5 #include "http.h"
   6
   7 #define PREV_BUF_SIZE 4096
   8
   9 struct alt_base
  10 {
  11         char *base;
  12         int got_indices;
  13         struct packed_git *packs;
  14         struct alt_base *next;
  15 };
  16
  17 enum object_request_state {
  18         WAITING,
  19         ABORTED,
  20         ACTIVE,
  21         COMPLETE,
  22 };
  23
  24 struct object_request
  25 {
  26         struct walker *walker;
  27         unsigned char sha1[20];
  28         struct alt_base *repo;
  29         char *url;
  30         char filename[PATH_MAX];
  31         char tmpfile[PATH_MAX];
  32         int local;
  33         enum object_request_state state;
  34         CURLcode curl_result;
  35         char errorstr[CURL_ERROR_SIZE];
  36         long http_code;
  37         unsigned char real_sha1[20];
  38         git_SHA_CTX c;
  39         z_stream stream;
  40         int zret;
  41         int rename;
  42         struct active_request_slot *slot;
  43         struct object_request *next;
  44 };
  45
  46 struct alternates_request {
  47         struct walker *walker;
  48         const char *base;
  49         char *url;
  50         struct strbuf *buffer;
  51         struct active_request_slot *slot;
  52         int http_specific;
  53 };
  54
  55 struct walker_data {
  56         const char *url;
  57         int got_alternates;
  58         struct alt_base *alt;
  59 };
  60
  61 static struct object_request *object_queue_head;
  62
  63 static size_t fwrite_sha1_file(void *ptr, size_t eltsize, size_t nmemb,
  64                                void *data)
  65 {
  66         unsigned char expn[4096];
  67         size_t size = eltsize * nmemb;
  68         int posn = 0;
  69         struct object_request *obj_req = (struct object_request *)data;
  70         do {
  71                 ssize_t retval = xwrite(obj_req->local,
  72                                         (char *) ptr + posn, size - posn);
  73                 if (retval < 0)
  74                         return posn;
  75                 posn += retval;
  76         } while (posn < size);
  77
  78         obj_req->stream.avail_in = size;
  79         obj_req->stream.next_in = ptr;
  80         do {
  81                 obj_req->stream.next_out = expn;
  82                 obj_req->stream.avail_out = sizeof(expn);
  83                 obj_req->zret = git_inflate(&obj_req->stream, Z_SYNC_FLUSH);
  84                 git_SHA1_Update(&obj_req->c, expn,
  85                                 sizeof(expn) - obj_req->stream.avail_out);
  86         } while (obj_req->stream.avail_in && obj_req->zret == Z_OK);
  87         data_received++;
  88         return size;
  89 }
  90
  91 static void fetch_alternates(struct walker *walker, const char *base);
  92
  93 static void process_object_response(void *callback_data);
  94
  95 static void start_object_request(struct walker *walker,
  96                                  struct object_request *obj_req)
  97 {
  98         char *hex = sha1_to_hex(obj_req->sha1);
  99         char prevfile[PATH_MAX];
 100         char *url;
 101         char *posn;
 102         int prevlocal;
 103         unsigned char prev_buf[PREV_BUF_SIZE];
 104         ssize_t prev_read = 0;
 105         long prev_posn = 0;
 106         char range[RANGE_HEADER_SIZE];
 107         struct curl_slist *range_header = NULL;
 108         struct active_request_slot *slot;
 109
 110         snprintf(prevfile, sizeof(prevfile), "%s.prev", obj_req->filename);
 111         unlink_or_warn(prevfile);
 112         rename(obj_req->tmpfile, prevfile);
 113         unlink_or_warn(obj_req->tmpfile);
 114
 115         if (obj_req->local != -1)
 116                 error("fd leakage in start: %d", obj_req->local);
 117         obj_req->local = open(obj_req->tmpfile,
 118                               O_WRONLY | O_CREAT | O_EXCL, 0666);
 119         /*
 120          * This could have failed due to the "lazy directory creation";
 121          * try to mkdir the last path component.
 122          */
 123         if (obj_req->local < 0 && errno == ENOENT) {
 124                 char *dir = strrchr(obj_req->tmpfile, '/');
 125                 if (dir) {
 126                         *dir = 0;
 127                         mkdir(obj_req->tmpfile, 0777);
 128                         *dir = '/';
 129                 }
 130                 obj_req->local = open(obj_req->tmpfile,
 131                                       O_WRONLY | O_CREAT | O_EXCL, 0666);
 132         }
 133
 134         if (obj_req->local < 0) {
 135                 obj_req->state = ABORTED;
 136                 error("Couldn't create temporary file %s for %s: %s",
 137                       obj_req->tmpfile, obj_req->filename, strerror(errno));
 138                 return;
 139         }
 140
 141         memset(&obj_req->stream, 0, sizeof(obj_req->stream));
 142
 143         git_inflate_init(&obj_req->stream);
 144
 145         git_SHA1_Init(&obj_req->c);
 146
 147         url = xmalloc(strlen(obj_req->repo->base) + 51);
 148         obj_req->url = xmalloc(strlen(obj_req->repo->base) + 51);
 149         strcpy(url, obj_req->repo->base);
 150         posn = url + strlen(obj_req->repo->base);
 151         strcpy(posn, "/objects/");
 152         posn += 9;
 153         memcpy(posn, hex, 2);
 154         posn += 2;
 155         *(posn++) = '/';
 156         strcpy(posn, hex + 2);
 157         strcpy(obj_req->url, url);
 158
 159         /*
 160          * If a previous temp file is present, process what was already
 161          * fetched.
 162          */
 163         prevlocal = open(prevfile, O_RDONLY);
 164         if (prevlocal != -1) {
 165                 do {
 166                         prev_read = xread(prevlocal, prev_buf, PREV_BUF_SIZE);
 167                         if (prev_read>0) {
 168                                 if (fwrite_sha1_file(prev_buf,
 169                                                      1,
 170                                                      prev_read,
 171                                                      obj_req) == prev_read)
 172                                         prev_posn += prev_read;
 173                                 else
 174                                         prev_read = -1;
 175                         }
 176                 } while (prev_read > 0);
 177                 close(prevlocal);
 178         }
 179         unlink_or_warn(prevfile);
 180
 181         /*
 182          * Reset inflate/SHA1 if there was an error reading the previous temp
 183          * file; also rewind to the beginning of the local file.
 184          */
 185         if (prev_read == -1) {
 186                 memset(&obj_req->stream, 0, sizeof(obj_req->stream));
 187                 git_inflate_init(&obj_req->stream);
 188                 git_SHA1_Init(&obj_req->c);
 189                 if (prev_posn>0) {
 190                         prev_posn = 0;
 191                         lseek(obj_req->local, 0, SEEK_SET);
 192                         ftruncate(obj_req->local, 0);
 193                 }
 194         }
 195
 196         slot = get_active_slot();
 197         slot->callback_func = process_object_response;
 198         slot->callback_data = obj_req;
 199         obj_req->slot = slot;
 200
 201         curl_easy_setopt(slot->curl, CURLOPT_FILE, obj_req);
 202         curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite_sha1_file);
 203         curl_easy_setopt(slot->curl, CURLOPT_ERRORBUFFER, obj_req->errorstr);
 204         curl_easy_setopt(slot->curl, CURLOPT_URL, url);
 205         curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, no_pragma_header);
 206
 207         /*
 208          * If we have successfully processed data from a previous fetch
 209          * attempt, only fetch the data we don't already have.
 210          */
 211         if (prev_posn>0) {
 212                 if (walker->get_verbosely)
 213                         fprintf(stderr,
 214                                 "Resuming fetch of object %s at byte %ld\n",
 215                                 hex, prev_posn);
 216                 sprintf(range, "Range: bytes=%ld-", prev_posn);
 217                 range_header = curl_slist_append(range_header, range);
 218                 curl_easy_setopt(slot->curl,
 219                                  CURLOPT_HTTPHEADER, range_header);
 220         }
 221
 222         /* Try to get the request started, abort the request on error */
 223         obj_req->state = ACTIVE;
 224         if (!start_active_slot(slot)) {
 225                 obj_req->state = ABORTED;
 226                 obj_req->slot = NULL;
 227                 close(obj_req->local);
 228                 obj_req->local = -1;
 229                 free(obj_req->url);
 230                 return;
 231         }
 232 }
 233
 234 static void finish_object_request(struct object_request *obj_req)
 235 {
 236         struct stat st;
 237
 238         close(obj_req->local);
 239         obj_req->local = -1;
 240
 241         if (obj_req->http_code == 416) {
 242                 fprintf(stderr, "Warning: requested range invalid; we may already have all the data.\n");
 243         } else if (obj_req->curl_result != CURLE_OK) {
 244                 if (stat(obj_req->tmpfile, &st) == 0)
 245                         if (st.st_size == 0)
 246                                 unlink_or_warn(obj_req->tmpfile);
 247                 return;
 248         }
 249
 250         git_inflate_end(&obj_req->stream);
 251         git_SHA1_Final(obj_req->real_sha1, &obj_req->c);
 252         if (obj_req->zret != Z_STREAM_END) {
 253                 unlink_or_warn(obj_req->tmpfile);
 254                 return;
 255         }
 256         if (hashcmp(obj_req->sha1, obj_req->real_sha1)) {
 257                 unlink_or_warn(obj_req->tmpfile);
 258                 return;
 259         }
 260         obj_req->rename =
 261                 move_temp_to_file(obj_req->tmpfile, obj_req->filename);
 262
 263         if (obj_req->rename == 0)
 264                 walker_say(obj_req->walker, "got %s\n", sha1_to_hex(obj_req->sha1));
 265 }
 266
 267 static void process_object_response(void *callback_data)
 268 {
 269         struct object_request *obj_req =
 270                 (struct object_request *)callback_data;
 271         struct walker *walker = obj_req->walker;
 272         struct walker_data *data = walker->data;
 273         struct alt_base *alt = data->alt;
 274
 275         obj_req->curl_result = obj_req->slot->curl_result;
 276         obj_req->http_code = obj_req->slot->http_code;
 277         obj_req->slot = NULL;
 278         obj_req->state = COMPLETE;
 279
 280         /* Use alternates if necessary */
 281         if (missing_target(obj_req)) {
 282                 fetch_alternates(walker, alt->base);
 283                 if (obj_req->repo->next != NULL) {
 284                         obj_req->repo =
 285                                 obj_req->repo->next;
 286                         close(obj_req->local);
 287                         obj_req->local = -1;
 288                         start_object_request(walker, obj_req);
 289                         return;
 290                 }
 291         }
 292
 293         finish_object_request(obj_req);
 294 }
 295
 296 static void release_object_request(struct object_request *obj_req)
 297 {
 298         struct object_request *entry = object_queue_head;
 299
 300         if (obj_req->local != -1)
 301                 error("fd leakage in release: %d", obj_req->local);
 302         if (obj_req == object_queue_head) {
 303                 object_queue_head = obj_req->next;
 304         } else {
 305                 while (entry->next != NULL && entry->next != obj_req)
 306                         entry = entry->next;
 307                 if (entry->next == obj_req)
 308                         entry->next = entry->next->next;
 309         }
 310
 311         free(obj_req->url);
 312         free(obj_req);
 313 }
 314
 315 #ifdef USE_CURL_MULTI
 316 static int fill_active_slot(struct walker *walker)
 317 {
 318         struct object_request *obj_req;
 319
 320         for (obj_req = object_queue_head; obj_req; obj_req = obj_req->next) {
 321                 if (obj_req->state == WAITING) {
 322                         if (has_sha1_file(obj_req->sha1))
 323                                 obj_req->state = COMPLETE;
 324                         else {
 325                                 start_object_request(walker, obj_req);
 326                                 return 1;
 327                         }
 328                 }
 329         }
 330         return 0;
 331 }
 332 #endif
 333
 334 static void prefetch(struct walker *walker, unsigned char *sha1)
 335 {
 336         struct object_request *newreq;
 337         struct object_request *tail;
 338         struct walker_data *data = walker->data;
 339         char *filename = sha1_file_name(sha1);
 340
 341         newreq = xmalloc(sizeof(*newreq));
 342         newreq->walker = walker;
 343         hashcpy(newreq->sha1, sha1);
 344         newreq->repo = data->alt;
 345         newreq->url = NULL;
 346         newreq->local = -1;
 347         newreq->state = WAITING;
 348         snprintf(newreq->filename, sizeof(newreq->filename), "%s", filename);
 349         snprintf(newreq->tmpfile, sizeof(newreq->tmpfile),
 350                  "%s.temp", filename);
 351         newreq->slot = NULL;
 352         newreq->next = NULL;
 353
 354         http_is_verbose = walker->get_verbosely;
 355
 356         if (object_queue_head == NULL) {
 357                 object_queue_head = newreq;
 358         } else {
 359                 tail = object_queue_head;
 360                 while (tail->next != NULL)
 361                         tail = tail->next;
 362                 tail->next = newreq;
 363         }
 364
 365 #ifdef USE_CURL_MULTI
 366         fill_active_slots();
 367         step_active_slots();
 368 #endif
 369 }
 370
 371 static int fetch_index(struct walker *walker, struct alt_base *repo, unsigned char *sha1)
 372 {
 373         int ret = 0;
 374         char *hex = xstrdup(sha1_to_hex(sha1));
 375         char *filename;
 376         char *url;
 377         char tmpfile[PATH_MAX];
 378         long prev_posn = 0;
 379         char range[RANGE_HEADER_SIZE];
 380         struct curl_slist *range_header = NULL;
 381
 382         FILE *indexfile;
 383         struct active_request_slot *slot;
 384         struct slot_results results;
 385
 386         /* Don't use the index if the pack isn't there */
 387         url = xmalloc(strlen(repo->base) + 64);
 388         sprintf(url, "%s/objects/pack/pack-%s.pack", repo->base, hex);
 389         slot = get_active_slot();
 390         slot->results = &results;
 391         curl_easy_setopt(slot->curl, CURLOPT_URL, url);
 392         curl_easy_setopt(slot->curl, CURLOPT_NOBODY, 1);
 393         if (start_active_slot(slot)) {
 394                 run_active_slot(slot);
 395                 if (results.curl_result != CURLE_OK) {
 396                         ret = error("Unable to verify pack %s is available",
 397                                      hex);
 398                         goto cleanup_pack;
 399                 }
 400         } else {
 401                 ret = error("Unable to start request");
 402                 goto cleanup_pack;
 403         }
 404
 405         if (has_pack_index(sha1)) {
 406                 ret = 0;
 407                 goto cleanup_pack;
 408         }
 409
 410         if (walker->get_verbosely)
 411                 fprintf(stderr, "Getting index for pack %s\n", hex);
 412
 413         sprintf(url, "%s/objects/pack/pack-%s.idx", repo->base, hex);
 414
 415         filename = sha1_pack_index_name(sha1);
 416         snprintf(tmpfile, sizeof(tmpfile), "%s.temp", filename);
 417         indexfile = fopen(tmpfile, "a");
 418         if (!indexfile) {
 419                 ret = error("Unable to open local file %s for pack index",
 420                             tmpfile);
 421                 goto cleanup_pack;
 422         }
 423
 424         slot = get_active_slot();
 425         slot->results = &results;
 426         curl_easy_setopt(slot->curl, CURLOPT_NOBODY, 0);
 427         curl_easy_setopt(slot->curl, CURLOPT_HTTPGET, 1);
 428         curl_easy_setopt(slot->curl, CURLOPT_FILE, indexfile);
 429         curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite);
 430         curl_easy_setopt(slot->curl, CURLOPT_URL, url);
 431         curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, no_pragma_header);
 432         slot->local = indexfile;
 433
 434         /*
 435          * If there is data present from a previous transfer attempt,
 436          * resume where it left off
 437          */
 438         prev_posn = ftell(indexfile);
 439         if (prev_posn>0) {
 440                 if (walker->get_verbosely)
 441                         fprintf(stderr,
 442                                 "Resuming fetch of index for pack %s at byte %ld\n",
 443                                 hex, prev_posn);
 444                 sprintf(range, "Range: bytes=%ld-", prev_posn);
 445                 range_header = curl_slist_append(range_header, range);
 446                 curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, range_header);
 447         }
 448
 449         if (start_active_slot(slot)) {
 450                 run_active_slot(slot);
 451                 if (results.curl_result != CURLE_OK) {
 452                         ret = error("Unable to get pack index %s\n%s", url,
 453                                     curl_errorstr);
 454                         goto cleanup_index;
 455                 }
 456         } else {
 457                 ret = error("Unable to start request");
 458                 goto cleanup_index;
 459         }
 460
 461         ret = move_temp_to_file(tmpfile, filename);
 462
 463 cleanup_index:
 464         fclose(indexfile);
 465         slot->local = NULL;
 466 cleanup_pack:
 467         free(url);
 468         free(hex);
 469         return ret;
 470 }
 471
 472 static int setup_index(struct walker *walker, struct alt_base *repo, unsigned char *sha1)
 473 {
 474         struct packed_git *new_pack;
 475         if (has_pack_file(sha1))
 476                 return 0; /* don't list this as something we can get */
 477
 478         if (fetch_index(walker, repo, sha1))
 479                 return -1;
 480
 481         new_pack = parse_pack_index(sha1);
 482         if (!new_pack)
 483                 return -1; /* parse_pack_index() already issued error message */
 484         new_pack->next = repo->packs;
 485         repo->packs = new_pack;
 486         return 0;
 487 }
 488
 489 static void process_alternates_response(void *callback_data)
 490 {
 491         struct alternates_request *alt_req =
 492                 (struct alternates_request *)callback_data;
 493         struct walker *walker = alt_req->walker;
 494         struct walker_data *cdata = walker->data;
 495         struct active_request_slot *slot = alt_req->slot;
 496         struct alt_base *tail = cdata->alt;
 497         const char *base = alt_req->base;
 498         static const char null_byte = '\0';
 499         char *data;
 500         int i = 0;
 501
 502         if (alt_req->http_specific) {
 503                 if (slot->curl_result != CURLE_OK ||
 504                     !alt_req->buffer->len) {
 505
 506                         /* Try reusing the slot to get non-http alternates */
 507                         alt_req->http_specific = 0;
 508                         sprintf(alt_req->url, "%s/objects/info/alternates",
 509                                 base);
 510                         curl_easy_setopt(slot->curl, CURLOPT_URL,
 511                                          alt_req->url);
 512                         active_requests++;
 513                         slot->in_use = 1;
 514                         if (slot->finished != NULL)
 515                                 (*slot->finished) = 0;
 516                         if (!start_active_slot(slot)) {
 517                                 cdata->got_alternates = -1;
 518                                 slot->in_use = 0;
 519                                 if (slot->finished != NULL)
 520                                         (*slot->finished) = 1;
 521                         }
 522                         return;
 523                 }
 524         } else if (slot->curl_result != CURLE_OK) {
 525                 if (!missing_target(slot)) {
 526                         cdata->got_alternates = -1;
 527                         return;
 528                 }
 529         }
 530
 531         fwrite_buffer(&null_byte, 1, 1, alt_req->buffer);
 532         alt_req->buffer->len--;
 533         data = alt_req->buffer->buf;
 534
 535         while (i < alt_req->buffer->len) {
 536                 int posn = i;
 537                 while (posn < alt_req->buffer->len && data[posn] != '\n')
 538                         posn++;
 539                 if (data[posn] == '\n') {
 540                         int okay = 0;
 541                         int serverlen = 0;
 542                         struct alt_base *newalt;
 543                         char *target = NULL;
 544                         if (data[i] == '/') {
 545                                 /*
 546                                  * This counts
 547                                  * http://git.host/pub/scm/linux.git/
 548                                  * -----------here^
 549                                  * so memcpy(dst, base, serverlen) will
 550                                  * copy up to "...git.host".
 551                                  */
 552                                 const char *colon_ss = strstr(base,"://");
 553                                 if (colon_ss) {
 554                                         serverlen = (strchr(colon_ss + 3, '/')
 555                                                      - base);
 556                                         okay = 1;
 557                                 }
 558                         } else if (!memcmp(data + i, "../", 3)) {
 559                                 /*
 560                                  * Relative URL; chop the corresponding
 561                                  * number of subpath from base (and ../
 562                                  * from data), and concatenate the result.
 563                                  *
 564                                  * The code first drops ../ from data, and
 565                                  * then drops one ../ from data and one path
 566                                  * from base.  IOW, one extra ../ is dropped
 567                                  * from data than path is dropped from base.
 568                                  *
 569                                  * This is not wrong.  The alternate in
 570                                  *     http://git.host/pub/scm/linux.git/
 571                                  * to borrow from
 572                                  *     http://git.host/pub/scm/linus.git/
 573                                  * is ../../linus.git/objects/.  You need
 574                                  * two ../../ to borrow from your direct
 575                                  * neighbour.
 576                                  */
 577                                 i += 3;
 578                                 serverlen = strlen(base);
 579                                 while (i + 2 < posn &&
 580                                        !memcmp(data + i, "../", 3)) {
 581                                         do {
 582                                                 serverlen--;
 583                                         } while (serverlen &&
 584                                                  base[serverlen - 1] != '/');
 585                                         i += 3;
 586                                 }
 587                                 /* If the server got removed, give up. */
 588                                 okay = strchr(base, ':') - base + 3 <
 589                                        serverlen;
 590                         } else if (alt_req->http_specific) {
 591                                 char *colon = strchr(data + i, ':');
 592                                 char *slash = strchr(data + i, '/');
 593                                 if (colon && slash && colon < data + posn &&
 594                                     slash < data + posn && colon < slash) {
 595                                         okay = 1;
 596                                 }
 597                         }
 598                         /* skip "objects\n" at end */
 599                         if (okay) {
 600                                 target = xmalloc(serverlen + posn - i - 6);
 601                                 memcpy(target, base, serverlen);
 602                                 memcpy(target + serverlen, data + i,
 603                                        posn - i - 7);
 604                                 target[serverlen + posn - i - 7] = 0;
 605                                 if (walker->get_verbosely)
 606                                         fprintf(stderr,
 607                                                 "Also look at %s\n", target);
 608                                 newalt = xmalloc(sizeof(*newalt));
 609                                 newalt->next = NULL;
 610                                 newalt->base = target;
 611                                 newalt->got_indices = 0;
 612                                 newalt->packs = NULL;
 613
 614                                 while (tail->next != NULL)
 615                                         tail = tail->next;
 616                                 tail->next = newalt;
 617                         }
 618                 }
 619                 i = posn + 1;
 620         }
 621
 622         cdata->got_alternates = 1;
 623 }
 624
 625 static void fetch_alternates(struct walker *walker, const char *base)
 626 {
 627         struct strbuf buffer = STRBUF_INIT;
 628         char *url;
 629         struct active_request_slot *slot;
 630         struct alternates_request alt_req;
 631         struct walker_data *cdata = walker->data;
 632
 633         /*
 634          * If another request has already started fetching alternates,
 635          * wait for them to arrive and return to processing this request's
 636          * curl message
 637          */
 638 #ifdef USE_CURL_MULTI
 639         while (cdata->got_alternates == 0) {
 640                 step_active_slots();
 641         }
 642 #endif
 643
 644         /* Nothing to do if they've already been fetched */
 645         if (cdata->got_alternates == 1)
 646                 return;
 647
 648         /* Start the fetch */
 649         cdata->got_alternates = 0;
 650
 651         if (walker->get_verbosely)
 652                 fprintf(stderr, "Getting alternates list for %s\n", base);
 653
 654         url = xmalloc(strlen(base) + 31);
 655         sprintf(url, "%s/objects/info/http-alternates", base);
 656
 657         /*
 658          * Use a callback to process the result, since another request
 659          * may fail and need to have alternates loaded before continuing
 660          */
 661         slot = get_active_slot();
 662         slot->callback_func = process_alternates_response;
 663         alt_req.walker = walker;
 664         slot->callback_data = &alt_req;
 665
 666         curl_easy_setopt(slot->curl, CURLOPT_FILE, &buffer);
 667         curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite_buffer);
 668         curl_easy_setopt(slot->curl, CURLOPT_URL, url);
 669
 670         alt_req.base = base;
 671         alt_req.url = url;
 672         alt_req.buffer = &buffer;
 673         alt_req.http_specific = 1;
 674         alt_req.slot = slot;
 675
 676         if (start_active_slot(slot))
 677                 run_active_slot(slot);
 678         else
 679                 cdata->got_alternates = -1;
 680
 681         strbuf_release(&buffer);
 682         free(url);
 683 }
 684
 685 static int fetch_indices(struct walker *walker, struct alt_base *repo)
 686 {
 687         unsigned char sha1[20];
 688         char *url;
 689         struct strbuf buffer = STRBUF_INIT;
 690         char *data;
 691         int i = 0;
 692         int ret = 0;
 693
 694         struct active_request_slot *slot;
 695         struct slot_results results;
 696
 697         if (repo->got_indices)
 698                 return 0;
 699
 700         if (walker->get_verbosely)
 701                 fprintf(stderr, "Getting pack list for %s\n", repo->base);
 702
 703         url = xmalloc(strlen(repo->base) + 21);
 704         sprintf(url, "%s/objects/info/packs", repo->base);
 705
 706         slot = get_active_slot();
 707         slot->results = &results;
 708         curl_easy_setopt(slot->curl, CURLOPT_FILE, &buffer);
 709         curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite_buffer);
 710         curl_easy_setopt(slot->curl, CURLOPT_URL, url);
 711         curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, NULL);
 712         if (start_active_slot(slot)) {
 713                 run_active_slot(slot);
 714                 if (results.curl_result != CURLE_OK) {
 715                         if (missing_target(&results)) {
 716                                 repo->got_indices = 1;
 717                                 goto cleanup;
 718                         } else {
 719                                 repo->got_indices = 0;
 720                                 ret = error("%s", curl_errorstr);
 721                                 goto cleanup;
 722                         }
 723                 }
 724         } else {
 725                 repo->got_indices = 0;
 726                 ret = error("Unable to start request");
 727                 goto cleanup;
 728         }
 729
 730         data = buffer.buf;
 731         while (i < buffer.len) {
 732                 switch (data[i]) {
 733                 case 'P':
 734                         i++;
 735                         if (i + 52 <= buffer.len &&
 736                             !prefixcmp(data + i, " pack-") &&
 737                             !prefixcmp(data + i + 46, ".pack\n")) {
 738                                 get_sha1_hex(data + i + 6, sha1);
 739                                 setup_index(walker, repo, sha1);
 740                                 i += 51;
 741                                 break;
 742                         }
 743                 default:
 744                         while (i < buffer.len && data[i] != '\n')
 745                                 i++;
 746                 }
 747                 i++;
 748         }
 749
 750         repo->got_indices = 1;
 751 cleanup:
 752         strbuf_release(&buffer);
 753         free(url);
 754         return ret;
 755 }
 756
 757 static int fetch_pack(struct walker *walker, struct alt_base *repo, unsigned char *sha1)
 758 {
 759         char *url;
 760         struct packed_git *target;
 761         struct packed_git **lst;
 762         FILE *packfile;
 763         char *filename;
 764         char tmpfile[PATH_MAX];
 765         int ret;
 766         long prev_posn = 0;
 767         char range[RANGE_HEADER_SIZE];
 768         struct curl_slist *range_header = NULL;
 769
 770         struct active_request_slot *slot;
 771         struct slot_results results;
 772
 773         if (fetch_indices(walker, repo))
 774                 return -1;
 775         target = find_sha1_pack(sha1, repo->packs);
 776         if (!target)
 777                 return -1;
 778
 779         if (walker->get_verbosely) {
 780                 fprintf(stderr, "Getting pack %s\n",
 781                         sha1_to_hex(target->sha1));
 782                 fprintf(stderr, " which contains %s\n",
 783                         sha1_to_hex(sha1));
 784         }
 785
 786         url = xmalloc(strlen(repo->base) + 65);
 787         sprintf(url, "%s/objects/pack/pack-%s.pack",
 788                 repo->base, sha1_to_hex(target->sha1));
 789
 790         filename = sha1_pack_name(target->sha1);
 791         snprintf(tmpfile, sizeof(tmpfile), "%s.temp", filename);
 792         packfile = fopen(tmpfile, "a");
 793         if (!packfile)
 794                 return error("Unable to open local file %s for pack",
 795                              tmpfile);
 796
 797         slot = get_active_slot();
 798         slot->results = &results;
 799         curl_easy_setopt(slot->curl, CURLOPT_FILE, packfile);
 800         curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite);
 801         curl_easy_setopt(slot->curl, CURLOPT_URL, url);
 802         curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, no_pragma_header);
 803         slot->local = packfile;
 804
 805         /*
 806          * If there is data present from a previous transfer attempt,
 807          * resume where it left off
 808          */
 809         prev_posn = ftell(packfile);
 810         if (prev_posn>0) {
 811                 if (walker->get_verbosely)
 812                         fprintf(stderr,
 813                                 "Resuming fetch of pack %s at byte %ld\n",
 814                                 sha1_to_hex(target->sha1), prev_posn);
 815                 sprintf(range, "Range: bytes=%ld-", prev_posn);
 816                 range_header = curl_slist_append(range_header, range);
 817                 curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, range_header);
 818         }
 819
 820         if (start_active_slot(slot)) {
 821                 run_active_slot(slot);
 822                 if (results.curl_result != CURLE_OK) {
 823                         fclose(packfile);
 824                         slot->local = NULL;
 825                         return error("Unable to get pack file %s\n%s", url,
 826                                      curl_errorstr);
 827                 }
 828         } else {
 829                 fclose(packfile);
 830                 slot->local = NULL;
 831                 return error("Unable to start request");
 832         }
 833
 834         target->pack_size = ftell(packfile);
 835         fclose(packfile);
 836         slot->local = NULL;
 837
 838         ret = move_temp_to_file(tmpfile, filename);
 839         if (ret)
 840                 return ret;
 841
 842         lst = &repo->packs;
 843         while (*lst != target)
 844                 lst = &((*lst)->next);
 845         *lst = (*lst)->next;
 846
 847         if (verify_pack(target))
 848                 return -1;
 849         install_packed_git(target);
 850
 851         return 0;
 852 }
 853
 854 static void abort_object_request(struct object_request *obj_req)
 855 {
 856         if (obj_req->local >= 0) {
 857                 close(obj_req->local);
 858                 obj_req->local = -1;
 859         }
 860         unlink_or_warn(obj_req->tmpfile);
 861         if (obj_req->slot) {
 862                 release_active_slot(obj_req->slot);
 863                 obj_req->slot = NULL;
 864         }
 865         release_object_request(obj_req);
 866 }
 867
 868 static int fetch_object(struct walker *walker, struct alt_base *repo, unsigned char *sha1)
 869 {
 870         char *hex = sha1_to_hex(sha1);
 871         int ret = 0;
 872         struct object_request *obj_req = object_queue_head;
 873
 874         while (obj_req != NULL && hashcmp(obj_req->sha1, sha1))
 875                 obj_req = obj_req->next;
 876         if (obj_req == NULL)
 877                 return error("Couldn't find request for %s in the queue", hex);
 878
 879         if (has_sha1_file(obj_req->sha1)) {
 880                 abort_object_request(obj_req);
 881                 return 0;
 882         }
 883
 884 #ifdef USE_CURL_MULTI
 885         while (obj_req->state == WAITING)
 886                 step_active_slots();
 887 #else
 888         start_object_request(walker, obj_req);
 889 #endif
 890
 891         while (obj_req->state == ACTIVE)
 892                 run_active_slot(obj_req->slot);
 893
 894         if (obj_req->local != -1) {
 895                 close(obj_req->local);
 896                 obj_req->local = -1;
 897         }
 898
 899         if (obj_req->state == ABORTED) {
 900                 ret = error("Request for %s aborted", hex);
 901         } else if (obj_req->curl_result != CURLE_OK &&
 902                    obj_req->http_code != 416) {
 903                 if (missing_target(obj_req))
 904                         ret = -1; /* Be silent, it is probably in a pack. */
 905                 else
 906                         ret = error("%s (curl_result = %d, http_code = %ld, sha1 = %s)",
 907                                     obj_req->errorstr, obj_req->curl_result,
 908                                     obj_req->http_code, hex);
 909         } else if (obj_req->zret != Z_STREAM_END) {
 910                 walker->corrupt_object_found++;
 911                 ret = error("File %s (%s) corrupt", hex, obj_req->url);
 912         } else if (hashcmp(obj_req->sha1, obj_req->real_sha1)) {
 913                 ret = error("File %s has bad hash", hex);
 914         } else if (obj_req->rename < 0) {
 915                 ret = error("unable to write sha1 filename %s",
 916                             obj_req->filename);
 917         }
 918
 919         release_object_request(obj_req);
 920         return ret;
 921 }
 922
 923 static int fetch(struct walker *walker, unsigned char *sha1)
 924 {
 925         struct walker_data *data = walker->data;
 926         struct alt_base *altbase = data->alt;
 927
 928         if (!fetch_object(walker, altbase, sha1))
 929                 return 0;
 930         while (altbase) {
 931                 if (!fetch_pack(walker, altbase, sha1))
 932                         return 0;
 933                 fetch_alternates(walker, data->alt->base);
 934                 altbase = altbase->next;
 935         }
 936         return error("Unable to find %s under %s", sha1_to_hex(sha1),
 937                      data->alt->base);
 938 }
 939
 940 static int fetch_ref(struct walker *walker, struct ref *ref)
 941 {
 942         struct walker_data *data = walker->data;
 943         return http_fetch_ref(data->alt->base, ref);
 944 }
 945
 946 static void cleanup(struct walker *walker)
 947 {
 948         http_cleanup();
 949 }
 950
 951 struct walker *get_http_walker(const char *url, struct remote *remote)
 952 {
 953         char *s;
 954         struct walker_data *data = xmalloc(sizeof(struct walker_data));
 955         struct walker *walker = xmalloc(sizeof(struct walker));
 956
 957         http_init(remote);
 958
 959         data->alt = xmalloc(sizeof(*data->alt));
 960         data->alt->base = xmalloc(strlen(url) + 1);
 961         strcpy(data->alt->base, url);
 962         for (s = data->alt->base + strlen(data->alt->base) - 1; *s == '/'; --s)
 963                 *s = 0;
 964
 965         data->alt->got_indices = 0;
 966         data->alt->packs = NULL;
 967         data->alt->next = NULL;
 968         data->got_alternates = -1;
 969
 970         walker->corrupt_object_found = 0;
 971         walker->fetch = fetch;
 972         walker->fetch_ref = fetch_ref;
 973         walker->prefetch = prefetch;
 974         walker->cleanup = cleanup;
 975         walker->data = data;
 976
 977 #ifdef USE_CURL_MULTI
 978         add_fill_function(walker, (int (*)(void *)) fill_active_slot);
 979 #endif
 980
 981         return walker;
 982 }