http-fetch.c

   1 #include "cache.h"
   2 #include "commit.h"
   3 #include "pack.h"
   4 #include "fetch.h"
   5
   6 #include <curl/curl.h>
   7 #include <curl/easy.h>
   8
   9 #if LIBCURL_VERSION_NUM >= 0x070908
  10 #define USE_CURL_MULTI
  11 #define DEFAULT_MAX_REQUESTS 5
  12 #endif
  13
  14 #if LIBCURL_VERSION_NUM < 0x070704
  15 #define curl_global_cleanup() do { /* nothing */ } while(0)
  16 #endif
  17 #if LIBCURL_VERSION_NUM < 0x070800
  18 #define curl_global_init(a) do { /* nothing */ } while(0)
  19 #endif
  20
  21 #if LIBCURL_VERSION_NUM < 0x070c04
  22 #define NO_CURL_EASY_DUPHANDLE
  23 #endif
  24
  25 #define PREV_BUF_SIZE 4096
  26 #define RANGE_HEADER_SIZE 30
  27
  28 static int active_requests = 0;
  29 static int data_received;
  30
  31 #ifdef USE_CURL_MULTI
  32 static int max_requests = -1;
  33 static CURLM *curlm;
  34 #endif
  35 #ifndef NO_CURL_EASY_DUPHANDLE
  36 static CURL *curl_default;
  37 #endif
  38 static struct curl_slist *pragma_header;
  39 static struct curl_slist *no_pragma_header;
  40 static struct curl_slist *no_range_header;
  41 static char curl_errorstr[CURL_ERROR_SIZE];
  42
  43 struct alt_base
  44 {
  45         char *base;
  46         int got_indices;
  47         struct packed_git *packs;
  48         struct alt_base *next;
  49 };
  50
  51 static struct alt_base *alt = NULL;
  52
  53 enum transfer_state {
  54         WAITING,
  55         ABORTED,
  56         ACTIVE,
  57         COMPLETE,
  58 };
  59
  60 struct transfer_request
  61 {
  62         unsigned char sha1[20];
  63         struct alt_base *repo;
  64         char *url;
  65         char filename[PATH_MAX];
  66         char tmpfile[PATH_MAX];
  67         int local;
  68         enum transfer_state state;
  69         CURLcode curl_result;
  70         char errorstr[CURL_ERROR_SIZE];
  71         long http_code;
  72         unsigned char real_sha1[20];
  73         SHA_CTX c;
  74         z_stream stream;
  75         int zret;
  76         int rename;
  77         struct active_request_slot *slot;
  78         struct transfer_request *next;
  79 };
  80
  81 struct active_request_slot
  82 {
  83         CURL *curl;
  84         FILE *local;
  85         int in_use;
  86         int done;
  87         CURLcode curl_result;
  88         struct active_request_slot *next;
  89 };
  90
  91 static struct transfer_request *request_queue_head = NULL;
  92 static struct active_request_slot *active_queue_head = NULL;
  93
  94 static int curl_ssl_verify = -1;
  95 static char *ssl_cert = NULL;
  96 #if LIBCURL_VERSION_NUM >= 0x070902
  97 static char *ssl_key = NULL;
  98 #endif
  99 #if LIBCURL_VERSION_NUM >= 0x070908
 100 static char *ssl_capath = NULL;
 101 #endif
 102 static char *ssl_cainfo = NULL;
 103
 104 struct buffer
 105 {
 106         size_t posn;
 107         size_t size;
 108         void *buffer;
 109 };
 110
 111 static int http_options(const char *var, const char *value)
 112 {
 113         if (!strcmp("http.sslverify", var)) {
 114                 if (curl_ssl_verify == -1) {
 115                         curl_ssl_verify = git_config_bool(var, value);
 116                 }
 117                 return 0;
 118         }
 119
 120         if (!strcmp("http.sslcert", var)) {
 121                 if (ssl_cert == NULL) {
 122                         ssl_cert = xmalloc(strlen(value)+1);
 123                         strcpy(ssl_cert, value);
 124                 }
 125                 return 0;
 126         }
 127 #if LIBCURL_VERSION_NUM >= 0x070902
 128         if (!strcmp("http.sslkey", var)) {
 129                 if (ssl_key == NULL) {
 130                         ssl_key = xmalloc(strlen(value)+1);
 131                         strcpy(ssl_key, value);
 132                 }
 133                 return 0;
 134         }
 135 #endif
 136 #if LIBCURL_VERSION_NUM >= 0x070908
 137         if (!strcmp("http.sslcapath", var)) {
 138                 if (ssl_capath == NULL) {
 139                         ssl_capath = xmalloc(strlen(value)+1);
 140                         strcpy(ssl_capath, value);
 141                 }
 142                 return 0;
 143         }
 144 #endif
 145         if (!strcmp("http.sslcainfo", var)) {
 146                 if (ssl_cainfo == NULL) {
 147                         ssl_cainfo = xmalloc(strlen(value)+1);
 148                         strcpy(ssl_cainfo, value);
 149                 }
 150                 return 0;
 151         }
 152
 153 #ifdef USE_CURL_MULTI
 154         if (!strcmp("http.maxrequests", var)) {
 155                 if (max_requests == -1)
 156                         max_requests = git_config_int(var, value);
 157                 return 0;
 158         }
 159 #endif
 160
 161         /* Fall back on the default ones */
 162         return git_default_config(var, value);
 163 }
 164
 165 static size_t fwrite_buffer(void *ptr, size_t eltsize, size_t nmemb,
 166                             struct buffer *buffer)
 167 {
 168         size_t size = eltsize * nmemb;
 169         if (size > buffer->size - buffer->posn)
 170                 size = buffer->size - buffer->posn;
 171         memcpy(buffer->buffer + buffer->posn, ptr, size);
 172         buffer->posn += size;
 173         data_received++;
 174         return size;
 175 }
 176
 177 static size_t fwrite_buffer_dynamic(const void *ptr, size_t eltsize,
 178                                     size_t nmemb, struct buffer *buffer)
 179 {
 180         size_t size = eltsize * nmemb;
 181         if (size > buffer->size - buffer->posn) {
 182                 buffer->size = buffer->size * 3 / 2;
 183                 if (buffer->size < buffer->posn + size)
 184                         buffer->size = buffer->posn + size;
 185                 buffer->buffer = xrealloc(buffer->buffer, buffer->size);
 186         }
 187         memcpy(buffer->buffer + buffer->posn, ptr, size);
 188         buffer->posn += size;
 189         data_received++;
 190         return size;
 191 }
 192
 193 static size_t fwrite_sha1_file(void *ptr, size_t eltsize, size_t nmemb,
 194                                void *data)
 195 {
 196         unsigned char expn[4096];
 197         size_t size = eltsize * nmemb;
 198         int posn = 0;
 199         struct transfer_request *request = (struct transfer_request *)data;
 200         do {
 201                 ssize_t retval = write(request->local,
 202                                        ptr + posn, size - posn);
 203                 if (retval < 0)
 204                         return posn;
 205                 posn += retval;
 206         } while (posn < size);
 207
 208         request->stream.avail_in = size;
 209         request->stream.next_in = ptr;
 210         do {
 211                 request->stream.next_out = expn;
 212                 request->stream.avail_out = sizeof(expn);
 213                 request->zret = inflate(&request->stream, Z_SYNC_FLUSH);
 214                 SHA1_Update(&request->c, expn,
 215                             sizeof(expn) - request->stream.avail_out);
 216         } while (request->stream.avail_in && request->zret == Z_OK);
 217         data_received++;
 218         return size;
 219 }
 220
 221 #ifdef USE_CURL_MULTI
 222 static void process_curl_messages(void);
 223 static void process_request_queue(void);
 224 #endif
 225
 226 static CURL* get_curl_handle(void)
 227 {
 228         CURL* result = curl_easy_init();
 229
 230         curl_easy_setopt(result, CURLOPT_SSL_VERIFYPEER, curl_ssl_verify);
 231 #if LIBCURL_VERSION_NUM >= 0x070907
 232         curl_easy_setopt(result, CURLOPT_NETRC, CURL_NETRC_OPTIONAL);
 233 #endif
 234
 235         if (ssl_cert != NULL)
 236                 curl_easy_setopt(result, CURLOPT_SSLCERT, ssl_cert);
 237 #if LIBCURL_VERSION_NUM >= 0x070902
 238         if (ssl_key != NULL)
 239                 curl_easy_setopt(result, CURLOPT_SSLKEY, ssl_key);
 240 #endif
 241 #if LIBCURL_VERSION_NUM >= 0x070908
 242         if (ssl_capath != NULL)
 243                 curl_easy_setopt(result, CURLOPT_CAPATH, ssl_capath);
 244 #endif
 245         if (ssl_cainfo != NULL)
 246                 curl_easy_setopt(result, CURLOPT_CAINFO, ssl_cainfo);
 247         curl_easy_setopt(result, CURLOPT_FAILONERROR, 1);
 248
 249         return result;
 250 }
 251
 252 static struct active_request_slot *get_active_slot(void)
 253 {
 254         struct active_request_slot *slot = active_queue_head;
 255         struct active_request_slot *newslot;
 256
 257 #ifdef USE_CURL_MULTI
 258         int num_transfers;
 259
 260         /* Wait for a slot to open up if the queue is full */
 261         while (active_requests >= max_requests) {
 262                 curl_multi_perform(curlm, &num_transfers);
 263                 if (num_transfers < active_requests) {
 264                         process_curl_messages();
 265                 }
 266         }
 267 #endif
 268
 269         while (slot != NULL && slot->in_use) {
 270                 slot = slot->next;
 271         }
 272         if (slot == NULL) {
 273                 newslot = xmalloc(sizeof(*newslot));
 274 #ifdef NO_CURL_EASY_DUPHANDLE
 275                 newslot->curl = get_curl_handle();
 276 #else
 277                 newslot->curl = curl_easy_duphandle(curl_default);
 278 #endif
 279                 newslot->in_use = 0;
 280                 newslot->next = NULL;
 281
 282                 slot = active_queue_head;
 283                 if (slot == NULL) {
 284                         active_queue_head = newslot;
 285                 } else {
 286                         while (slot->next != NULL) {
 287                                 slot = slot->next;
 288                         }
 289                         slot->next = newslot;
 290                 }
 291                 slot = newslot;
 292         }
 293
 294         active_requests++;
 295         slot->in_use = 1;
 296         slot->done = 0;
 297         slot->local = NULL;
 298         curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, pragma_header);
 299         curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, no_range_header);
 300         curl_easy_setopt(slot->curl, CURLOPT_ERRORBUFFER, curl_errorstr);
 301
 302         return slot;
 303 }
 304
 305 static int start_active_slot(struct active_request_slot *slot)
 306 {
 307 #ifdef USE_CURL_MULTI
 308         CURLMcode curlm_result = curl_multi_add_handle(curlm, slot->curl);
 309
 310         if (curlm_result != CURLM_OK &&
 311             curlm_result != CURLM_CALL_MULTI_PERFORM) {
 312                 active_requests--;
 313                 slot->in_use = 0;
 314                 return 0;
 315         }
 316 #endif
 317         return 1;
 318 }
 319
 320 static void run_active_slot(struct active_request_slot *slot)
 321 {
 322 #ifdef USE_CURL_MULTI
 323         int num_transfers;
 324         long last_pos = 0;
 325         long current_pos;
 326         fd_set readfds;
 327         fd_set writefds;
 328         fd_set excfds;
 329         int max_fd;
 330         struct timeval select_timeout;
 331         CURLMcode curlm_result;
 332
 333         while (!slot->done) {
 334                 data_received = 0;
 335                 do {
 336                         curlm_result = curl_multi_perform(curlm,
 337                                                           &num_transfers);
 338                 } while (curlm_result == CURLM_CALL_MULTI_PERFORM);
 339                 if (num_transfers < active_requests) {
 340                         process_curl_messages();
 341                         process_request_queue();
 342                 }
 343
 344                 if (!data_received && slot->local != NULL) {
 345                         current_pos = ftell(slot->local);
 346                         if (current_pos > last_pos)
 347                                 data_received++;
 348                         last_pos = current_pos;
 349                 }
 350
 351                 if (!slot->done && !data_received) {
 352                         max_fd = 0;
 353                         FD_ZERO(&readfds);
 354                         FD_ZERO(&writefds);
 355                         FD_ZERO(&excfds);
 356                         select_timeout.tv_sec = 0;
 357                         select_timeout.tv_usec = 50000;
 358                         select(max_fd, &readfds, &writefds,
 359                                &excfds, &select_timeout);
 360                 }
 361         }
 362 #else
 363         slot->curl_result = curl_easy_perform(slot->curl);
 364         active_requests--;
 365 #endif
 366 }
 367
 368 static void start_request(struct transfer_request *request)
 369 {
 370         char *hex = sha1_to_hex(request->sha1);
 371         char prevfile[PATH_MAX];
 372         char *url;
 373         char *posn;
 374         int prevlocal;
 375         unsigned char prev_buf[PREV_BUF_SIZE];
 376         ssize_t prev_read = 0;
 377         long prev_posn = 0;
 378         char range[RANGE_HEADER_SIZE];
 379         struct curl_slist *range_header = NULL;
 380         struct active_request_slot *slot;
 381
 382         snprintf(prevfile, sizeof(prevfile), "%s.prev", request->filename);
 383         unlink(prevfile);
 384         rename(request->tmpfile, prevfile);
 385         unlink(request->tmpfile);
 386
 387         request->local = open(request->tmpfile,
 388                               O_WRONLY | O_CREAT | O_EXCL, 0666);
 389         /* This could have failed due to the "lazy directory creation";
 390          * try to mkdir the last path component.
 391          */
 392         if (request->local < 0 && errno == ENOENT) {
 393                 char *dir = strrchr(request->tmpfile, '/');
 394                 if (dir) {
 395                         *dir = 0;
 396                         mkdir(request->tmpfile, 0777);
 397                         *dir = '/';
 398                 }
 399                 request->local = open(request->tmpfile,
 400                                       O_WRONLY | O_CREAT | O_EXCL, 0666);
 401         }
 402
 403         if (request->local < 0) {
 404                 request->state = ABORTED;
 405                 error("Couldn't create temporary file %s for %s: %s\n",
 406                       request->tmpfile, request->filename, strerror(errno));
 407                 return;
 408         }
 409
 410         memset(&request->stream, 0, sizeof(request->stream));
 411
 412         inflateInit(&request->stream);
 413
 414         SHA1_Init(&request->c);
 415
 416         url = xmalloc(strlen(request->repo->base) + 50);
 417         request->url = xmalloc(strlen(request->repo->base) + 50);
 418         strcpy(url, request->repo->base);
 419         posn = url + strlen(request->repo->base);
 420         strcpy(posn, "objects/");
 421         posn += 8;
 422         memcpy(posn, hex, 2);
 423         posn += 2;
 424         *(posn++) = '/';
 425         strcpy(posn, hex + 2);
 426         strcpy(request->url, url);
 427
 428         /* If a previous temp file is present, process what was already
 429            fetched. */
 430         prevlocal = open(prevfile, O_RDONLY);
 431         if (prevlocal != -1) {
 432                 do {
 433                         prev_read = read(prevlocal, prev_buf, PREV_BUF_SIZE);
 434                         if (prev_read>0) {
 435                                 if (fwrite_sha1_file(prev_buf,
 436                                                      1,
 437                                                      prev_read,
 438                                                      request) == prev_read) {
 439                                         prev_posn += prev_read;
 440                                 } else {
 441                                         prev_read = -1;
 442                                 }
 443                         }
 444                 } while (prev_read > 0);
 445                 close(prevlocal);
 446         }
 447         unlink(prevfile);
 448
 449         /* Reset inflate/SHA1 if there was an error reading the previous temp
 450            file; also rewind to the beginning of the local file. */
 451         if (prev_read == -1) {
 452                 memset(&request->stream, 0, sizeof(request->stream));
 453                 inflateInit(&request->stream);
 454                 SHA1_Init(&request->c);
 455                 if (prev_posn>0) {
 456                         prev_posn = 0;
 457                         lseek(request->local, SEEK_SET, 0);
 458                         ftruncate(request->local, 0);
 459                 }
 460         }
 461
 462         slot = get_active_slot();
 463         curl_easy_setopt(slot->curl, CURLOPT_FILE, request);
 464         curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite_sha1_file);
 465         curl_easy_setopt(slot->curl, CURLOPT_ERRORBUFFER, request->errorstr);
 466         curl_easy_setopt(slot->curl, CURLOPT_URL, url);
 467         curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, no_pragma_header);
 468
 469         /* If we have successfully processed data from a previous fetch
 470            attempt, only fetch the data we don't already have. */
 471         if (prev_posn>0) {
 472                 if (get_verbosely)
 473                         fprintf(stderr,
 474                                 "Resuming fetch of object %s at byte %ld\n",
 475                                 hex, prev_posn);
 476                 sprintf(range, "Range: bytes=%ld-", prev_posn);
 477                 range_header = curl_slist_append(range_header, range);
 478                 curl_easy_setopt(slot->curl,
 479                                  CURLOPT_HTTPHEADER, range_header);
 480         }
 481
 482         /* Try to get the request started, abort the request on error */
 483         if (!start_active_slot(slot)) {
 484                 request->state = ABORTED;
 485                 close(request->local);
 486                 free(request->url);
 487                 return;
 488         }
 489
 490         request->slot = slot;
 491         request->state = ACTIVE;
 492 }
 493
 494 static void finish_request(struct transfer_request *request)
 495 {
 496         fchmod(request->local, 0444);
 497         close(request->local);
 498
 499         if (request->http_code == 416) {
 500                 fprintf(stderr, "Warning: requested range invalid; we may already have all the data.\n");
 501         } else if (request->curl_result != CURLE_OK) {
 502                 return;
 503         }
 504
 505         inflateEnd(&request->stream);
 506         SHA1_Final(request->real_sha1, &request->c);
 507         if (request->zret != Z_STREAM_END) {
 508                 unlink(request->tmpfile);
 509                 return;
 510         }
 511         if (memcmp(request->sha1, request->real_sha1, 20)) {
 512                 unlink(request->tmpfile);
 513                 return;
 514         }
 515         request->rename =
 516                 move_temp_to_file(request->tmpfile, request->filename);
 517
 518         if (request->rename == 0)
 519                 pull_say("got %s\n", sha1_to_hex(request->sha1));
 520 }
 521
 522 static void release_request(struct transfer_request *request)
 523 {
 524         struct transfer_request *entry = request_queue_head;
 525
 526         if (request == request_queue_head) {
 527                 request_queue_head = request->next;
 528         } else {
 529                 while (entry->next != NULL && entry->next != request)
 530                         entry = entry->next;
 531                 if (entry->next == request)
 532                         entry->next = entry->next->next;
 533         }
 534
 535         free(request->url);
 536         free(request);
 537 }
 538
 539 #ifdef USE_CURL_MULTI
 540 void process_curl_messages(void)
 541 {
 542         int num_messages;
 543         struct active_request_slot *slot;
 544         struct transfer_request *request = NULL;
 545         CURLMsg *curl_message = curl_multi_info_read(curlm, &num_messages);
 546
 547         while (curl_message != NULL) {
 548                 if (curl_message->msg == CURLMSG_DONE) {
 549                         slot = active_queue_head;
 550                         while (slot != NULL &&
 551                                slot->curl != curl_message->easy_handle)
 552                                 slot = slot->next;
 553                         if (slot != NULL) {
 554                                 curl_multi_remove_handle(curlm, slot->curl);
 555                                 active_requests--;
 556                                 slot->done = 1;
 557                                 slot->in_use = 0;
 558                                 slot->curl_result = curl_message->data.result;
 559                                 request = request_queue_head;
 560                                 while (request != NULL &&
 561                                        request->slot != slot)
 562                                         request = request->next;
 563                         } else {
 564                                 fprintf(stderr, "Received DONE message for unknown request!\n");
 565                         }
 566                         if (request != NULL) {
 567                                 request->curl_result =
 568                                         curl_message->data.result;
 569                                 curl_easy_getinfo(slot->curl,
 570                                                   CURLINFO_HTTP_CODE,
 571                                                   &request->http_code);
 572                                 request->slot = NULL;
 573
 574                                 /* Use alternates if necessary */
 575                                 if (request->http_code == 404 &&
 576                                     request->repo->next != NULL) {
 577                                         request->repo = request->repo->next;
 578                                         start_request(request);
 579                                 } else {
 580                                         finish_request(request);
 581                                         request->state = COMPLETE;
 582                                 }
 583                         }
 584                 } else {
 585                         fprintf(stderr, "Unknown CURL message received: %d\n",
 586                                 (int)curl_message->msg);
 587                 }
 588                 curl_message = curl_multi_info_read(curlm, &num_messages);
 589         }
 590 }
 591
 592 void process_request_queue(void)
 593 {
 594         struct transfer_request *request = request_queue_head;
 595         int num_transfers;
 596
 597         while (active_requests < max_requests && request != NULL) {
 598                 if (request->state == WAITING) {
 599                         if (has_sha1_file(request->sha1))
 600                                 release_request(request);
 601                         else
 602                                 start_request(request);
 603                         curl_multi_perform(curlm, &num_transfers);
 604                 }
 605                 request = request->next;
 606         }
 607 }
 608 #endif
 609
 610 void prefetch(unsigned char *sha1)
 611 {
 612         struct transfer_request *newreq;
 613         struct transfer_request *tail;
 614         char *filename = sha1_file_name(sha1);
 615
 616         newreq = xmalloc(sizeof(*newreq));
 617         memcpy(newreq->sha1, sha1, 20);
 618         newreq->repo = alt;
 619         newreq->url = NULL;
 620         newreq->local = -1;
 621         newreq->state = WAITING;
 622         snprintf(newreq->filename, sizeof(newreq->filename), "%s", filename);
 623         snprintf(newreq->tmpfile, sizeof(newreq->tmpfile),
 624                  "%s.temp", filename);
 625         newreq->next = NULL;
 626
 627         if (request_queue_head == NULL) {
 628                 request_queue_head = newreq;
 629         } else {
 630                 tail = request_queue_head;
 631                 while (tail->next != NULL) {
 632                         tail = tail->next;
 633                 }
 634                 tail->next = newreq;
 635         }
 636 #ifdef USE_CURL_MULTI
 637         process_request_queue();
 638         process_curl_messages();
 639 #endif
 640 }
 641
 642 static int fetch_index(struct alt_base *repo, unsigned char *sha1)
 643 {
 644         char *hex = sha1_to_hex(sha1);
 645         char *filename;
 646         char *url;
 647         char tmpfile[PATH_MAX];
 648         long prev_posn = 0;
 649         char range[RANGE_HEADER_SIZE];
 650         struct curl_slist *range_header = NULL;
 651
 652         FILE *indexfile;
 653         struct active_request_slot *slot;
 654
 655         if (has_pack_index(sha1))
 656                 return 0;
 657
 658         if (get_verbosely)
 659                 fprintf(stderr, "Getting index for pack %s\n", hex);
 660
 661         url = xmalloc(strlen(repo->base) + 64);
 662         sprintf(url, "%s/objects/pack/pack-%s.idx", repo->base, hex);
 663
 664         filename = sha1_pack_index_name(sha1);
 665         snprintf(tmpfile, sizeof(tmpfile), "%s.temp", filename);
 666         indexfile = fopen(tmpfile, "a");
 667         if (!indexfile)
 668                 return error("Unable to open local file %s for pack index",
 669                              filename);
 670
 671         slot = get_active_slot();
 672         curl_easy_setopt(slot->curl, CURLOPT_FILE, indexfile);
 673         curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite);
 674         curl_easy_setopt(slot->curl, CURLOPT_URL, url);
 675         curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, no_pragma_header);
 676         slot->local = indexfile;
 677
 678         /* If there is data present from a previous transfer attempt,
 679            resume where it left off */
 680         prev_posn = ftell(indexfile);
 681         if (prev_posn>0) {
 682                 if (get_verbosely)
 683                         fprintf(stderr,
 684                                 "Resuming fetch of index for pack %s at byte %ld\n",
 685                                 hex, prev_posn);
 686                 sprintf(range, "Range: bytes=%ld-", prev_posn);
 687                 range_header = curl_slist_append(range_header, range);
 688                 curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, range_header);
 689         }
 690
 691         if (start_active_slot(slot)) {
 692                 run_active_slot(slot);
 693                 if (slot->curl_result != CURLE_OK) {
 694                         fclose(indexfile);
 695                         return error("Unable to get pack index %s\n%s", url,
 696                                      curl_errorstr);
 697                 }
 698         } else {
 699                 return error("Unable to start request");
 700         }
 701
 702         fclose(indexfile);
 703
 704         return move_temp_to_file(tmpfile, filename);
 705 }
 706
 707 static int setup_index(struct alt_base *repo, unsigned char *sha1)
 708 {
 709         struct packed_git *new_pack;
 710         if (has_pack_file(sha1))
 711                 return 0; // don't list this as something we can get
 712
 713         if (fetch_index(repo, sha1))
 714                 return -1;
 715
 716         new_pack = parse_pack_index(sha1);
 717         new_pack->next = repo->packs;
 718         repo->packs = new_pack;
 719         return 0;
 720 }
 721
 722 static int fetch_alternates(char *base)
 723 {
 724         int ret = 0;
 725         struct buffer buffer;
 726         char *url;
 727         char *data;
 728         int i = 0;
 729         int http_specific = 1;
 730         struct alt_base *tail = alt;
 731         static const char null_byte = '\0';
 732
 733         struct active_request_slot *slot;
 734
 735         data = xmalloc(4096);
 736         buffer.size = 4096;
 737         buffer.posn = 0;
 738         buffer.buffer = data;
 739
 740         if (get_verbosely)
 741                 fprintf(stderr, "Getting alternates list\n");
 742
 743         url = xmalloc(strlen(base) + 31);
 744         sprintf(url, "%s/objects/info/http-alternates", base);
 745
 746         slot = get_active_slot();
 747         curl_easy_setopt(slot->curl, CURLOPT_FILE, &buffer);
 748         curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION,
 749                          fwrite_buffer_dynamic);
 750         curl_easy_setopt(slot->curl, CURLOPT_URL, url);
 751         if (start_active_slot(slot)) {
 752                 run_active_slot(slot);
 753                 if (slot->curl_result != CURLE_OK || !buffer.posn) {
 754                         http_specific = 0;
 755
 756                         sprintf(url, "%s/objects/info/alternates", base);
 757
 758                         slot = get_active_slot();
 759                         curl_easy_setopt(slot->curl, CURLOPT_FILE, &buffer);
 760                         curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION,
 761                                          fwrite_buffer_dynamic);
 762                         curl_easy_setopt(slot->curl, CURLOPT_URL, url);
 763                         if (start_active_slot(slot)) {
 764                                 run_active_slot(slot);
 765                                 if (slot->curl_result != CURLE_OK) {
 766                                         free(buffer.buffer);
 767                                         return 0;
 768                                 }
 769                         }
 770                 }
 771         } else {
 772                 free(buffer.buffer);
 773                 return 0;
 774         }
 775
 776         fwrite_buffer_dynamic(&null_byte, 1, 1, &buffer);
 777         buffer.posn--;
 778         data = buffer.buffer;
 779
 780         while (i < buffer.posn) {
 781                 int posn = i;
 782                 while (posn < buffer.posn && data[posn] != '\n')
 783                         posn++;
 784                 if (data[posn] == '\n') {
 785                         int okay = 0;
 786                         int serverlen = 0;
 787                         struct alt_base *newalt;
 788                         char *target = NULL;
 789                         if (data[i] == '/') {
 790                                 serverlen = strchr(base + 8, '/') - base;
 791                                 okay = 1;
 792                         } else if (!memcmp(data + i, "../", 3)) {
 793                                 i += 3;
 794                                 serverlen = strlen(base);
 795                                 while (i + 2 < posn &&
 796                                        !memcmp(data + i, "../", 3)) {
 797                                         do {
 798                                                 serverlen--;
 799                                         } while (serverlen &&
 800                                                  base[serverlen - 1] != '/');
 801                                         i += 3;
 802                                 }
 803                                 // If the server got removed, give up.
 804                                 okay = strchr(base, ':') - base + 3 <
 805                                         serverlen;
 806                         } else if (http_specific) {
 807                                 char *colon = strchr(data + i, ':');
 808                                 char *slash = strchr(data + i, '/');
 809                                 if (colon && slash && colon < data + posn &&
 810                                     slash < data + posn && colon < slash) {
 811                                         okay = 1;
 812                                 }
 813                         }
 814                         // skip 'objects' at end
 815                         if (okay) {
 816                                 target = xmalloc(serverlen + posn - i - 6);
 817                                 strncpy(target, base, serverlen);
 818                                 strncpy(target + serverlen, data + i,
 819                                         posn - i - 7);
 820                                 target[serverlen + posn - i - 7] = '\0';
 821                                 if (get_verbosely)
 822                                         fprintf(stderr,
 823                                                 "Also look at %s\n", target);
 824                                 newalt = xmalloc(sizeof(*newalt));
 825                                 newalt->next = NULL;
 826                                 newalt->base = target;
 827                                 newalt->got_indices = 0;
 828                                 newalt->packs = NULL;
 829                                 while (tail->next != NULL)
 830                                         tail = tail->next;
 831                                 tail->next = newalt;
 832                                 ret++;
 833                         }
 834                 }
 835                 i = posn + 1;
 836         }
 837
 838         free(buffer.buffer);
 839         return ret;
 840 }
 841
 842 static int fetch_indices(struct alt_base *repo)
 843 {
 844         unsigned char sha1[20];
 845         char *url;
 846         struct buffer buffer;
 847         char *data;
 848         int i = 0;
 849
 850         struct active_request_slot *slot;
 851
 852         if (repo->got_indices)
 853                 return 0;
 854
 855         data = xmalloc(4096);
 856         buffer.size = 4096;
 857         buffer.posn = 0;
 858         buffer.buffer = data;
 859
 860         if (get_verbosely)
 861                 fprintf(stderr, "Getting pack list\n");
 862
 863         url = xmalloc(strlen(repo->base) + 21);
 864         sprintf(url, "%s/objects/info/packs", repo->base);
 865
 866         slot = get_active_slot();
 867         curl_easy_setopt(slot->curl, CURLOPT_FILE, &buffer);
 868         curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION,
 869                          fwrite_buffer_dynamic);
 870         curl_easy_setopt(slot->curl, CURLOPT_URL, url);
 871         curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, NULL);
 872         if (start_active_slot(slot)) {
 873                 run_active_slot(slot);
 874                 if (slot->curl_result != CURLE_OK) {
 875                         free(buffer.buffer);
 876                         return error("%s", curl_errorstr);
 877                 }
 878         } else {
 879                 free(buffer.buffer);
 880                 return error("Unable to start request");
 881         }
 882
 883         data = buffer.buffer;
 884         while (i < buffer.posn) {
 885                 switch (data[i]) {
 886                 case 'P':
 887                         i++;
 888                         if (i + 52 < buffer.posn &&
 889                             !strncmp(data + i, " pack-", 6) &&
 890                             !strncmp(data + i + 46, ".pack\n", 6)) {
 891                                 get_sha1_hex(data + i + 6, sha1);
 892                                 setup_index(repo, sha1);
 893                                 i += 51;
 894                                 break;
 895                         }
 896                 default:
 897                         while (data[i] != '\n')
 898                                 i++;
 899                 }
 900                 i++;
 901         }
 902
 903         free(buffer.buffer);
 904         repo->got_indices = 1;
 905         return 0;
 906 }
 907
 908 static int fetch_pack(struct alt_base *repo, unsigned char *sha1)
 909 {
 910         char *url;
 911         struct packed_git *target;
 912         struct packed_git **lst;
 913         FILE *packfile;
 914         char *filename;
 915         char tmpfile[PATH_MAX];
 916         int ret;
 917         long prev_posn = 0;
 918         char range[RANGE_HEADER_SIZE];
 919         struct curl_slist *range_header = NULL;
 920
 921         struct active_request_slot *slot;
 922
 923         if (fetch_indices(repo))
 924                 return -1;
 925         target = find_sha1_pack(sha1, repo->packs);
 926         if (!target)
 927                 return -1;
 928
 929         if (get_verbosely) {
 930                 fprintf(stderr, "Getting pack %s\n",
 931                         sha1_to_hex(target->sha1));
 932                 fprintf(stderr, " which contains %s\n",
 933                         sha1_to_hex(sha1));
 934         }
 935
 936         url = xmalloc(strlen(repo->base) + 65);
 937         sprintf(url, "%s/objects/pack/pack-%s.pack",
 938                 repo->base, sha1_to_hex(target->sha1));
 939
 940         filename = sha1_pack_name(target->sha1);
 941         snprintf(tmpfile, sizeof(tmpfile), "%s.temp", filename);
 942         packfile = fopen(tmpfile, "a");
 943         if (!packfile)
 944                 return error("Unable to open local file %s for pack",
 945                              filename);
 946
 947         slot = get_active_slot();
 948         curl_easy_setopt(slot->curl, CURLOPT_FILE, packfile);
 949         curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite);
 950         curl_easy_setopt(slot->curl, CURLOPT_URL, url);
 951         curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, no_pragma_header);
 952         slot->local = packfile;
 953
 954         /* If there is data present from a previous transfer attempt,
 955            resume where it left off */
 956         prev_posn = ftell(packfile);
 957         if (prev_posn>0) {
 958                 if (get_verbosely)
 959                         fprintf(stderr,
 960                                 "Resuming fetch of pack %s at byte %ld\n",
 961                                 sha1_to_hex(target->sha1), prev_posn);
 962                 sprintf(range, "Range: bytes=%ld-", prev_posn);
 963                 range_header = curl_slist_append(range_header, range);
 964                 curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, range_header);
 965         }
 966
 967         if (start_active_slot(slot)) {
 968                 run_active_slot(slot);
 969                 if (slot->curl_result != CURLE_OK) {
 970                         fclose(packfile);
 971                         return error("Unable to get pack file %s\n%s", url,
 972                                      curl_errorstr);
 973                 }
 974         } else {
 975                 return error("Unable to start request");
 976         }
 977
 978         fclose(packfile);
 979
 980         ret = move_temp_to_file(tmpfile, filename);
 981         if (ret)
 982                 return ret;
 983
 984         lst = &repo->packs;
 985         while (*lst != target)
 986                 lst = &((*lst)->next);
 987         *lst = (*lst)->next;
 988
 989         if (verify_pack(target, 0))
 990                 return -1;
 991         install_packed_git(target);
 992
 993         return 0;
 994 }
 995
 996 static int fetch_object(struct alt_base *repo, unsigned char *sha1)
 997 {
 998         char *hex = sha1_to_hex(sha1);
 999         int ret;
1000         struct transfer_request *request = request_queue_head;
1001
1002         while (request != NULL && memcmp(request->sha1, sha1, 20))
1003                 request = request->next;
1004         if (request == NULL)
1005                 return error("Couldn't find request for %s in the queue", hex);
1006
1007         if (has_sha1_file(request->sha1)) {
1008                 release_request(request);
1009                 return 0;
1010         }
1011
1012 #ifdef USE_CURL_MULTI
1013         while (request->state == WAITING) {
1014                 int num_transfers;
1015                 curl_multi_perform(curlm, &num_transfers);
1016                 if (num_transfers < active_requests) {
1017                         process_curl_messages();
1018                         process_request_queue();
1019                 }
1020         }
1021 #else
1022         start_request(request);
1023 #endif
1024
1025         while (request->state == ACTIVE) {
1026                 run_active_slot(request->slot);
1027 #ifndef USE_CURL_MULTI
1028                 request->curl_result = request->slot->curl_result;
1029                 curl_easy_getinfo(request->slot->curl,
1030                                   CURLINFO_HTTP_CODE,
1031                                   &request->http_code);
1032                 request->slot = NULL;
1033
1034                 /* Use alternates if necessary */
1035                 if (request->http_code == 404 &&
1036                     request->repo->next != NULL) {
1037                         request->repo = request->repo->next;
1038                         start_request(request);
1039                 } else {
1040                         finish_request(request);
1041                         request->state = COMPLETE;
1042                 }
1043 #endif
1044         }
1045
1046         if (request->state == ABORTED) {
1047                 release_request(request);
1048                 return error("Request for %s aborted", hex);
1049         }
1050
1051         if (request->curl_result != CURLE_OK && request->http_code != 416) {
1052                 ret = error("%s (curl_result = %d, http_code = %ld, sha1 = %s)",
1053                             request->errorstr, request->curl_result,
1054                             request->http_code, hex);
1055                 release_request(request);
1056                 return ret;
1057         }
1058
1059         if (request->zret != Z_STREAM_END) {
1060                 ret = error("File %s (%s) corrupt\n", hex, request->url);
1061                 release_request(request);
1062                 return ret;
1063         }
1064
1065         if (memcmp(request->sha1, request->real_sha1, 20)) {
1066                 release_request(request);
1067                 return error("File %s has bad hash\n", hex);
1068         }
1069
1070         if (request->rename < 0) {
1071                 ret = error("unable to write sha1 filename %s: %s",
1072                             request->filename,
1073                             strerror(request->rename));
1074                 release_request(request);
1075                 return ret;
1076         }
1077
1078         release_request(request);
1079         return 0;
1080 }
1081
1082 int fetch(unsigned char *sha1)
1083 {
1084         struct alt_base *altbase = alt;
1085
1086         if (!fetch_object(altbase, sha1))
1087                 return 0;
1088         while (altbase) {
1089                 if (!fetch_pack(altbase, sha1))
1090                         return 0;
1091                 altbase = altbase->next;
1092         }
1093         return error("Unable to find %s under %s\n", sha1_to_hex(sha1),
1094                      alt->base);
1095 }
1096
1097 static inline int needs_quote(int ch)
1098 {
1099         switch (ch) {
1100         case '/': case '-': case '.':
1101         case 'A'...'Z': case 'a'...'z': case '0'...'9':
1102                 return 0;
1103         default:
1104                 return 1;
1105         }
1106 }
1107
1108 static inline int hex(int v)
1109 {
1110         if (v < 10) return '0' + v;
1111         else return 'A' + v - 10;
1112 }
1113
1114 static char *quote_ref_url(const char *base, const char *ref)
1115 {
1116         const char *cp;
1117         char *dp, *qref;
1118         int len, baselen, ch;
1119
1120         baselen = strlen(base);
1121         len = baselen + 6; /* "refs/" + NUL */
1122         for (cp = ref; (ch = *cp) != 0; cp++, len++)
1123                 if (needs_quote(ch))
1124                         len += 2; /* extra two hex plus replacement % */
1125         qref = xmalloc(len);
1126         memcpy(qref, base, baselen);
1127         memcpy(qref + baselen, "refs/", 5);
1128         for (cp = ref, dp = qref + baselen + 5; (ch = *cp) != 0; cp++) {
1129                 if (needs_quote(ch)) {
1130                         *dp++ = '%';
1131                         *dp++ = hex((ch >> 4) & 0xF);
1132                         *dp++ = hex(ch & 0xF);
1133                 }
1134                 else
1135                         *dp++ = ch;
1136         }
1137         *dp = 0;
1138
1139         return qref;
1140 }
1141
1142 int fetch_ref(char *ref, unsigned char *sha1)
1143 {
1144         char *url;
1145         char hex[42];
1146         struct buffer buffer;
1147         char *base = alt->base;
1148         struct active_request_slot *slot;
1149         buffer.size = 41;
1150         buffer.posn = 0;
1151         buffer.buffer = hex;
1152         hex[41] = '\0';
1153
1154         url = quote_ref_url(base, ref);
1155         slot = get_active_slot();
1156         curl_easy_setopt(slot->curl, CURLOPT_FILE, &buffer);
1157         curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite_buffer);
1158         curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, NULL);
1159         curl_easy_setopt(slot->curl, CURLOPT_URL, url);
1160         if (start_active_slot(slot)) {
1161                 run_active_slot(slot);
1162                 if (slot->curl_result != CURLE_OK)
1163                         return error("Couldn't get %s for %s\n%s",
1164                                      url, ref, curl_errorstr);
1165         } else {
1166                 return error("Unable to start request");
1167         }
1168
1169         hex[40] = '\0';
1170         get_sha1_hex(hex, sha1);
1171         return 0;
1172 }
1173
1174 int main(int argc, char **argv)
1175 {
1176         char *commit_id;
1177         char *url;
1178         int arg = 1;
1179         struct active_request_slot *slot;
1180
1181         while (arg < argc && argv[arg][0] == '-') {
1182                 if (argv[arg][1] == 't') {
1183                         get_tree = 1;
1184                 } else if (argv[arg][1] == 'c') {
1185                         get_history = 1;
1186                 } else if (argv[arg][1] == 'a') {
1187                         get_all = 1;
1188                         get_tree = 1;
1189                         get_history = 1;
1190                 } else if (argv[arg][1] == 'v') {
1191                         get_verbosely = 1;
1192                 } else if (argv[arg][1] == 'w') {
1193                         write_ref = argv[arg + 1];
1194                         arg++;
1195                 } else if (!strcmp(argv[arg], "--recover")) {
1196                         get_recover = 1;
1197                 }
1198                 arg++;
1199         }
1200         if (argc < arg + 2) {
1201                 usage("git-http-fetch [-c] [-t] [-a] [-d] [-v] [--recover] [-w ref] commit-id url");
1202                 return 1;
1203         }
1204         commit_id = argv[arg];
1205         url = argv[arg + 1];
1206
1207         curl_global_init(CURL_GLOBAL_ALL);
1208
1209 #ifdef USE_CURL_MULTI
1210         {
1211                 char *http_max_requests = getenv("GIT_HTTP_MAX_REQUESTS");
1212                 if (http_max_requests != NULL)
1213                         max_requests = atoi(http_max_requests);
1214         }
1215
1216         curlm = curl_multi_init();
1217         if (curlm == NULL) {
1218                 fprintf(stderr, "Error creating curl multi handle.\n");
1219                 return 1;
1220         }
1221 #endif
1222
1223         if (getenv("GIT_SSL_NO_VERIFY"))
1224                 curl_ssl_verify = 0;
1225
1226         ssl_cert = getenv("GIT_SSL_CERT");
1227 #if LIBCURL_VERSION_NUM >= 0x070902
1228         ssl_key = getenv("GIT_SSL_KEY");
1229 #endif
1230 #if LIBCURL_VERSION_NUM >= 0x070908
1231         ssl_capath = getenv("GIT_SSL_CAPATH");
1232 #endif
1233         ssl_cainfo = getenv("GIT_SSL_CAINFO");
1234
1235         git_config(http_options);
1236
1237         if (curl_ssl_verify == -1)
1238                 curl_ssl_verify = 1;
1239
1240 #ifdef USE_CURL_MULTI
1241         if (max_requests < 1)
1242                 max_requests = DEFAULT_MAX_REQUESTS;
1243 #endif
1244
1245         pragma_header = curl_slist_append(pragma_header, "Pragma: no-cache");
1246         no_pragma_header = curl_slist_append(no_pragma_header, "Pragma:");
1247         no_range_header = curl_slist_append(no_range_header, "Range:");
1248
1249 #ifndef NO_CURL_EASY_DUPHANDLE
1250         curl_default = get_curl_handle();
1251 #endif
1252
1253         alt = xmalloc(sizeof(*alt));
1254         alt->base = url;
1255         alt->got_indices = 0;
1256         alt->packs = NULL;
1257         alt->next = NULL;
1258         fetch_alternates(alt->base);
1259
1260         if (pull(commit_id))
1261                 return 1;
1262
1263         curl_slist_free_all(pragma_header);
1264         curl_slist_free_all(no_pragma_header);
1265         curl_slist_free_all(no_range_header);
1266 #ifndef NO_CURL_EASY_DUPHANDLE
1267         curl_easy_cleanup(curl_default);
1268 #endif
1269         slot = active_queue_head;
1270         while (slot != NULL) {
1271                 curl_easy_cleanup(slot->curl);
1272                 slot = slot->next;
1273         }
1274 #ifdef USE_CURL_MULTI
1275         curl_multi_cleanup(curlm);
1276 #endif
1277         curl_global_cleanup();
1278         return 0;
1279 }