?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
builtin/grep: add grep.fallbackToNoIndex config
[git.git] / http.c
blob9448c50f0f9ae159aa73feae6d980335265c168f
1 #include "git-compat-util.h"
2 #include "http.h"
3 #include "pack.h"
4 #include "sideband.h"
5 #include "run-command.h"
6 #include "url.h"
7 #include "urlmatch.h"
8 #include "credential.h"
9 #include "version.h"
10 #include "pkt-line.h"
11 #include "gettext.h"
12 #include "transport.h"
13
14 int active_requests;
15 int http_is_verbose;
16 size_t http_post_buffer = 16 * LARGE_PACKET_MAX;
17
18 #if LIBCURL_VERSION_NUM >= 0x070a06
19 #define LIBCURL_CAN_HANDLE_AUTH_ANY
20 #endif
21
22 static int min_curl_sessions = 1;
23 static int curl_session_count;
24 #ifdef USE_CURL_MULTI
25 static int max_requests = -1;
26 static CURLM *curlm;
27 #endif
28 #ifndef NO_CURL_EASY_DUPHANDLE
29 static CURL *curl_default;
30 #endif
31
32 #define PREV_BUF_SIZE 4096
33 #define RANGE_HEADER_SIZE 30
34
35 char curl_errorstr[CURL_ERROR_SIZE];
36
37 static int curl_ssl_verify = -1;
38 static int curl_ssl_try;
39 static const char *ssl_cert;
40 #if LIBCURL_VERSION_NUM >= 0x070903
41 static const char *ssl_key;
42 #endif
43 #if LIBCURL_VERSION_NUM >= 0x070908
44 static const char *ssl_capath;
45 #endif
46 static const char *ssl_cainfo;
47 static long curl_low_speed_limit = -1;
48 static long curl_low_speed_time = -1;
49 static int curl_ftp_no_epsv;
50 static const char *curl_http_proxy;
51 static const char *curl_cookie_file;
52 static int curl_save_cookies;
53 struct credential http_auth = CREDENTIAL_INIT;
54 static int http_proactive_auth;
55 static const char *user_agent;
56
57 #if LIBCURL_VERSION_NUM >= 0x071700
58 /* Use CURLOPT_KEYPASSWD as is */
59 #elif LIBCURL_VERSION_NUM >= 0x070903
60 #define CURLOPT_KEYPASSWD CURLOPT_SSLKEYPASSWD
61 #else
62 #define CURLOPT_KEYPASSWD CURLOPT_SSLCERTPASSWD
63 #endif
64
65 static struct credential cert_auth = CREDENTIAL_INIT;
66 static int ssl_cert_password_required;
67 #ifdef LIBCURL_CAN_HANDLE_AUTH_ANY
68 static unsigned long http_auth_methods = CURLAUTH_ANY;
69 #endif
70
71 static struct curl_slist *pragma_header;
72 static struct curl_slist *no_pragma_header;
73
74 static struct active_request_slot *active_queue_head;
75
76 static char *cached_accept_language;
77
78 size_t fread_buffer(char *ptr, size_t eltsize, size_t nmemb, void *buffer_)
79 {
80 size_t size = eltsize * nmemb;
81 struct buffer *buffer = buffer_;
82
83 if (size > buffer->buf.len - buffer->posn)
84 size = buffer->buf.len - buffer->posn;
85 memcpy(ptr, buffer->buf.buf + buffer->posn, size);
86 buffer->posn += size;
87
88 return size;
89 }
90
91 #ifndef NO_CURL_IOCTL
92 curlioerr ioctl_buffer(CURL *handle, int cmd, void *clientp)
93 {
94 struct buffer *buffer = clientp;
95
96 switch (cmd) {
97 case CURLIOCMD_NOP:
98 return CURLIOE_OK;
99
100 case CURLIOCMD_RESTARTREAD:
101 buffer->posn = 0;
102 return CURLIOE_OK;
103
104 default:
105 return CURLIOE_UNKNOWNCMD;
106 }
107 }
108 #endif
109
110 size_t fwrite_buffer(char *ptr, size_t eltsize, size_t nmemb, void *buffer_)
111 {
112 size_t size = eltsize * nmemb;
113 struct strbuf *buffer = buffer_;
114
115 strbuf_add(buffer, ptr, size);
116 return size;
117 }
118
119 size_t fwrite_null(char *ptr, size_t eltsize, size_t nmemb, void *strbuf)
120 {
121 return eltsize * nmemb;
122 }
123
124 static void closedown_active_slot(struct active_request_slot *slot)
125 {
126 active_requests--;
127 slot->in_use = 0;
128 }
129
130 static void finish_active_slot(struct active_request_slot *slot)
131 {
132 closedown_active_slot(slot);
133 curl_easy_getinfo(slot->curl, CURLINFO_HTTP_CODE, &slot->http_code);
134
135 if (slot->finished != NULL)
136 (*slot->finished) = 1;
137
138 /* Store slot results so they can be read after the slot is reused */
139 if (slot->results != NULL) {
140 slot->results->curl_result = slot->curl_result;
141 slot->results->http_code = slot->http_code;
142 #if LIBCURL_VERSION_NUM >= 0x070a08
143 curl_easy_getinfo(slot->curl, CURLINFO_HTTPAUTH_AVAIL,
144 &slot->results->auth_avail);
145 #else
146 slot->results->auth_avail = 0;
147 #endif
148 }
149
150 /* Run callback if appropriate */
151 if (slot->callback_func != NULL)
152 slot->callback_func(slot->callback_data);
153 }
154
155 #ifdef USE_CURL_MULTI
156 static void process_curl_messages(void)
157 {
158 int num_messages;
159 struct active_request_slot *slot;
160 CURLMsg *curl_message = curl_multi_info_read(curlm, &num_messages);
161
162 while (curl_message != NULL) {
163 if (curl_message->msg == CURLMSG_DONE) {
164 int curl_result = curl_message->data.result;
165 slot = active_queue_head;
166 while (slot != NULL &&
167 slot->curl != curl_message->easy_handle)
168 slot = slot->next;
169 if (slot != NULL) {
170 curl_multi_remove_handle(curlm, slot->curl);
171 slot->curl_result = curl_result;
172 finish_active_slot(slot);
173 } else {
174 fprintf(stderr, "Received DONE message for unknown request!\n");
175 }
176 } else {
177 fprintf(stderr, "Unknown CURL message received: %d\n",
178 (int)curl_message->msg);
179 }
180 curl_message = curl_multi_info_read(curlm, &num_messages);
181 }
182 }
183 #endif
184
185 static int http_options(const char *var, const char *value, void *cb)
186 {
187 if (!strcmp("http.sslverify", var)) {
188 curl_ssl_verify = git_config_bool(var, value);
189 return 0;
190 }
191 if (!strcmp("http.sslcert", var))
192 return git_config_string(&ssl_cert, var, value);
193 #if LIBCURL_VERSION_NUM >= 0x070903
194 if (!strcmp("http.sslkey", var))
195 return git_config_string(&ssl_key, var, value);
196 #endif
197 #if LIBCURL_VERSION_NUM >= 0x070908
198 if (!strcmp("http.sslcapath", var))
199 return git_config_string(&ssl_capath, var, value);
200 #endif
201 if (!strcmp("http.sslcainfo", var))
202 return git_config_string(&ssl_cainfo, var, value);
203 if (!strcmp("http.sslcertpasswordprotected", var)) {
204 ssl_cert_password_required = git_config_bool(var, value);
205 return 0;
206 }
207 if (!strcmp("http.ssltry", var)) {
208 curl_ssl_try = git_config_bool(var, value);
209 return 0;
210 }
211 if (!strcmp("http.minsessions", var)) {
212 min_curl_sessions = git_config_int(var, value);
213 #ifndef USE_CURL_MULTI
214 if (min_curl_sessions > 1)
215 min_curl_sessions = 1;
216 #endif
217 return 0;
218 }
219 #ifdef USE_CURL_MULTI
220 if (!strcmp("http.maxrequests", var)) {
221 max_requests = git_config_int(var, value);
222 return 0;
223 }
224 #endif
225 if (!strcmp("http.lowspeedlimit", var)) {
226 curl_low_speed_limit = (long)git_config_int(var, value);
227 return 0;
228 }
229 if (!strcmp("http.lowspeedtime", var)) {
230 curl_low_speed_time = (long)git_config_int(var, value);
231 return 0;
232 }
233
234 if (!strcmp("http.noepsv", var)) {
235 curl_ftp_no_epsv = git_config_bool(var, value);
236 return 0;
237 }
238 if (!strcmp("http.proxy", var))
239 return git_config_string(&curl_http_proxy, var, value);
240
241 if (!strcmp("http.cookiefile", var))
242 return git_config_string(&curl_cookie_file, var, value);
243 if (!strcmp("http.savecookies", var)) {
244 curl_save_cookies = git_config_bool(var, value);
245 return 0;
246 }
247
248 if (!strcmp("http.postbuffer", var)) {
249 http_post_buffer = git_config_int(var, value);
250 if (http_post_buffer < LARGE_PACKET_MAX)
251 http_post_buffer = LARGE_PACKET_MAX;
252 return 0;
253 }
254
255 if (!strcmp("http.useragent", var))
256 return git_config_string(&user_agent, var, value);
257
258 /* Fall back on the default ones */
259 return git_default_config(var, value, cb);
260 }
261
262 static void init_curl_http_auth(CURL *result)
263 {
264 if (!http_auth.username)
265 return;
266
267 credential_fill(&http_auth);
268
269 #if LIBCURL_VERSION_NUM >= 0x071301
270 curl_easy_setopt(result, CURLOPT_USERNAME, http_auth.username);
271 curl_easy_setopt(result, CURLOPT_PASSWORD, http_auth.password);
272 #else
273 {
274 static struct strbuf up = STRBUF_INIT;
275 /*
276 * Note that we assume we only ever have a single set of
277 * credentials in a given program run, so we do not have
278 * to worry about updating this buffer, only setting its
279 * initial value.
280 */
281 if (!up.len)
282 strbuf_addf(&up, "%s:%s",
283 http_auth.username, http_auth.password);
284 curl_easy_setopt(result, CURLOPT_USERPWD, up.buf);
285 }
286 #endif
287 }
288
289 static int has_cert_password(void)
290 {
291 if (ssl_cert == NULL || ssl_cert_password_required != 1)
292 return 0;
293 if (!cert_auth.password) {
294 cert_auth.protocol = xstrdup("cert");
295 cert_auth.username = xstrdup("");
296 cert_auth.path = xstrdup(ssl_cert);
297 credential_fill(&cert_auth);
298 }
299 return 1;
300 }
301
302 #if LIBCURL_VERSION_NUM >= 0x071900
303 static void set_curl_keepalive(CURL *c)
304 {
305 curl_easy_setopt(c, CURLOPT_TCP_KEEPALIVE, 1);
306 }
307
308 #elif LIBCURL_VERSION_NUM >= 0x071000
309 static int sockopt_callback(void *client, curl_socket_t fd, curlsocktype type)
310 {
311 int ka = 1;
312 int rc;
313 socklen_t len = (socklen_t)sizeof(ka);
314
315 if (type != CURLSOCKTYPE_IPCXN)
316 return 0;
317
318 rc = setsockopt(fd, SOL_SOCKET, SO_KEEPALIVE, (void *)&ka, len);
319 if (rc < 0)
320 warning("unable to set SO_KEEPALIVE on socket %s",
321 strerror(errno));
322
323 return 0; /* CURL_SOCKOPT_OK only exists since curl 7.21.5 */
324 }
325
326 static void set_curl_keepalive(CURL *c)
327 {
328 curl_easy_setopt(c, CURLOPT_SOCKOPTFUNCTION, sockopt_callback);
329 }
330
331 #else
332 static void set_curl_keepalive(CURL *c)
333 {
334 /* not supported on older curl versions */
335 }
336 #endif
337
338 static CURL *get_curl_handle(void)
339 {
340 CURL *result = curl_easy_init();
341 long allowed_protocols = 0;
342
343 if (!result)
344 die("curl_easy_init failed");
345
346 if (!curl_ssl_verify) {
347 curl_easy_setopt(result, CURLOPT_SSL_VERIFYPEER, 0);
348 curl_easy_setopt(result, CURLOPT_SSL_VERIFYHOST, 0);
349 } else {
350 /* Verify authenticity of the peer's certificate */
351 curl_easy_setopt(result, CURLOPT_SSL_VERIFYPEER, 1);
352 /* The name in the cert must match whom we tried to connect */
353 curl_easy_setopt(result, CURLOPT_SSL_VERIFYHOST, 2);
354 }
355
356 #if LIBCURL_VERSION_NUM >= 0x070907
357 curl_easy_setopt(result, CURLOPT_NETRC, CURL_NETRC_OPTIONAL);
358 #endif
359 #ifdef LIBCURL_CAN_HANDLE_AUTH_ANY
360 curl_easy_setopt(result, CURLOPT_HTTPAUTH, CURLAUTH_ANY);
361 #endif
362
363 if (http_proactive_auth)
364 init_curl_http_auth(result);
365
366 if (ssl_cert != NULL)
367 curl_easy_setopt(result, CURLOPT_SSLCERT, ssl_cert);
368 if (has_cert_password())
369 curl_easy_setopt(result, CURLOPT_KEYPASSWD, cert_auth.password);
370 #if LIBCURL_VERSION_NUM >= 0x070903
371 if (ssl_key != NULL)
372 curl_easy_setopt(result, CURLOPT_SSLKEY, ssl_key);
373 #endif
374 #if LIBCURL_VERSION_NUM >= 0x070908
375 if (ssl_capath != NULL)
376 curl_easy_setopt(result, CURLOPT_CAPATH, ssl_capath);
377 #endif
378 if (ssl_cainfo != NULL)
379 curl_easy_setopt(result, CURLOPT_CAINFO, ssl_cainfo);
380
381 if (curl_low_speed_limit > 0 && curl_low_speed_time > 0) {
382 curl_easy_setopt(result, CURLOPT_LOW_SPEED_LIMIT,
383 curl_low_speed_limit);
384 curl_easy_setopt(result, CURLOPT_LOW_SPEED_TIME,
385 curl_low_speed_time);
386 }
387
388 curl_easy_setopt(result, CURLOPT_FOLLOWLOCATION, 1);
389 curl_easy_setopt(result, CURLOPT_MAXREDIRS, 20);
390 #if LIBCURL_VERSION_NUM >= 0x071301
391 curl_easy_setopt(result, CURLOPT_POSTREDIR, CURL_REDIR_POST_ALL);
392 #elif LIBCURL_VERSION_NUM >= 0x071101
393 curl_easy_setopt(result, CURLOPT_POST301, 1);
394 #endif
395 #if LIBCURL_VERSION_NUM >= 0x071304
396 if (is_transport_allowed("http"))
397 allowed_protocols |= CURLPROTO_HTTP;
398 if (is_transport_allowed("https"))
399 allowed_protocols |= CURLPROTO_HTTPS;
400 if (is_transport_allowed("ftp"))
401 allowed_protocols |= CURLPROTO_FTP;
402 if (is_transport_allowed("ftps"))
403 allowed_protocols |= CURLPROTO_FTPS;
404 curl_easy_setopt(result, CURLOPT_REDIR_PROTOCOLS, allowed_protocols);
405 #else
406 if (transport_restrict_protocols())
407 warning("protocol restrictions not applied to curl redirects because\n"
408 "your curl version is too old (>= 7.19.4)");
409 #endif
410
411 if (getenv("GIT_CURL_VERBOSE"))
412 curl_easy_setopt(result, CURLOPT_VERBOSE, 1);
413
414 curl_easy_setopt(result, CURLOPT_USERAGENT,
415 user_agent ? user_agent : git_user_agent());
416
417 if (curl_ftp_no_epsv)
418 curl_easy_setopt(result, CURLOPT_FTP_USE_EPSV, 0);
419
420 #ifdef CURLOPT_USE_SSL
421 if (curl_ssl_try)
422 curl_easy_setopt(result, CURLOPT_USE_SSL, CURLUSESSL_TRY);
423 #endif
424
425 if (curl_http_proxy) {
426 curl_easy_setopt(result, CURLOPT_PROXY, curl_http_proxy);
427 }
428 #if LIBCURL_VERSION_NUM >= 0x070a07
429 curl_easy_setopt(result, CURLOPT_PROXYAUTH, CURLAUTH_ANY);
430 #endif
431
432 set_curl_keepalive(result);
433
434 return result;
435 }
436
437 static void set_from_env(const char **var, const char *envname)
438 {
439 const char *val = getenv(envname);
440 if (val)
441 *var = val;
442 }
443
444 void http_init(struct remote *remote, const char *url, int proactive_auth)
445 {
446 char *low_speed_limit;
447 char *low_speed_time;
448 char *normalized_url;
449 struct urlmatch_config config = { STRING_LIST_INIT_DUP };
450
451 config.section = "http";
452 config.key = NULL;
453 config.collect_fn = http_options;
454 config.cascade_fn = git_default_config;
455 config.cb = NULL;
456
457 http_is_verbose = 0;
458 normalized_url = url_normalize(url, &config.url);
459
460 git_config(urlmatch_config_entry, &config);
461 free(normalized_url);
462
463 if (curl_global_init(CURL_GLOBAL_ALL) != CURLE_OK)
464 die("curl_global_init failed");
465
466 http_proactive_auth = proactive_auth;
467
468 if (remote && remote->http_proxy)
469 curl_http_proxy = xstrdup(remote->http_proxy);
470
471 pragma_header = curl_slist_append(pragma_header, "Pragma: no-cache");
472 no_pragma_header = curl_slist_append(no_pragma_header, "Pragma:");
473
474 #ifdef USE_CURL_MULTI
475 {
476 char *http_max_requests = getenv("GIT_HTTP_MAX_REQUESTS");
477 if (http_max_requests != NULL)
478 max_requests = atoi(http_max_requests);
479 }
480
481 curlm = curl_multi_init();
482 if (!curlm)
483 die("curl_multi_init failed");
484 #endif
485
486 if (getenv("GIT_SSL_NO_VERIFY"))
487 curl_ssl_verify = 0;
488
489 set_from_env(&ssl_cert, "GIT_SSL_CERT");
490 #if LIBCURL_VERSION_NUM >= 0x070903
491 set_from_env(&ssl_key, "GIT_SSL_KEY");
492 #endif
493 #if LIBCURL_VERSION_NUM >= 0x070908
494 set_from_env(&ssl_capath, "GIT_SSL_CAPATH");
495 #endif
496 set_from_env(&ssl_cainfo, "GIT_SSL_CAINFO");
497
498 set_from_env(&user_agent, "GIT_HTTP_USER_AGENT");
499
500 low_speed_limit = getenv("GIT_HTTP_LOW_SPEED_LIMIT");
501 if (low_speed_limit != NULL)
502 curl_low_speed_limit = strtol(low_speed_limit, NULL, 10);
503 low_speed_time = getenv("GIT_HTTP_LOW_SPEED_TIME");
504 if (low_speed_time != NULL)
505 curl_low_speed_time = strtol(low_speed_time, NULL, 10);
506
507 if (curl_ssl_verify == -1)
508 curl_ssl_verify = 1;
509
510 curl_session_count = 0;
511 #ifdef USE_CURL_MULTI
512 if (max_requests < 1)
513 max_requests = DEFAULT_MAX_REQUESTS;
514 #endif
515
516 if (getenv("GIT_CURL_FTP_NO_EPSV"))
517 curl_ftp_no_epsv = 1;
518
519 if (url) {
520 credential_from_url(&http_auth, url);
521 if (!ssl_cert_password_required &&
522 getenv("GIT_SSL_CERT_PASSWORD_PROTECTED") &&
523 starts_with(url, "https://"))
524 ssl_cert_password_required = 1;
525 }
526
527 #ifndef NO_CURL_EASY_DUPHANDLE
528 curl_default = get_curl_handle();
529 #endif
530 }
531
532 void http_cleanup(void)
533 {
534 struct active_request_slot *slot = active_queue_head;
535
536 while (slot != NULL) {
537 struct active_request_slot *next = slot->next;
538 if (slot->curl != NULL) {
539 #ifdef USE_CURL_MULTI
540 curl_multi_remove_handle(curlm, slot->curl);
541 #endif
542 curl_easy_cleanup(slot->curl);
543 }
544 free(slot);
545 slot = next;
546 }
547 active_queue_head = NULL;
548
549 #ifndef NO_CURL_EASY_DUPHANDLE
550 curl_easy_cleanup(curl_default);
551 #endif
552
553 #ifdef USE_CURL_MULTI
554 curl_multi_cleanup(curlm);
555 #endif
556 curl_global_cleanup();
557
558 curl_slist_free_all(pragma_header);
559 pragma_header = NULL;
560
561 curl_slist_free_all(no_pragma_header);
562 no_pragma_header = NULL;
563
564 if (curl_http_proxy) {
565 free((void *)curl_http_proxy);
566 curl_http_proxy = NULL;
567 }
568
569 if (cert_auth.password != NULL) {
570 memset(cert_auth.password, 0, strlen(cert_auth.password));
571 free(cert_auth.password);
572 cert_auth.password = NULL;
573 }
574 ssl_cert_password_required = 0;
575
576 free(cached_accept_language);
577 cached_accept_language = NULL;
578 }
579
580 struct active_request_slot *get_active_slot(void)
581 {
582 struct active_request_slot *slot = active_queue_head;
583 struct active_request_slot *newslot;
584
585 #ifdef USE_CURL_MULTI
586 int num_transfers;
587
588 /* Wait for a slot to open up if the queue is full */
589 while (active_requests >= max_requests) {
590 curl_multi_perform(curlm, &num_transfers);
591 if (num_transfers < active_requests)
592 process_curl_messages();
593 }
594 #endif
595
596 while (slot != NULL && slot->in_use)
597 slot = slot->next;
598
599 if (slot == NULL) {
600 newslot = xmalloc(sizeof(*newslot));
601 newslot->curl = NULL;
602 newslot->in_use = 0;
603 newslot->next = NULL;
604
605 slot = active_queue_head;
606 if (slot == NULL) {
607 active_queue_head = newslot;
608 } else {
609 while (slot->next != NULL)
610 slot = slot->next;
611 slot->next = newslot;
612 }
613 slot = newslot;
614 }
615
616 if (slot->curl == NULL) {
617 #ifdef NO_CURL_EASY_DUPHANDLE
618 slot->curl = get_curl_handle();
619 #else
620 slot->curl = curl_easy_duphandle(curl_default);
621 #endif
622 curl_session_count++;
623 }
624
625 active_requests++;
626 slot->in_use = 1;
627 slot->results = NULL;
628 slot->finished = NULL;
629 slot->callback_data = NULL;
630 slot->callback_func = NULL;
631 curl_easy_setopt(slot->curl, CURLOPT_COOKIEFILE, curl_cookie_file);
632 if (curl_save_cookies)
633 curl_easy_setopt(slot->curl, CURLOPT_COOKIEJAR, curl_cookie_file);
634 curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, pragma_header);
635 curl_easy_setopt(slot->curl, CURLOPT_ERRORBUFFER, curl_errorstr);
636 curl_easy_setopt(slot->curl, CURLOPT_CUSTOMREQUEST, NULL);
637 curl_easy_setopt(slot->curl, CURLOPT_READFUNCTION, NULL);
638 curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, NULL);
639 curl_easy_setopt(slot->curl, CURLOPT_POSTFIELDS, NULL);
640 curl_easy_setopt(slot->curl, CURLOPT_UPLOAD, 0);
641 curl_easy_setopt(slot->curl, CURLOPT_HTTPGET, 1);
642 curl_easy_setopt(slot->curl, CURLOPT_FAILONERROR, 1);
643 #ifdef LIBCURL_CAN_HANDLE_AUTH_ANY
644 curl_easy_setopt(slot->curl, CURLOPT_HTTPAUTH, http_auth_methods);
645 #endif
646 if (http_auth.password)
647 init_curl_http_auth(slot->curl);
648
649 return slot;
650 }
651
652 int start_active_slot(struct active_request_slot *slot)
653 {
654 #ifdef USE_CURL_MULTI
655 CURLMcode curlm_result = curl_multi_add_handle(curlm, slot->curl);
656 int num_transfers;
657
658 if (curlm_result != CURLM_OK &&
659 curlm_result != CURLM_CALL_MULTI_PERFORM) {
660 active_requests--;
661 slot->in_use = 0;
662 return 0;
663 }
664
665 /*
666 * We know there must be something to do, since we just added
667 * something.
668 */
669 curl_multi_perform(curlm, &num_transfers);
670 #endif
671 return 1;
672 }
673
674 #ifdef USE_CURL_MULTI
675 struct fill_chain {
676 void *data;
677 int (*fill)(void *);
678 struct fill_chain *next;
679 };
680
681 static struct fill_chain *fill_cfg;
682
683 void add_fill_function(void *data, int (*fill)(void *))
684 {
685 struct fill_chain *new = xmalloc(sizeof(*new));
686 struct fill_chain **linkp = &fill_cfg;
687 new->data = data;
688 new->fill = fill;
689 new->next = NULL;
690 while (*linkp)
691 linkp = &(*linkp)->next;
692 *linkp = new;
693 }
694
695 void fill_active_slots(void)
696 {
697 struct active_request_slot *slot = active_queue_head;
698
699 while (active_requests < max_requests) {
700 struct fill_chain *fill;
701 for (fill = fill_cfg; fill; fill = fill->next)
702 if (fill->fill(fill->data))
703 break;
704
705 if (!fill)
706 break;
707 }
708
709 while (slot != NULL) {
710 if (!slot->in_use && slot->curl != NULL
711 && curl_session_count > min_curl_sessions) {
712 curl_easy_cleanup(slot->curl);
713 slot->curl = NULL;
714 curl_session_count--;
715 }
716 slot = slot->next;
717 }
718 }
719
720 void step_active_slots(void)
721 {
722 int num_transfers;
723 CURLMcode curlm_result;
724
725 do {
726 curlm_result = curl_multi_perform(curlm, &num_transfers);
727 } while (curlm_result == CURLM_CALL_MULTI_PERFORM);
728 if (num_transfers < active_requests) {
729 process_curl_messages();
730 fill_active_slots();
731 }
732 }
733 #endif
734
735 void run_active_slot(struct active_request_slot *slot)
736 {
737 #ifdef USE_CURL_MULTI
738 fd_set readfds;
739 fd_set writefds;
740 fd_set excfds;
741 int max_fd;
742 struct timeval select_timeout;
743 int finished = 0;
744
745 slot->finished = &finished;
746 while (!finished) {
747 step_active_slots();
748
749 if (slot->in_use) {
750 #if LIBCURL_VERSION_NUM >= 0x070f04
751 long curl_timeout;
752 curl_multi_timeout(curlm, &curl_timeout);
753 if (curl_timeout == 0) {
754 continue;
755 } else if (curl_timeout == -1) {
756 select_timeout.tv_sec = 0;
757 select_timeout.tv_usec = 50000;
758 } else {
759 select_timeout.tv_sec = curl_timeout / 1000;
760 select_timeout.tv_usec = (curl_timeout % 1000) * 1000;
761 }
762 #else
763 select_timeout.tv_sec = 0;
764 select_timeout.tv_usec = 50000;
765 #endif
766
767 max_fd = -1;
768 FD_ZERO(&readfds);
769 FD_ZERO(&writefds);
770 FD_ZERO(&excfds);
771 curl_multi_fdset(curlm, &readfds, &writefds, &excfds, &max_fd);
772
773 /*
774 * It can happen that curl_multi_timeout returns a pathologically
775 * long timeout when curl_multi_fdset returns no file descriptors
776 * to read. See commit message for more details.
777 */
778 if (max_fd < 0 &&
779 (select_timeout.tv_sec > 0 ||
780 select_timeout.tv_usec > 50000)) {
781 select_timeout.tv_sec = 0;
782 select_timeout.tv_usec = 50000;
783 }
784
785 select(max_fd+1, &readfds, &writefds, &excfds, &select_timeout);
786 }
787 }
788 #else
789 while (slot->in_use) {
790 slot->curl_result = curl_easy_perform(slot->curl);
791 finish_active_slot(slot);
792 }
793 #endif
794 }
795
796 static void release_active_slot(struct active_request_slot *slot)
797 {
798 closedown_active_slot(slot);
799 if (slot->curl && curl_session_count > min_curl_sessions) {
800 #ifdef USE_CURL_MULTI
801 curl_multi_remove_handle(curlm, slot->curl);
802 #endif
803 curl_easy_cleanup(slot->curl);
804 slot->curl = NULL;
805 curl_session_count--;
806 }
807 #ifdef USE_CURL_MULTI
808 fill_active_slots();
809 #endif
810 }
811
812 void finish_all_active_slots(void)
813 {
814 struct active_request_slot *slot = active_queue_head;
815
816 while (slot != NULL)
817 if (slot->in_use) {
818 run_active_slot(slot);
819 slot = active_queue_head;
820 } else {
821 slot = slot->next;
822 }
823 }
824
825 /* Helpers for modifying and creating URLs */
826 static inline int needs_quote(int ch)
827 {
828 if (((ch >= 'A') && (ch <= 'Z'))
829 || ((ch >= 'a') && (ch <= 'z'))
830 || ((ch >= '0') && (ch <= '9'))
831 || (ch == '/')
832 || (ch == '-')
833 || (ch == '.'))
834 return 0;
835 return 1;
836 }
837
838 static char *quote_ref_url(const char *base, const char *ref)
839 {
840 struct strbuf buf = STRBUF_INIT;
841 const char *cp;
842 int ch;
843
844 end_url_with_slash(&buf, base);
845
846 for (cp = ref; (ch = *cp) != 0; cp++)
847 if (needs_quote(ch))
848 strbuf_addf(&buf, "%%%02x", ch);
849 else
850 strbuf_addch(&buf, *cp);
851
852 return strbuf_detach(&buf, NULL);
853 }
854
855 void append_remote_object_url(struct strbuf *buf, const char *url,
856 const char *hex,
857 int only_two_digit_prefix)
858 {
859 end_url_with_slash(buf, url);
860
861 strbuf_addf(buf, "objects/%.*s/", 2, hex);
862 if (!only_two_digit_prefix)
863 strbuf_addf(buf, "%s", hex+2);
864 }
865
866 char *get_remote_object_url(const char *url, const char *hex,
867 int only_two_digit_prefix)
868 {
869 struct strbuf buf = STRBUF_INIT;
870 append_remote_object_url(&buf, url, hex, only_two_digit_prefix);
871 return strbuf_detach(&buf, NULL);
872 }
873
874 static int handle_curl_result(struct slot_results *results)
875 {
876 /*
877 * If we see a failing http code with CURLE_OK, we have turned off
878 * FAILONERROR (to keep the server's custom error response), and should
879 * translate the code into failure here.
880 */
881 if (results->curl_result == CURLE_OK &&
882 results->http_code >= 400) {
883 results->curl_result = CURLE_HTTP_RETURNED_ERROR;
884 /*
885 * Normally curl will already have put the "reason phrase"
886 * from the server into curl_errorstr; unfortunately without
887 * FAILONERROR it is lost, so we can give only the numeric
888 * status code.
889 */
890 snprintf(curl_errorstr, sizeof(curl_errorstr),
891 "The requested URL returned error: %ld",
892 results->http_code);
893 }
894
895 if (results->curl_result == CURLE_OK) {
896 credential_approve(&http_auth);
897 return HTTP_OK;
898 } else if (missing_target(results))
899 return HTTP_MISSING_TARGET;
900 else if (results->http_code == 401) {
901 if (http_auth.username && http_auth.password) {
902 credential_reject(&http_auth);
903 return HTTP_NOAUTH;
904 } else {
905 #ifdef LIBCURL_CAN_HANDLE_AUTH_ANY
906 http_auth_methods &= ~CURLAUTH_GSSNEGOTIATE;
907 #endif
908 return HTTP_REAUTH;
909 }
910 } else {
911 #if LIBCURL_VERSION_NUM >= 0x070c00
912 if (!curl_errorstr[0])
913 strlcpy(curl_errorstr,
914 curl_easy_strerror(results->curl_result),
915 sizeof(curl_errorstr));
916 #endif
917 return HTTP_ERROR;
918 }
919 }
920
921 int run_one_slot(struct active_request_slot *slot,
922 struct slot_results *results)
923 {
924 slot->results = results;
925 if (!start_active_slot(slot)) {
926 snprintf(curl_errorstr, sizeof(curl_errorstr),
927 "failed to start HTTP request");
928 return HTTP_START_FAILED;
929 }
930
931 run_active_slot(slot);
932 return handle_curl_result(results);
933 }
934
935 static CURLcode curlinfo_strbuf(CURL *curl, CURLINFO info, struct strbuf *buf)
936 {
937 char *ptr;
938 CURLcode ret;
939
940 strbuf_reset(buf);
941 ret = curl_easy_getinfo(curl, info, &ptr);
942 if (!ret && ptr)
943 strbuf_addstr(buf, ptr);
944 return ret;
945 }
946
947 /*
948 * Check for and extract a content-type parameter. "raw"
949 * should be positioned at the start of the potential
950 * parameter, with any whitespace already removed.
951 *
952 * "name" is the name of the parameter. The value is appended
953 * to "out".
954 */
955 static int extract_param(const char *raw, const char *name,
956 struct strbuf *out)
957 {
958 size_t len = strlen(name);
959
960 if (strncasecmp(raw, name, len))
961 return -1;
962 raw += len;
963
964 if (*raw != '=')
965 return -1;
966 raw++;
967
968 while (*raw && !isspace(*raw) && *raw != ';')
969 strbuf_addch(out, *raw++);
970 return 0;
971 }
972
973 /*
974 * Extract a normalized version of the content type, with any
975 * spaces suppressed, all letters lowercased, and no trailing ";"
976 * or parameters.
977 *
978 * Note that we will silently remove even invalid whitespace. For
979 * example, "text / plain" is specifically forbidden by RFC 2616,
980 * but "text/plain" is the only reasonable output, and this keeps
981 * our code simple.
982 *
983 * If the "charset" argument is not NULL, store the value of any
984 * charset parameter there.
985 *
986 * Example:
987 * "TEXT/PLAIN; charset=utf-8" -> "text/plain", "utf-8"
988 * "text / plain" -> "text/plain"
989 */
990 static void extract_content_type(struct strbuf *raw, struct strbuf *type,
991 struct strbuf *charset)
992 {
993 const char *p;
994
995 strbuf_reset(type);
996 strbuf_grow(type, raw->len);
997 for (p = raw->buf; *p; p++) {
998 if (isspace(*p))
999 continue;
1000 if (*p == ';') {
1001 p++;
1002 break;
1003 }
1004 strbuf_addch(type, tolower(*p));
1005 }
1006
1007 if (!charset)
1008 return;
1009
1010 strbuf_reset(charset);
1011 while (*p) {
1012 while (isspace(*p) || *p == ';')
1013 p++;
1014 if (!extract_param(p, "charset", charset))
1015 return;
1016 while (*p && !isspace(*p))
1017 p++;
1018 }
1019
1020 if (!charset->len && starts_with(type->buf, "text/"))
1021 strbuf_addstr(charset, "ISO-8859-1");
1022 }
1023
1024 static void write_accept_language(struct strbuf *buf)
1025 {
1026 /*
1027 * MAX_DECIMAL_PLACES must not be larger than 3. If it is larger than
1028 * that, q-value will be smaller than 0.001, the minimum q-value the
1029 * HTTP specification allows. See
1030 * http://tools.ietf.org/html/rfc7231#section-5.3.1 for q-value.
1031 */
1032 const int MAX_DECIMAL_PLACES = 3;
1033 const int MAX_LANGUAGE_TAGS = 1000;
1034 const int MAX_ACCEPT_LANGUAGE_HEADER_SIZE = 4000;
1035 char **language_tags = NULL;
1036 int num_langs = 0;
1037 const char *s = get_preferred_languages();
1038 int i;
1039 struct strbuf tag = STRBUF_INIT;
1040
1041 /* Don't add Accept-Language header if no language is preferred. */
1042 if (!s)
1043 return;
1044
1045 /*
1046 * Split the colon-separated string of preferred languages into
1047 * language_tags array.
1048 */
1049 do {
1050 /* collect language tag */
1051 for (; *s && (isalnum(*s) || *s == '_'); s++)
1052 strbuf_addch(&tag, *s == '_' ? '-' : *s);
1053
1054 /* skip .codeset, @modifier and any other unnecessary parts */
1055 while (*s && *s != ':')
1056 s++;
1057
1058 if (tag.len) {
1059 num_langs++;
1060 REALLOC_ARRAY(language_tags, num_langs);
1061 language_tags[num_langs - 1] = strbuf_detach(&tag, NULL);
1062 if (num_langs >= MAX_LANGUAGE_TAGS - 1) /* -1 for '*' */
1063 break;
1064 }
1065 } while (*s++);
1066
1067 /* write Accept-Language header into buf */
1068 if (num_langs) {
1069 int last_buf_len = 0;
1070 int max_q;
1071 int decimal_places;
1072 char q_format[32];
1073
1074 /* add '*' */
1075 REALLOC_ARRAY(language_tags, num_langs + 1);
1076 language_tags[num_langs++] = "*"; /* it's OK; this won't be freed */
1077
1078 /* compute decimal_places */
1079 for (max_q = 1, decimal_places = 0;
1080 max_q < num_langs && decimal_places <= MAX_DECIMAL_PLACES;
1081 decimal_places++, max_q *= 10)
1082 ;
1083
1084 sprintf(q_format, ";q=0.%%0%dd", decimal_places);
1085
1086 strbuf_addstr(buf, "Accept-Language: ");
1087
1088 for (i = 0; i < num_langs; i++) {
1089 if (i > 0)
1090 strbuf_addstr(buf, ", ");
1091
1092 strbuf_addstr(buf, language_tags[i]);
1093
1094 if (i > 0)
1095 strbuf_addf(buf, q_format, max_q - i);
1096
1097 if (buf->len > MAX_ACCEPT_LANGUAGE_HEADER_SIZE) {
1098 strbuf_remove(buf, last_buf_len, buf->len - last_buf_len);
1099 break;
1100 }
1101
1102 last_buf_len = buf->len;
1103 }
1104 }
1105
1106 /* free language tags -- last one is a static '*' */
1107 for (i = 0; i < num_langs - 1; i++)
1108 free(language_tags[i]);
1109 free(language_tags);
1110 }
1111
1112 /*
1113 * Get an Accept-Language header which indicates user's preferred languages.
1114 *
1115 * Examples:
1116 * LANGUAGE= -> ""
1117 * LANGUAGE=ko:en -> "Accept-Language: ko, en; q=0.9, *; q=0.1"
1118 * LANGUAGE=ko_KR.UTF-8:sr@latin -> "Accept-Language: ko-KR, sr; q=0.9, *; q=0.1"
1119 * LANGUAGE=ko LANG=en_US.UTF-8 -> "Accept-Language: ko, *; q=0.1"
1120 * LANGUAGE= LANG=en_US.UTF-8 -> "Accept-Language: en-US, *; q=0.1"
1121 * LANGUAGE= LANG=C -> ""
1122 */
1123 static const char *get_accept_language(void)
1124 {
1125 if (!cached_accept_language) {
1126 struct strbuf buf = STRBUF_INIT;
1127 write_accept_language(&buf);
1128 if (buf.len > 0)
1129 cached_accept_language = strbuf_detach(&buf, NULL);
1130 }
1131
1132 return cached_accept_language;
1133 }
1134
1135 /* http_request() targets */
1136 #define HTTP_REQUEST_STRBUF 0
1137 #define HTTP_REQUEST_FILE 1
1138
1139 static int http_request(const char *url,
1140 void *result, int target,
1141 const struct http_get_options *options)
1142 {
1143 struct active_request_slot *slot;
1144 struct slot_results results;
1145 struct curl_slist *headers = NULL;
1146 struct strbuf buf = STRBUF_INIT;
1147 const char *accept_language;
1148 int ret;
1149
1150 slot = get_active_slot();
1151 curl_easy_setopt(slot->curl, CURLOPT_HTTPGET, 1);
1152
1153 if (result == NULL) {
1154 curl_easy_setopt(slot->curl, CURLOPT_NOBODY, 1);
1155 } else {
1156 curl_easy_setopt(slot->curl, CURLOPT_NOBODY, 0);
1157 curl_easy_setopt(slot->curl, CURLOPT_FILE, result);
1158
1159 if (target == HTTP_REQUEST_FILE) {
1160 long posn = ftell(result);
1161 curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION,
1162 fwrite);
1163 if (posn > 0) {
1164 strbuf_addf(&buf, "Range: bytes=%ld-", posn);
1165 headers = curl_slist_append(headers, buf.buf);
1166 strbuf_reset(&buf);
1167 }
1168 } else
1169 curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION,
1170 fwrite_buffer);
1171 }
1172
1173 accept_language = get_accept_language();
1174
1175 if (accept_language)
1176 headers = curl_slist_append(headers, accept_language);
1177
1178 strbuf_addstr(&buf, "Pragma:");
1179 if (options && options->no_cache)
1180 strbuf_addstr(&buf, " no-cache");
1181 if (options && options->keep_error)
1182 curl_easy_setopt(slot->curl, CURLOPT_FAILONERROR, 0);
1183
1184 headers = curl_slist_append(headers, buf.buf);
1185
1186 curl_easy_setopt(slot->curl, CURLOPT_URL, url);
1187 curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, headers);
1188 curl_easy_setopt(slot->curl, CURLOPT_ENCODING, "gzip");
1189
1190 ret = run_one_slot(slot, &results);
1191
1192 if (options && options->content_type) {
1193 struct strbuf raw = STRBUF_INIT;
1194 curlinfo_strbuf(slot->curl, CURLINFO_CONTENT_TYPE, &raw);
1195 extract_content_type(&raw, options->content_type,
1196 options->charset);
1197 strbuf_release(&raw);
1198 }
1199
1200 if (options && options->effective_url)
1201 curlinfo_strbuf(slot->curl, CURLINFO_EFFECTIVE_URL,
1202 options->effective_url);
1203
1204 curl_slist_free_all(headers);
1205 strbuf_release(&buf);
1206
1207 return ret;
1208 }
1209
1210 /*
1211 * Update the "base" url to a more appropriate value, as deduced by
1212 * redirects seen when requesting a URL starting with "url".
1213 *
1214 * The "asked" parameter is a URL that we asked curl to access, and must begin
1215 * with "base".
1216 *
1217 * The "got" parameter is the URL that curl reported to us as where we ended
1218 * up.
1219 *
1220 * Returns 1 if we updated the base url, 0 otherwise.
1221 *
1222 * Our basic strategy is to compare "base" and "asked" to find the bits
1223 * specific to our request. We then strip those bits off of "got" to yield the
1224 * new base. So for example, if our base is "http://example.com/foo.git",
1225 * and we ask for "http://example.com/foo.git/info/refs", we might end up
1226 * with "https://other.example.com/foo.git/info/refs". We would want the
1227 * new URL to become "https://other.example.com/foo.git".
1228 *
1229 * Note that this assumes a sane redirect scheme. It's entirely possible
1230 * in the example above to end up at a URL that does not even end in
1231 * "info/refs". In such a case we simply punt, as there is not much we can
1232 * do (and such a scheme is unlikely to represent a real git repository,
1233 * which means we are likely about to abort anyway).
1234 */
1235 static int update_url_from_redirect(struct strbuf *base,
1236 const char *asked,
1237 const struct strbuf *got)
1238 {
1239 const char *tail;
1240 size_t tail_len;
1241
1242 if (!strcmp(asked, got->buf))
1243 return 0;
1244
1245 if (!skip_prefix(asked, base->buf, &tail))
1246 die("BUG: update_url_from_redirect: %s is not a superset of %s",
1247 asked, base->buf);
1248
1249 tail_len = strlen(tail);
1250
1251 if (got->len < tail_len ||
1252 strcmp(tail, got->buf + got->len - tail_len))
1253 return 0; /* insane redirect scheme */
1254
1255 strbuf_reset(base);
1256 strbuf_add(base, got->buf, got->len - tail_len);
1257 return 1;
1258 }
1259
1260 static int http_request_reauth(const char *url,
1261 void *result, int target,
1262 struct http_get_options *options)
1263 {
1264 int ret = http_request(url, result, target, options);
1265
1266 if (options && options->effective_url && options->base_url) {
1267 if (update_url_from_redirect(options->base_url,
1268 url, options->effective_url)) {
1269 credential_from_url(&http_auth, options->base_url->buf);
1270 url = options->effective_url->buf;
1271 }
1272 }
1273
1274 if (ret != HTTP_REAUTH)
1275 return ret;
1276
1277 /*
1278 * If we are using KEEP_ERROR, the previous request may have
1279 * put cruft into our output stream; we should clear it out before
1280 * making our next request. We only know how to do this for
1281 * the strbuf case, but that is enough to satisfy current callers.
1282 */
1283 if (options && options->keep_error) {
1284 switch (target) {
1285 case HTTP_REQUEST_STRBUF:
1286 strbuf_reset(result);
1287 break;
1288 default:
1289 die("BUG: HTTP_KEEP_ERROR is only supported with strbufs");
1290 }
1291 }
1292
1293 credential_fill(&http_auth);
1294
1295 return http_request(url, result, target, options);
1296 }
1297
1298 int http_get_strbuf(const char *url,
1299 struct strbuf *result,
1300 struct http_get_options *options)
1301 {
1302 return http_request_reauth(url, result, HTTP_REQUEST_STRBUF, options);
1303 }
1304
1305 /*
1306 * Downloads a URL and stores the result in the given file.
1307 *
1308 * If a previous interrupted download is detected (i.e. a previous temporary
1309 * file is still around) the download is resumed.
1310 */
1311 static int http_get_file(const char *url, const char *filename,
1312 struct http_get_options *options)
1313 {
1314 int ret;
1315 struct strbuf tmpfile = STRBUF_INIT;
1316 FILE *result;
1317
1318 strbuf_addf(&tmpfile, "%s.temp", filename);
1319 result = fopen(tmpfile.buf, "a");
1320 if (!result) {
1321 error("Unable to open local file %s", tmpfile.buf);
1322 ret = HTTP_ERROR;
1323 goto cleanup;
1324 }
1325
1326 ret = http_request_reauth(url, result, HTTP_REQUEST_FILE, options);
1327 fclose(result);
1328
1329 if (ret == HTTP_OK && move_temp_to_file(tmpfile.buf, filename))
1330 ret = HTTP_ERROR;
1331 cleanup:
1332 strbuf_release(&tmpfile);
1333 return ret;
1334 }
1335
1336 int http_fetch_ref(const char *base, struct ref *ref)
1337 {
1338 struct http_get_options options = {0};
1339 char *url;
1340 struct strbuf buffer = STRBUF_INIT;
1341 int ret = -1;
1342
1343 options.no_cache = 1;
1344
1345 url = quote_ref_url(base, ref->name);
1346 if (http_get_strbuf(url, &buffer, &options) == HTTP_OK) {
1347 strbuf_rtrim(&buffer);
1348 if (buffer.len == 40)
1349 ret = get_sha1_hex(buffer.buf, ref->old_sha1);
1350 else if (starts_with(buffer.buf, "ref: ")) {
1351 ref->symref = xstrdup(buffer.buf + 5);
1352 ret = 0;
1353 }
1354 }
1355
1356 strbuf_release(&buffer);
1357 free(url);
1358 return ret;
1359 }
1360
1361 /* Helpers for fetching packs */
1362 static char *fetch_pack_index(unsigned char *sha1, const char *base_url)
1363 {
1364 char *url, *tmp;
1365 struct strbuf buf = STRBUF_INIT;
1366
1367 if (http_is_verbose)
1368 fprintf(stderr, "Getting index for pack %s\n", sha1_to_hex(sha1));
1369
1370 end_url_with_slash(&buf, base_url);
1371 strbuf_addf(&buf, "objects/pack/pack-%s.idx", sha1_to_hex(sha1));
1372 url = strbuf_detach(&buf, NULL);
1373
1374 strbuf_addf(&buf, "%s.temp", sha1_pack_index_name(sha1));
1375 tmp = strbuf_detach(&buf, NULL);
1376
1377 if (http_get_file(url, tmp, NULL) != HTTP_OK) {
1378 error("Unable to get pack index %s", url);
1379 free(tmp);
1380 tmp = NULL;
1381 }
1382
1383 free(url);
1384 return tmp;
1385 }
1386
1387 static int fetch_and_setup_pack_index(struct packed_git **packs_head,
1388 unsigned char *sha1, const char *base_url)
1389 {
1390 struct packed_git *new_pack;
1391 char *tmp_idx = NULL;
1392 int ret;
1393
1394 if (has_pack_index(sha1)) {
1395 new_pack = parse_pack_index(sha1, sha1_pack_index_name(sha1));
1396 if (!new_pack)
1397 return -1; /* parse_pack_index() already issued error message */
1398 goto add_pack;
1399 }
1400
1401 tmp_idx = fetch_pack_index(sha1, base_url);
1402 if (!tmp_idx)
1403 return -1;
1404
1405 new_pack = parse_pack_index(sha1, tmp_idx);
1406 if (!new_pack) {
1407 unlink(tmp_idx);
1408 free(tmp_idx);
1409
1410 return -1; /* parse_pack_index() already issued error message */
1411 }
1412
1413 ret = verify_pack_index(new_pack);
1414 if (!ret) {
1415 close_pack_index(new_pack);
1416 ret = move_temp_to_file(tmp_idx, sha1_pack_index_name(sha1));
1417 }
1418 free(tmp_idx);
1419 if (ret)
1420 return -1;
1421
1422 add_pack:
1423 new_pack->next = *packs_head;
1424 *packs_head = new_pack;
1425 return 0;
1426 }
1427
1428 int http_get_info_packs(const char *base_url, struct packed_git **packs_head)
1429 {
1430 struct http_get_options options = {0};
1431 int ret = 0, i = 0;
1432 char *url, *data;
1433 struct strbuf buf = STRBUF_INIT;
1434 unsigned char sha1[20];
1435
1436 end_url_with_slash(&buf, base_url);
1437 strbuf_addstr(&buf, "objects/info/packs");
1438 url = strbuf_detach(&buf, NULL);
1439
1440 options.no_cache = 1;
1441 ret = http_get_strbuf(url, &buf, &options);
1442 if (ret != HTTP_OK)
1443 goto cleanup;
1444
1445 data = buf.buf;
1446 while (i < buf.len) {
1447 switch (data[i]) {
1448 case 'P':
1449 i++;
1450 if (i + 52 <= buf.len &&
1451 starts_with(data + i, " pack-") &&
1452 starts_with(data + i + 46, ".pack\n")) {
1453 get_sha1_hex(data + i + 6, sha1);
1454 fetch_and_setup_pack_index(packs_head, sha1,
1455 base_url);
1456 i += 51;
1457 break;
1458 }
1459 default:
1460 while (i < buf.len && data[i] != '\n')
1461 i++;
1462 }
1463 i++;
1464 }
1465
1466 cleanup:
1467 free(url);
1468 return ret;
1469 }
1470
1471 void release_http_pack_request(struct http_pack_request *preq)
1472 {
1473 if (preq->packfile != NULL) {
1474 fclose(preq->packfile);
1475 preq->packfile = NULL;
1476 }
1477 if (preq->range_header != NULL) {
1478 curl_slist_free_all(preq->range_header);
1479 preq->range_header = NULL;
1480 }
1481 preq->slot = NULL;
1482 free(preq->url);
1483 free(preq);
1484 }
1485
1486 int finish_http_pack_request(struct http_pack_request *preq)
1487 {
1488 struct packed_git **lst;
1489 struct packed_git *p = preq->target;
1490 char *tmp_idx;
1491 struct child_process ip = CHILD_PROCESS_INIT;
1492 const char *ip_argv[8];
1493
1494 close_pack_index(p);
1495
1496 fclose(preq->packfile);
1497 preq->packfile = NULL;
1498
1499 lst = preq->lst;
1500 while (*lst != p)
1501 lst = &((*lst)->next);
1502 *lst = (*lst)->next;
1503
1504 tmp_idx = xstrdup(preq->tmpfile);
1505 strcpy(tmp_idx + strlen(tmp_idx) - strlen(".pack.temp"),
1506 ".idx.temp");
1507
1508 ip_argv[0] = "index-pack";
1509 ip_argv[1] = "-o";
1510 ip_argv[2] = tmp_idx;
1511 ip_argv[3] = preq->tmpfile;
1512 ip_argv[4] = NULL;
1513
1514 ip.argv = ip_argv;
1515 ip.git_cmd = 1;
1516 ip.no_stdin = 1;
1517 ip.no_stdout = 1;
1518
1519 if (run_command(&ip)) {
1520 unlink(preq->tmpfile);
1521 unlink(tmp_idx);
1522 free(tmp_idx);
1523 return -1;
1524 }
1525
1526 unlink(sha1_pack_index_name(p->sha1));
1527
1528 if (move_temp_to_file(preq->tmpfile, sha1_pack_name(p->sha1))
1529 || move_temp_to_file(tmp_idx, sha1_pack_index_name(p->sha1))) {
1530 free(tmp_idx);
1531 return -1;
1532 }
1533
1534 install_packed_git(p);
1535 free(tmp_idx);
1536 return 0;
1537 }
1538
1539 struct http_pack_request *new_http_pack_request(
1540 struct packed_git *target, const char *base_url)
1541 {
1542 long prev_posn = 0;
1543 char range[RANGE_HEADER_SIZE];
1544 struct strbuf buf = STRBUF_INIT;
1545 struct http_pack_request *preq;
1546
1547 preq = xcalloc(1, sizeof(*preq));
1548 preq->target = target;
1549
1550 end_url_with_slash(&buf, base_url);
1551 strbuf_addf(&buf, "objects/pack/pack-%s.pack",
1552 sha1_to_hex(target->sha1));
1553 preq->url = strbuf_detach(&buf, NULL);
1554
1555 snprintf(preq->tmpfile, sizeof(preq->tmpfile), "%s.temp",
1556 sha1_pack_name(target->sha1));
1557 preq->packfile = fopen(preq->tmpfile, "a");
1558 if (!preq->packfile) {
1559 error("Unable to open local file %s for pack",
1560 preq->tmpfile);
1561 goto abort;
1562 }
1563
1564 preq->slot = get_active_slot();
1565 curl_easy_setopt(preq->slot->curl, CURLOPT_FILE, preq->packfile);
1566 curl_easy_setopt(preq->slot->curl, CURLOPT_WRITEFUNCTION, fwrite);
1567 curl_easy_setopt(preq->slot->curl, CURLOPT_URL, preq->url);
1568 curl_easy_setopt(preq->slot->curl, CURLOPT_HTTPHEADER,
1569 no_pragma_header);
1570
1571 /*
1572 * If there is data present from a previous transfer attempt,
1573 * resume where it left off
1574 */
1575 prev_posn = ftell(preq->packfile);
1576 if (prev_posn>0) {
1577 if (http_is_verbose)
1578 fprintf(stderr,
1579 "Resuming fetch of pack %s at byte %ld\n",
1580 sha1_to_hex(target->sha1), prev_posn);
1581 sprintf(range, "Range: bytes=%ld-", prev_posn);
1582 preq->range_header = curl_slist_append(NULL, range);
1583 curl_easy_setopt(preq->slot->curl, CURLOPT_HTTPHEADER,
1584 preq->range_header);
1585 }
1586
1587 return preq;
1588
1589 abort:
1590 free(preq->url);
1591 free(preq);
1592 return NULL;
1593 }
1594
1595 /* Helpers for fetching objects (loose) */
1596 static size_t fwrite_sha1_file(char *ptr, size_t eltsize, size_t nmemb,
1597 void *data)
1598 {
1599 unsigned char expn[4096];
1600 size_t size = eltsize * nmemb;
1601 int posn = 0;
1602 struct http_object_request *freq =
1603 (struct http_object_request *)data;
1604 do {
1605 ssize_t retval = xwrite(freq->localfile,
1606 (char *) ptr + posn, size - posn);
1607 if (retval < 0)
1608 return posn;
1609 posn += retval;
1610 } while (posn < size);
1611
1612 freq->stream.avail_in = size;
1613 freq->stream.next_in = (void *)ptr;
1614 do {
1615 freq->stream.next_out = expn;
1616 freq->stream.avail_out = sizeof(expn);
1617 freq->zret = git_inflate(&freq->stream, Z_SYNC_FLUSH);
1618 git_SHA1_Update(&freq->c, expn,
1619 sizeof(expn) - freq->stream.avail_out);
1620 } while (freq->stream.avail_in && freq->zret == Z_OK);
1621 return size;
1622 }
1623
1624 struct http_object_request *new_http_object_request(const char *base_url,
1625 unsigned char *sha1)
1626 {
1627 char *hex = sha1_to_hex(sha1);
1628 const char *filename;
1629 char prevfile[PATH_MAX];
1630 int prevlocal;
1631 char prev_buf[PREV_BUF_SIZE];
1632 ssize_t prev_read = 0;
1633 long prev_posn = 0;
1634 char range[RANGE_HEADER_SIZE];
1635 struct curl_slist *range_header = NULL;
1636 struct http_object_request *freq;
1637
1638 freq = xcalloc(1, sizeof(*freq));
1639 hashcpy(freq->sha1, sha1);
1640 freq->localfile = -1;
1641
1642 filename = sha1_file_name(sha1);
1643 snprintf(freq->tmpfile, sizeof(freq->tmpfile),
1644 "%s.temp", filename);
1645
1646 snprintf(prevfile, sizeof(prevfile), "%s.prev", filename);
1647 unlink_or_warn(prevfile);
1648 rename(freq->tmpfile, prevfile);
1649 unlink_or_warn(freq->tmpfile);
1650
1651 if (freq->localfile != -1)
1652 error("fd leakage in start: %d", freq->localfile);
1653 freq->localfile = open(freq->tmpfile,
1654 O_WRONLY | O_CREAT | O_EXCL, 0666);
1655 /*
1656 * This could have failed due to the "lazy directory creation";
1657 * try to mkdir the last path component.
1658 */
1659 if (freq->localfile < 0 && errno == ENOENT) {
1660 char *dir = strrchr(freq->tmpfile, '/');
1661 if (dir) {
1662 *dir = 0;
1663 mkdir(freq->tmpfile, 0777);
1664 *dir = '/';
1665 }
1666 freq->localfile = open(freq->tmpfile,
1667 O_WRONLY | O_CREAT | O_EXCL, 0666);
1668 }
1669
1670 if (freq->localfile < 0) {
1671 error("Couldn't create temporary file %s: %s",
1672 freq->tmpfile, strerror(errno));
1673 goto abort;
1674 }
1675
1676 git_inflate_init(&freq->stream);
1677
1678 git_SHA1_Init(&freq->c);
1679
1680 freq->url = get_remote_object_url(base_url, hex, 0);
1681
1682 /*
1683 * If a previous temp file is present, process what was already
1684 * fetched.
1685 */
1686 prevlocal = open(prevfile, O_RDONLY);
1687 if (prevlocal != -1) {
1688 do {
1689 prev_read = xread(prevlocal, prev_buf, PREV_BUF_SIZE);
1690 if (prev_read>0) {
1691 if (fwrite_sha1_file(prev_buf,
1692 1,
1693 prev_read,
1694 freq) == prev_read) {
1695 prev_posn += prev_read;
1696 } else {
1697 prev_read = -1;
1698 }
1699 }
1700 } while (prev_read > 0);
1701 close(prevlocal);
1702 }
1703 unlink_or_warn(prevfile);
1704
1705 /*
1706 * Reset inflate/SHA1 if there was an error reading the previous temp
1707 * file; also rewind to the beginning of the local file.
1708 */
1709 if (prev_read == -1) {
1710 memset(&freq->stream, 0, sizeof(freq->stream));
1711 git_inflate_init(&freq->stream);
1712 git_SHA1_Init(&freq->c);
1713 if (prev_posn>0) {
1714 prev_posn = 0;
1715 lseek(freq->localfile, 0, SEEK_SET);
1716 if (ftruncate(freq->localfile, 0) < 0) {
1717 error("Couldn't truncate temporary file %s: %s",
1718 freq->tmpfile, strerror(errno));
1719 goto abort;
1720 }
1721 }
1722 }
1723
1724 freq->slot = get_active_slot();
1725
1726 curl_easy_setopt(freq->slot->curl, CURLOPT_FILE, freq);
1727 curl_easy_setopt(freq->slot->curl, CURLOPT_WRITEFUNCTION, fwrite_sha1_file);
1728 curl_easy_setopt(freq->slot->curl, CURLOPT_ERRORBUFFER, freq->errorstr);
1729 curl_easy_setopt(freq->slot->curl, CURLOPT_URL, freq->url);
1730 curl_easy_setopt(freq->slot->curl, CURLOPT_HTTPHEADER, no_pragma_header);
1731
1732 /*
1733 * If we have successfully processed data from a previous fetch
1734 * attempt, only fetch the data we don't already have.
1735 */
1736 if (prev_posn>0) {
1737 if (http_is_verbose)
1738 fprintf(stderr,
1739 "Resuming fetch of object %s at byte %ld\n",
1740 hex, prev_posn);
1741 sprintf(range, "Range: bytes=%ld-", prev_posn);
1742 range_header = curl_slist_append(range_header, range);
1743 curl_easy_setopt(freq->slot->curl,
1744 CURLOPT_HTTPHEADER, range_header);
1745 }
1746
1747 return freq;
1748
1749 abort:
1750 free(freq->url);
1751 free(freq);
1752 return NULL;
1753 }
1754
1755 void process_http_object_request(struct http_object_request *freq)
1756 {
1757 if (freq->slot == NULL)
1758 return;
1759 freq->curl_result = freq->slot->curl_result;
1760 freq->http_code = freq->slot->http_code;
1761 freq->slot = NULL;
1762 }
1763
1764 int finish_http_object_request(struct http_object_request *freq)
1765 {
1766 struct stat st;
1767
1768 close(freq->localfile);
1769 freq->localfile = -1;
1770
1771 process_http_object_request(freq);
1772
1773 if (freq->http_code == 416) {
1774 warning("requested range invalid; we may already have all the data.");
1775 } else if (freq->curl_result != CURLE_OK) {
1776 if (stat(freq->tmpfile, &st) == 0)
1777 if (st.st_size == 0)
1778 unlink_or_warn(freq->tmpfile);
1779 return -1;
1780 }
1781
1782 git_inflate_end(&freq->stream);
1783 git_SHA1_Final(freq->real_sha1, &freq->c);
1784 if (freq->zret != Z_STREAM_END) {
1785 unlink_or_warn(freq->tmpfile);
1786 return -1;
1787 }
1788 if (hashcmp(freq->sha1, freq->real_sha1)) {
1789 unlink_or_warn(freq->tmpfile);
1790 return -1;
1791 }
1792 freq->rename =
1793 move_temp_to_file(freq->tmpfile, sha1_file_name(freq->sha1));
1794
1795 return freq->rename;
1796 }
1797
1798 void abort_http_object_request(struct http_object_request *freq)
1799 {
1800 unlink_or_warn(freq->tmpfile);
1801
1802 release_http_object_request(freq);
1803 }
1804
1805 void release_http_object_request(struct http_object_request *freq)
1806 {
1807 if (freq->localfile != -1) {
1808 close(freq->localfile);
1809 freq->localfile = -1;
1810 }
1811 if (freq->url != NULL) {
1812 free(freq->url);
1813 freq->url = NULL;
1814 }
1815 if (freq->slot != NULL) {
1816 freq->slot->callback_func = NULL;
1817 freq->slot->callback_data = NULL;
1818 release_active_slot(freq->slot);
1819 freq->slot = NULL;
1820 }
1821 }
The Best VCS Around