11 struct packed_git
*packs
;
12 struct alt_base
*next
;
15 enum object_request_state
{
22 struct object_request
{
23 struct walker
*walker
;
24 unsigned char sha1
[20];
25 struct alt_base
*repo
;
26 enum object_request_state state
;
27 struct http_object_request
*req
;
28 struct list_head node
;
31 struct alternates_request
{
32 struct walker
*walker
;
35 struct strbuf
*buffer
;
36 struct active_request_slot
*slot
;
46 static LIST_HEAD(object_queue_head
);
48 static void fetch_alternates(struct walker
*walker
, const char *base
);
50 static void process_object_response(void *callback_data
);
52 static void start_object_request(struct walker
*walker
,
53 struct object_request
*obj_req
)
55 struct active_request_slot
*slot
;
56 struct http_object_request
*req
;
58 req
= new_http_object_request(obj_req
->repo
->base
, obj_req
->sha1
);
60 obj_req
->state
= ABORTED
;
66 slot
->callback_func
= process_object_response
;
67 slot
->callback_data
= obj_req
;
69 /* Try to get the request started, abort the request on error */
70 obj_req
->state
= ACTIVE
;
71 if (!start_active_slot(slot
)) {
72 obj_req
->state
= ABORTED
;
73 release_http_object_request(req
);
78 static void finish_object_request(struct object_request
*obj_req
)
80 if (finish_http_object_request(obj_req
->req
))
83 if (obj_req
->req
->rename
== 0)
84 walker_say(obj_req
->walker
, "got %s\n", sha1_to_hex(obj_req
->sha1
));
87 static void process_object_response(void *callback_data
)
89 struct object_request
*obj_req
=
90 (struct object_request
*)callback_data
;
91 struct walker
*walker
= obj_req
->walker
;
92 struct walker_data
*data
= walker
->data
;
93 struct alt_base
*alt
= data
->alt
;
95 process_http_object_request(obj_req
->req
);
96 obj_req
->state
= COMPLETE
;
98 /* Use alternates if necessary */
99 if (missing_target(obj_req
->req
)) {
100 fetch_alternates(walker
, alt
->base
);
101 if (obj_req
->repo
->next
!= NULL
) {
104 release_http_object_request(obj_req
->req
);
105 start_object_request(walker
, obj_req
);
110 finish_object_request(obj_req
);
113 static void release_object_request(struct object_request
*obj_req
)
115 if (obj_req
->req
!=NULL
&& obj_req
->req
->localfile
!= -1)
116 error("fd leakage in release: %d", obj_req
->req
->localfile
);
118 list_del(&obj_req
->node
);
122 #ifdef USE_CURL_MULTI
123 static int fill_active_slot(struct walker
*walker
)
125 struct object_request
*obj_req
;
126 struct list_head
*pos
, *tmp
, *head
= &object_queue_head
;
128 list_for_each_safe(pos
, tmp
, head
) {
129 obj_req
= list_entry(pos
, struct object_request
, node
);
130 if (obj_req
->state
== WAITING
) {
131 if (has_sha1_file(obj_req
->sha1
))
132 obj_req
->state
= COMPLETE
;
134 start_object_request(walker
, obj_req
);
143 static void prefetch(struct walker
*walker
, unsigned char *sha1
)
145 struct object_request
*newreq
;
146 struct walker_data
*data
= walker
->data
;
148 newreq
= xmalloc(sizeof(*newreq
));
149 newreq
->walker
= walker
;
150 hashcpy(newreq
->sha1
, sha1
);
151 newreq
->repo
= data
->alt
;
152 newreq
->state
= WAITING
;
155 http_is_verbose
= walker
->get_verbosely
;
156 list_add_tail(&newreq
->node
, &object_queue_head
);
158 #ifdef USE_CURL_MULTI
164 static int is_alternate_allowed(const char *url
)
166 const char *protocols
[] = {
167 "http", "https", "ftp", "ftps"
171 if (http_follow_config
!= HTTP_FOLLOW_ALWAYS
) {
172 warning("alternate disabled by http.followRedirects: %s", url
);
176 for (i
= 0; i
< ARRAY_SIZE(protocols
); i
++) {
178 if (skip_prefix(url
, protocols
[i
], &end
) &&
179 starts_with(end
, "://"))
183 if (i
>= ARRAY_SIZE(protocols
)) {
184 warning("ignoring alternate with unknown protocol: %s", url
);
187 if (!is_transport_allowed(protocols
[i
], 0)) {
188 warning("ignoring alternate with restricted protocol: %s", url
);
195 static void process_alternates_response(void *callback_data
)
197 struct alternates_request
*alt_req
=
198 (struct alternates_request
*)callback_data
;
199 struct walker
*walker
= alt_req
->walker
;
200 struct walker_data
*cdata
= walker
->data
;
201 struct active_request_slot
*slot
= alt_req
->slot
;
202 struct alt_base
*tail
= cdata
->alt
;
203 const char *base
= alt_req
->base
;
204 const char null_byte
= '\0';
208 if (alt_req
->http_specific
) {
209 if (slot
->curl_result
!= CURLE_OK
||
210 !alt_req
->buffer
->len
) {
212 /* Try reusing the slot to get non-http alternates */
213 alt_req
->http_specific
= 0;
214 strbuf_reset(alt_req
->url
);
215 strbuf_addf(alt_req
->url
, "%s/objects/info/alternates",
217 curl_easy_setopt(slot
->curl
, CURLOPT_URL
,
221 if (slot
->finished
!= NULL
)
222 (*slot
->finished
) = 0;
223 if (!start_active_slot(slot
)) {
224 cdata
->got_alternates
= -1;
226 if (slot
->finished
!= NULL
)
227 (*slot
->finished
) = 1;
231 } else if (slot
->curl_result
!= CURLE_OK
) {
232 if (!missing_target(slot
)) {
233 cdata
->got_alternates
= -1;
238 fwrite_buffer((char *)&null_byte
, 1, 1, alt_req
->buffer
);
239 alt_req
->buffer
->len
--;
240 data
= alt_req
->buffer
->buf
;
242 while (i
< alt_req
->buffer
->len
) {
244 while (posn
< alt_req
->buffer
->len
&& data
[posn
] != '\n')
246 if (data
[posn
] == '\n') {
249 struct alt_base
*newalt
;
250 if (data
[i
] == '/') {
253 * http://git.host/pub/scm/linux.git/
255 * so memcpy(dst, base, serverlen) will
256 * copy up to "...git.host".
258 const char *colon_ss
= strstr(base
,"://");
260 serverlen
= (strchr(colon_ss
+ 3, '/')
264 } else if (!memcmp(data
+ i
, "../", 3)) {
266 * Relative URL; chop the corresponding
267 * number of subpath from base (and ../
268 * from data), and concatenate the result.
270 * The code first drops ../ from data, and
271 * then drops one ../ from data and one path
272 * from base. IOW, one extra ../ is dropped
273 * from data than path is dropped from base.
275 * This is not wrong. The alternate in
276 * http://git.host/pub/scm/linux.git/
278 * http://git.host/pub/scm/linus.git/
279 * is ../../linus.git/objects/. You need
280 * two ../../ to borrow from your direct
284 serverlen
= strlen(base
);
285 while (i
+ 2 < posn
&&
286 !memcmp(data
+ i
, "../", 3)) {
289 } while (serverlen
&&
290 base
[serverlen
- 1] != '/');
293 /* If the server got removed, give up. */
294 okay
= strchr(base
, ':') - base
+ 3 <
296 } else if (alt_req
->http_specific
) {
297 char *colon
= strchr(data
+ i
, ':');
298 char *slash
= strchr(data
+ i
, '/');
299 if (colon
&& slash
&& colon
< data
+ posn
&&
300 slash
< data
+ posn
&& colon
< slash
) {
304 /* skip "objects\n" at end */
306 struct strbuf target
= STRBUF_INIT
;
307 strbuf_add(&target
, base
, serverlen
);
308 strbuf_add(&target
, data
+ i
, posn
- i
- 7);
310 if (is_alternate_allowed(target
.buf
)) {
311 warning("adding alternate object store: %s",
313 newalt
= xmalloc(sizeof(*newalt
));
315 newalt
->base
= strbuf_detach(&target
, NULL
);
316 newalt
->got_indices
= 0;
317 newalt
->packs
= NULL
;
319 while (tail
->next
!= NULL
)
323 strbuf_release(&target
);
330 cdata
->got_alternates
= 1;
333 static void fetch_alternates(struct walker
*walker
, const char *base
)
335 struct strbuf buffer
= STRBUF_INIT
;
336 struct strbuf url
= STRBUF_INIT
;
337 struct active_request_slot
*slot
;
338 struct alternates_request alt_req
;
339 struct walker_data
*cdata
= walker
->data
;
342 * If another request has already started fetching alternates,
343 * wait for them to arrive and return to processing this request's
346 #ifdef USE_CURL_MULTI
347 while (cdata
->got_alternates
== 0) {
352 /* Nothing to do if they've already been fetched */
353 if (cdata
->got_alternates
== 1)
356 /* Start the fetch */
357 cdata
->got_alternates
= 0;
359 if (walker
->get_verbosely
)
360 fprintf(stderr
, "Getting alternates list for %s\n", base
);
362 strbuf_addf(&url
, "%s/objects/info/http-alternates", base
);
365 * Use a callback to process the result, since another request
366 * may fail and need to have alternates loaded before continuing
368 slot
= get_active_slot();
369 slot
->callback_func
= process_alternates_response
;
370 alt_req
.walker
= walker
;
371 slot
->callback_data
= &alt_req
;
373 curl_easy_setopt(slot
->curl
, CURLOPT_FILE
, &buffer
);
374 curl_easy_setopt(slot
->curl
, CURLOPT_WRITEFUNCTION
, fwrite_buffer
);
375 curl_easy_setopt(slot
->curl
, CURLOPT_URL
, url
.buf
);
379 alt_req
.buffer
= &buffer
;
380 alt_req
.http_specific
= 1;
383 if (start_active_slot(slot
))
384 run_active_slot(slot
);
386 cdata
->got_alternates
= -1;
388 strbuf_release(&buffer
);
389 strbuf_release(&url
);
392 static int fetch_indices(struct walker
*walker
, struct alt_base
*repo
)
396 if (repo
->got_indices
)
399 if (walker
->get_verbosely
)
400 fprintf(stderr
, "Getting pack list for %s\n", repo
->base
);
402 switch (http_get_info_packs(repo
->base
, &repo
->packs
)) {
404 case HTTP_MISSING_TARGET
:
405 repo
->got_indices
= 1;
409 repo
->got_indices
= 0;
416 static int http_fetch_pack(struct walker
*walker
, struct alt_base
*repo
, unsigned char *sha1
)
418 struct packed_git
*target
;
420 struct slot_results results
;
421 struct http_pack_request
*preq
;
423 if (fetch_indices(walker
, repo
))
425 target
= find_sha1_pack(sha1
, repo
->packs
);
429 if (walker
->get_verbosely
) {
430 fprintf(stderr
, "Getting pack %s\n",
431 sha1_to_hex(target
->sha1
));
432 fprintf(stderr
, " which contains %s\n",
436 preq
= new_http_pack_request(target
, repo
->base
);
439 preq
->lst
= &repo
->packs
;
440 preq
->slot
->results
= &results
;
442 if (start_active_slot(preq
->slot
)) {
443 run_active_slot(preq
->slot
);
444 if (results
.curl_result
!= CURLE_OK
) {
445 error("Unable to get pack file %s\n%s", preq
->url
,
450 error("Unable to start request");
454 ret
= finish_http_pack_request(preq
);
455 release_http_pack_request(preq
);
465 static void abort_object_request(struct object_request
*obj_req
)
467 release_object_request(obj_req
);
470 static int fetch_object(struct walker
*walker
, unsigned char *sha1
)
472 char *hex
= sha1_to_hex(sha1
);
474 struct object_request
*obj_req
= NULL
;
475 struct http_object_request
*req
;
476 struct list_head
*pos
, *head
= &object_queue_head
;
478 list_for_each(pos
, head
) {
479 obj_req
= list_entry(pos
, struct object_request
, node
);
480 if (!hashcmp(obj_req
->sha1
, sha1
))
484 return error("Couldn't find request for %s in the queue", hex
);
486 if (has_sha1_file(obj_req
->sha1
)) {
487 if (obj_req
->req
!= NULL
)
488 abort_http_object_request(obj_req
->req
);
489 abort_object_request(obj_req
);
493 #ifdef USE_CURL_MULTI
494 while (obj_req
->state
== WAITING
)
497 start_object_request(walker
, obj_req
);
501 * obj_req->req might change when fetching alternates in the callback
502 * process_object_response; therefore, the "shortcut" variable, req,
503 * is used only after we're done with slots.
505 while (obj_req
->state
== ACTIVE
)
506 run_active_slot(obj_req
->req
->slot
);
510 if (req
->localfile
!= -1) {
511 close(req
->localfile
);
516 * we turned off CURLOPT_FAILONERROR to avoid losing a
517 * persistent connection and got CURLE_OK.
519 if (req
->http_code
>= 300 && req
->curl_result
== CURLE_OK
&&
520 (starts_with(req
->url
, "http://") ||
521 starts_with(req
->url
, "https://"))) {
522 req
->curl_result
= CURLE_HTTP_RETURNED_ERROR
;
523 xsnprintf(req
->errorstr
, sizeof(req
->errorstr
),
524 "HTTP request failed");
527 if (obj_req
->state
== ABORTED
) {
528 ret
= error("Request for %s aborted", hex
);
529 } else if (req
->curl_result
!= CURLE_OK
&&
530 req
->http_code
!= 416) {
531 if (missing_target(req
))
532 ret
= -1; /* Be silent, it is probably in a pack. */
534 ret
= error("%s (curl_result = %d, http_code = %ld, sha1 = %s)",
535 req
->errorstr
, req
->curl_result
,
536 req
->http_code
, hex
);
537 } else if (req
->zret
!= Z_STREAM_END
) {
538 walker
->corrupt_object_found
++;
539 ret
= error("File %s (%s) corrupt", hex
, req
->url
);
540 } else if (hashcmp(obj_req
->sha1
, req
->real_sha1
)) {
541 ret
= error("File %s has bad hash", hex
);
542 } else if (req
->rename
< 0) {
543 ret
= error("unable to write sha1 filename %s",
544 sha1_file_name(req
->sha1
));
547 release_http_object_request(req
);
548 release_object_request(obj_req
);
552 static int fetch(struct walker
*walker
, unsigned char *sha1
)
554 struct walker_data
*data
= walker
->data
;
555 struct alt_base
*altbase
= data
->alt
;
557 if (!fetch_object(walker
, sha1
))
560 if (!http_fetch_pack(walker
, altbase
, sha1
))
562 fetch_alternates(walker
, data
->alt
->base
);
563 altbase
= altbase
->next
;
565 return error("Unable to find %s under %s", sha1_to_hex(sha1
),
569 static int fetch_ref(struct walker
*walker
, struct ref
*ref
)
571 struct walker_data
*data
= walker
->data
;
572 return http_fetch_ref(data
->alt
->base
, ref
);
575 static void cleanup(struct walker
*walker
)
577 struct walker_data
*data
= walker
->data
;
578 struct alt_base
*alt
, *alt_next
;
583 alt_next
= alt
->next
;
595 struct walker
*get_http_walker(const char *url
)
598 struct walker_data
*data
= xmalloc(sizeof(struct walker_data
));
599 struct walker
*walker
= xmalloc(sizeof(struct walker
));
601 data
->alt
= xmalloc(sizeof(*data
->alt
));
602 data
->alt
->base
= xstrdup(url
);
603 for (s
= data
->alt
->base
+ strlen(data
->alt
->base
) - 1; *s
== '/'; --s
)
606 data
->alt
->got_indices
= 0;
607 data
->alt
->packs
= NULL
;
608 data
->alt
->next
= NULL
;
609 data
->got_alternates
= -1;
611 walker
->corrupt_object_found
= 0;
612 walker
->fetch
= fetch
;
613 walker
->fetch_ref
= fetch_ref
;
614 walker
->prefetch
= prefetch
;
615 walker
->cleanup
= cleanup
;
618 #ifdef USE_CURL_MULTI
619 add_fill_function(walker
, (int (*)(void *)) fill_active_slot
);