8 #include "object-store.h"
13 struct packed_git
*packs
;
14 struct alt_base
*next
;
17 enum object_request_state
{
24 struct object_request
{
25 struct walker
*walker
;
26 unsigned char sha1
[20];
27 struct alt_base
*repo
;
28 enum object_request_state state
;
29 struct http_object_request
*req
;
30 struct list_head node
;
33 struct alternates_request
{
34 struct walker
*walker
;
37 struct strbuf
*buffer
;
38 struct active_request_slot
*slot
;
48 static LIST_HEAD(object_queue_head
);
50 static void fetch_alternates(struct walker
*walker
, const char *base
);
52 static void process_object_response(void *callback_data
);
54 static void start_object_request(struct walker
*walker
,
55 struct object_request
*obj_req
)
57 struct active_request_slot
*slot
;
58 struct http_object_request
*req
;
60 req
= new_http_object_request(obj_req
->repo
->base
, obj_req
->sha1
);
62 obj_req
->state
= ABORTED
;
68 slot
->callback_func
= process_object_response
;
69 slot
->callback_data
= obj_req
;
71 /* Try to get the request started, abort the request on error */
72 obj_req
->state
= ACTIVE
;
73 if (!start_active_slot(slot
)) {
74 obj_req
->state
= ABORTED
;
75 release_http_object_request(req
);
80 static void finish_object_request(struct object_request
*obj_req
)
82 if (finish_http_object_request(obj_req
->req
))
85 if (obj_req
->req
->rename
== 0)
86 walker_say(obj_req
->walker
, "got %s\n", sha1_to_hex(obj_req
->sha1
));
89 static void process_object_response(void *callback_data
)
91 struct object_request
*obj_req
=
92 (struct object_request
*)callback_data
;
93 struct walker
*walker
= obj_req
->walker
;
94 struct walker_data
*data
= walker
->data
;
95 struct alt_base
*alt
= data
->alt
;
97 process_http_object_request(obj_req
->req
);
98 obj_req
->state
= COMPLETE
;
100 /* Use alternates if necessary */
101 if (missing_target(obj_req
->req
)) {
102 fetch_alternates(walker
, alt
->base
);
103 if (obj_req
->repo
->next
!= NULL
) {
106 release_http_object_request(obj_req
->req
);
107 start_object_request(walker
, obj_req
);
112 finish_object_request(obj_req
);
115 static void release_object_request(struct object_request
*obj_req
)
117 if (obj_req
->req
!=NULL
&& obj_req
->req
->localfile
!= -1)
118 error("fd leakage in release: %d", obj_req
->req
->localfile
);
120 list_del(&obj_req
->node
);
124 #ifdef USE_CURL_MULTI
125 static int fill_active_slot(struct walker
*walker
)
127 struct object_request
*obj_req
;
128 struct list_head
*pos
, *tmp
, *head
= &object_queue_head
;
130 list_for_each_safe(pos
, tmp
, head
) {
131 obj_req
= list_entry(pos
, struct object_request
, node
);
132 if (obj_req
->state
== WAITING
) {
133 if (has_sha1_file(obj_req
->sha1
))
134 obj_req
->state
= COMPLETE
;
136 start_object_request(walker
, obj_req
);
145 static void prefetch(struct walker
*walker
, unsigned char *sha1
)
147 struct object_request
*newreq
;
148 struct walker_data
*data
= walker
->data
;
150 newreq
= xmalloc(sizeof(*newreq
));
151 newreq
->walker
= walker
;
152 hashcpy(newreq
->sha1
, sha1
);
153 newreq
->repo
= data
->alt
;
154 newreq
->state
= WAITING
;
157 http_is_verbose
= walker
->get_verbosely
;
158 list_add_tail(&newreq
->node
, &object_queue_head
);
160 #ifdef USE_CURL_MULTI
166 static int is_alternate_allowed(const char *url
)
168 const char *protocols
[] = {
169 "http", "https", "ftp", "ftps"
173 if (http_follow_config
!= HTTP_FOLLOW_ALWAYS
) {
174 warning("alternate disabled by http.followRedirects: %s", url
);
178 for (i
= 0; i
< ARRAY_SIZE(protocols
); i
++) {
180 if (skip_prefix(url
, protocols
[i
], &end
) &&
181 starts_with(end
, "://"))
185 if (i
>= ARRAY_SIZE(protocols
)) {
186 warning("ignoring alternate with unknown protocol: %s", url
);
189 if (!is_transport_allowed(protocols
[i
], 0)) {
190 warning("ignoring alternate with restricted protocol: %s", url
);
197 static void process_alternates_response(void *callback_data
)
199 struct alternates_request
*alt_req
=
200 (struct alternates_request
*)callback_data
;
201 struct walker
*walker
= alt_req
->walker
;
202 struct walker_data
*cdata
= walker
->data
;
203 struct active_request_slot
*slot
= alt_req
->slot
;
204 struct alt_base
*tail
= cdata
->alt
;
205 const char *base
= alt_req
->base
;
206 const char null_byte
= '\0';
210 if (alt_req
->http_specific
) {
211 if (slot
->curl_result
!= CURLE_OK
||
212 !alt_req
->buffer
->len
) {
214 /* Try reusing the slot to get non-http alternates */
215 alt_req
->http_specific
= 0;
216 strbuf_reset(alt_req
->url
);
217 strbuf_addf(alt_req
->url
, "%s/objects/info/alternates",
219 curl_easy_setopt(slot
->curl
, CURLOPT_URL
,
223 if (slot
->finished
!= NULL
)
224 (*slot
->finished
) = 0;
225 if (!start_active_slot(slot
)) {
226 cdata
->got_alternates
= -1;
228 if (slot
->finished
!= NULL
)
229 (*slot
->finished
) = 1;
233 } else if (slot
->curl_result
!= CURLE_OK
) {
234 if (!missing_target(slot
)) {
235 cdata
->got_alternates
= -1;
240 fwrite_buffer((char *)&null_byte
, 1, 1, alt_req
->buffer
);
241 alt_req
->buffer
->len
--;
242 data
= alt_req
->buffer
->buf
;
244 while (i
< alt_req
->buffer
->len
) {
246 while (posn
< alt_req
->buffer
->len
&& data
[posn
] != '\n')
248 if (data
[posn
] == '\n') {
251 struct alt_base
*newalt
;
252 if (data
[i
] == '/') {
255 * http://git.host/pub/scm/linux.git/
257 * so memcpy(dst, base, serverlen) will
258 * copy up to "...git.host".
260 const char *colon_ss
= strstr(base
,"://");
262 serverlen
= (strchr(colon_ss
+ 3, '/')
266 } else if (!memcmp(data
+ i
, "../", 3)) {
268 * Relative URL; chop the corresponding
269 * number of subpath from base (and ../
270 * from data), and concatenate the result.
272 * The code first drops ../ from data, and
273 * then drops one ../ from data and one path
274 * from base. IOW, one extra ../ is dropped
275 * from data than path is dropped from base.
277 * This is not wrong. The alternate in
278 * http://git.host/pub/scm/linux.git/
280 * http://git.host/pub/scm/linus.git/
281 * is ../../linus.git/objects/. You need
282 * two ../../ to borrow from your direct
286 serverlen
= strlen(base
);
287 while (i
+ 2 < posn
&&
288 !memcmp(data
+ i
, "../", 3)) {
291 } while (serverlen
&&
292 base
[serverlen
- 1] != '/');
295 /* If the server got removed, give up. */
296 okay
= strchr(base
, ':') - base
+ 3 <
298 } else if (alt_req
->http_specific
) {
299 char *colon
= strchr(data
+ i
, ':');
300 char *slash
= strchr(data
+ i
, '/');
301 if (colon
&& slash
&& colon
< data
+ posn
&&
302 slash
< data
+ posn
&& colon
< slash
) {
307 struct strbuf target
= STRBUF_INIT
;
308 strbuf_add(&target
, base
, serverlen
);
309 strbuf_add(&target
, data
+ i
, posn
- i
);
310 if (!strbuf_strip_suffix(&target
, "objects")) {
311 warning("ignoring alternate that does"
312 " not end in 'objects': %s",
314 strbuf_release(&target
);
315 } else if (is_alternate_allowed(target
.buf
)) {
316 warning("adding alternate object store: %s",
318 newalt
= xmalloc(sizeof(*newalt
));
320 newalt
->base
= strbuf_detach(&target
, NULL
);
321 newalt
->got_indices
= 0;
322 newalt
->packs
= NULL
;
324 while (tail
->next
!= NULL
)
328 strbuf_release(&target
);
335 cdata
->got_alternates
= 1;
338 static void fetch_alternates(struct walker
*walker
, const char *base
)
340 struct strbuf buffer
= STRBUF_INIT
;
341 struct strbuf url
= STRBUF_INIT
;
342 struct active_request_slot
*slot
;
343 struct alternates_request alt_req
;
344 struct walker_data
*cdata
= walker
->data
;
347 * If another request has already started fetching alternates,
348 * wait for them to arrive and return to processing this request's
351 #ifdef USE_CURL_MULTI
352 while (cdata
->got_alternates
== 0) {
357 /* Nothing to do if they've already been fetched */
358 if (cdata
->got_alternates
== 1)
361 /* Start the fetch */
362 cdata
->got_alternates
= 0;
364 if (walker
->get_verbosely
)
365 fprintf(stderr
, "Getting alternates list for %s\n", base
);
367 strbuf_addf(&url
, "%s/objects/info/http-alternates", base
);
370 * Use a callback to process the result, since another request
371 * may fail and need to have alternates loaded before continuing
373 slot
= get_active_slot();
374 slot
->callback_func
= process_alternates_response
;
375 alt_req
.walker
= walker
;
376 slot
->callback_data
= &alt_req
;
378 curl_easy_setopt(slot
->curl
, CURLOPT_FILE
, &buffer
);
379 curl_easy_setopt(slot
->curl
, CURLOPT_WRITEFUNCTION
, fwrite_buffer
);
380 curl_easy_setopt(slot
->curl
, CURLOPT_URL
, url
.buf
);
384 alt_req
.buffer
= &buffer
;
385 alt_req
.http_specific
= 1;
388 if (start_active_slot(slot
))
389 run_active_slot(slot
);
391 cdata
->got_alternates
= -1;
393 strbuf_release(&buffer
);
394 strbuf_release(&url
);
397 static int fetch_indices(struct walker
*walker
, struct alt_base
*repo
)
401 if (repo
->got_indices
)
404 if (walker
->get_verbosely
)
405 fprintf(stderr
, "Getting pack list for %s\n", repo
->base
);
407 switch (http_get_info_packs(repo
->base
, &repo
->packs
)) {
409 case HTTP_MISSING_TARGET
:
410 repo
->got_indices
= 1;
414 repo
->got_indices
= 0;
421 static int http_fetch_pack(struct walker
*walker
, struct alt_base
*repo
, unsigned char *sha1
)
423 struct packed_git
*target
;
425 struct slot_results results
;
426 struct http_pack_request
*preq
;
428 if (fetch_indices(walker
, repo
))
430 target
= find_sha1_pack(sha1
, repo
->packs
);
434 if (walker
->get_verbosely
) {
435 fprintf(stderr
, "Getting pack %s\n",
436 sha1_to_hex(target
->sha1
));
437 fprintf(stderr
, " which contains %s\n",
441 preq
= new_http_pack_request(target
, repo
->base
);
444 preq
->lst
= &repo
->packs
;
445 preq
->slot
->results
= &results
;
447 if (start_active_slot(preq
->slot
)) {
448 run_active_slot(preq
->slot
);
449 if (results
.curl_result
!= CURLE_OK
) {
450 error("Unable to get pack file %s\n%s", preq
->url
,
455 error("Unable to start request");
459 ret
= finish_http_pack_request(preq
);
460 release_http_pack_request(preq
);
470 static void abort_object_request(struct object_request
*obj_req
)
472 release_object_request(obj_req
);
475 static int fetch_object(struct walker
*walker
, unsigned char *sha1
)
477 char *hex
= sha1_to_hex(sha1
);
479 struct object_request
*obj_req
= NULL
;
480 struct http_object_request
*req
;
481 struct list_head
*pos
, *head
= &object_queue_head
;
483 list_for_each(pos
, head
) {
484 obj_req
= list_entry(pos
, struct object_request
, node
);
485 if (!hashcmp(obj_req
->sha1
, sha1
))
489 return error("Couldn't find request for %s in the queue", hex
);
491 if (has_sha1_file(obj_req
->sha1
)) {
492 if (obj_req
->req
!= NULL
)
493 abort_http_object_request(obj_req
->req
);
494 abort_object_request(obj_req
);
498 #ifdef USE_CURL_MULTI
499 while (obj_req
->state
== WAITING
)
502 start_object_request(walker
, obj_req
);
506 * obj_req->req might change when fetching alternates in the callback
507 * process_object_response; therefore, the "shortcut" variable, req,
508 * is used only after we're done with slots.
510 while (obj_req
->state
== ACTIVE
)
511 run_active_slot(obj_req
->req
->slot
);
515 if (req
->localfile
!= -1) {
516 close(req
->localfile
);
521 * we turned off CURLOPT_FAILONERROR to avoid losing a
522 * persistent connection and got CURLE_OK.
524 if (req
->http_code
>= 300 && req
->curl_result
== CURLE_OK
&&
525 (starts_with(req
->url
, "http://") ||
526 starts_with(req
->url
, "https://"))) {
527 req
->curl_result
= CURLE_HTTP_RETURNED_ERROR
;
528 xsnprintf(req
->errorstr
, sizeof(req
->errorstr
),
529 "HTTP request failed");
532 if (obj_req
->state
== ABORTED
) {
533 ret
= error("Request for %s aborted", hex
);
534 } else if (req
->curl_result
!= CURLE_OK
&&
535 req
->http_code
!= 416) {
536 if (missing_target(req
))
537 ret
= -1; /* Be silent, it is probably in a pack. */
539 ret
= error("%s (curl_result = %d, http_code = %ld, sha1 = %s)",
540 req
->errorstr
, req
->curl_result
,
541 req
->http_code
, hex
);
542 } else if (req
->zret
!= Z_STREAM_END
) {
543 walker
->corrupt_object_found
++;
544 ret
= error("File %s (%s) corrupt", hex
, req
->url
);
545 } else if (hashcmp(obj_req
->sha1
, req
->real_sha1
)) {
546 ret
= error("File %s has bad hash", hex
);
547 } else if (req
->rename
< 0) {
548 struct strbuf buf
= STRBUF_INIT
;
549 sha1_file_name(&buf
, req
->sha1
);
550 ret
= error("unable to write sha1 filename %s", buf
.buf
);
551 strbuf_release(&buf
);
554 release_http_object_request(req
);
555 release_object_request(obj_req
);
559 static int fetch(struct walker
*walker
, unsigned char *sha1
)
561 struct walker_data
*data
= walker
->data
;
562 struct alt_base
*altbase
= data
->alt
;
564 if (!fetch_object(walker
, sha1
))
567 if (!http_fetch_pack(walker
, altbase
, sha1
))
569 fetch_alternates(walker
, data
->alt
->base
);
570 altbase
= altbase
->next
;
572 return error("Unable to find %s under %s", sha1_to_hex(sha1
),
576 static int fetch_ref(struct walker
*walker
, struct ref
*ref
)
578 struct walker_data
*data
= walker
->data
;
579 return http_fetch_ref(data
->alt
->base
, ref
);
582 static void cleanup(struct walker
*walker
)
584 struct walker_data
*data
= walker
->data
;
585 struct alt_base
*alt
, *alt_next
;
590 alt_next
= alt
->next
;
602 struct walker
*get_http_walker(const char *url
)
605 struct walker_data
*data
= xmalloc(sizeof(struct walker_data
));
606 struct walker
*walker
= xmalloc(sizeof(struct walker
));
608 data
->alt
= xmalloc(sizeof(*data
->alt
));
609 data
->alt
->base
= xstrdup(url
);
610 for (s
= data
->alt
->base
+ strlen(data
->alt
->base
) - 1; *s
== '/'; --s
)
613 data
->alt
->got_indices
= 0;
614 data
->alt
->packs
= NULL
;
615 data
->alt
->next
= NULL
;
616 data
->got_alternates
= -1;
618 walker
->corrupt_object_found
= 0;
619 walker
->fetch
= fetch
;
620 walker
->fetch_ref
= fetch_ref
;
621 walker
->prefetch
= prefetch
;
622 walker
->cleanup
= cleanup
;
625 #ifdef USE_CURL_MULTI
626 add_fill_function(walker
, (int (*)(void *)) fill_active_slot
);