1 /* Copyright (c) 2001-2004, Roger Dingledine.
2 * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
3 * Copyright (c) 2007-2017, The Tor Project, Inc. */
4 /* See LICENSE for licensing information */
6 #define DIRECTORY_PRIVATE
12 #include "circuitbuild.h"
14 #include "connection.h"
15 #include "connection_edge.h"
16 #include "conscache.h"
18 #include "consdiffmgr.h"
21 #include "directory.h"
24 #include "entrynodes.h"
27 #include "hs_common.h"
28 #include "hs_client.h"
30 #include "microdesc.h"
31 #include "networkstatus.h"
35 #include "rendclient.h"
36 #include "rendcommon.h"
37 #include "rendservice.h"
40 #include "routerlist.h"
41 #include "routerparse.h"
42 #include "routerset.h"
43 #include "shared_random.h"
45 #if defined(EXPORTMALLINFO) && defined(HAVE_MALLOC_H) && defined(HAVE_MALLINFO)
53 * \brief Code to send and fetch information from directory authorities and
56 * Directory caches and authorities use dirserv.c to generate the results of a
57 * query and stream them to the connection; clients use routerparse.c to parse
60 * Every directory request has a dir_connection_t on the client side and on
61 * the server side. In most cases, the dir_connection_t object is a linked
62 * connection, tunneled through an edge_connection_t so that it can be a
63 * stream on the Tor network. The only non-tunneled connections are those
64 * that are used to upload material (descriptors and votes) to authorities.
65 * Among tunneled connections, some use one-hop circuits, and others use
66 * multi-hop circuits for anonymity.
68 * Directory requests are launched by calling
69 * directory_initiate_request(). This
70 * launch the connection, will construct an HTTP request with
71 * directory_send_command(), send the and wait for a response. The client
72 * later handles the response with connection_dir_client_reached_eof(),
73 * which passes the information received to another part of Tor.
75 * On the server side, requests are read in directory_handle_command(),
76 * which dispatches first on the request type (GET or POST), and then on
77 * the URL requested. GET requests are processed with a table-based
78 * dispatcher in url_table[]. The process of handling larger GET requests
79 * is complicated because we need to avoid allocating a copy of all the
80 * data to be sent to the client in one huge buffer. Instead, we spool the
81 * data into the buffer using logic in connection_dirserv_flushed_some() in
82 * dirserv.c. (TODO: If we extended buf.c to have a zero-copy
83 * reference-based buffer type, we could remove most of that code, at the
84 * cost of a bit more reference counting.)
87 /* In-points to directory.c:
89 * - directory_post_to_dirservers(), called from
90 * router_upload_dir_desc_to_dirservers() in router.c
91 * upload_service_descriptor() in rendservice.c
92 * - directory_get_from_dirserver(), called from
93 * rend_client_refetch_renddesc() in rendclient.c
94 * run_scheduled_events() in main.c
96 * - connection_dir_process_inbuf(), called from
97 * connection_process_inbuf() in connection.c
98 * - connection_dir_finished_flushing(), called from
99 * connection_finished_flushing() in connection.c
100 * - connection_dir_finished_connecting(), called from
101 * connection_finished_connecting() in connection.c
103 static void directory_send_command(dir_connection_t
*conn
,
105 const directory_request_t
*request
);
106 static int body_is_plausible(const char *body
, size_t body_len
, int purpose
);
107 static void http_set_address_origin(const char *headers
, connection_t
*conn
);
108 static void connection_dir_download_routerdesc_failed(dir_connection_t
*conn
);
109 static void connection_dir_bridge_routerdesc_failed(dir_connection_t
*conn
);
110 static void connection_dir_download_cert_failed(
111 dir_connection_t
*conn
, int status_code
);
112 static void connection_dir_retry_bridges(smartlist_t
*descs
);
113 static void dir_routerdesc_download_failed(smartlist_t
*failed
,
117 int was_descriptor_digests
);
118 static void dir_microdesc_download_failed(smartlist_t
*failed
,
121 static int client_likes_consensus(const struct consensus_cache_entry_t
*ent
,
122 const char *want_url
);
124 static void connection_dir_close_consensus_fetches(
125 dir_connection_t
*except_this_one
, const char *resource
);
127 /********* START VARIABLES **********/
129 /** How far in the future do we allow a directory server to tell us it is
130 * before deciding that one of us has the wrong time? */
131 #define ALLOW_DIRECTORY_TIME_SKEW (30*60)
133 #define X_ADDRESS_HEADER "X-Your-Address-Is: "
134 #define X_OR_DIFF_FROM_CONSENSUS_HEADER "X-Or-Diff-From-Consensus: "
136 /** HTTP cache control: how long do we tell proxies they can cache each
137 * kind of document we serve? */
138 #define FULL_DIR_CACHE_LIFETIME (60*60)
139 #define RUNNINGROUTERS_CACHE_LIFETIME (20*60)
140 #define DIRPORTFRONTPAGE_CACHE_LIFETIME (20*60)
141 #define NETWORKSTATUS_CACHE_LIFETIME (5*60)
142 #define ROUTERDESC_CACHE_LIFETIME (30*60)
143 #define ROUTERDESC_BY_DIGEST_CACHE_LIFETIME (48*60*60)
144 #define ROBOTS_CACHE_LIFETIME (24*60*60)
145 #define MICRODESC_CACHE_LIFETIME (48*60*60)
147 /********* END VARIABLES ************/
149 /** Return false if the directory purpose <b>dir_purpose</b>
150 * does not require an anonymous (three-hop) connection.
152 * Return true 1) by default, 2) if all directory actions have
153 * specifically been configured to be over an anonymous connection,
154 * or 3) if the router is a bridge */
156 purpose_needs_anonymity(uint8_t dir_purpose
, uint8_t router_purpose
,
157 const char *resource
)
159 if (get_options()->AllDirActionsPrivate
)
162 if (router_purpose
== ROUTER_PURPOSE_BRIDGE
) {
163 if (dir_purpose
== DIR_PURPOSE_FETCH_SERVERDESC
164 && resource
&& !strcmp(resource
, "authority.z")) {
165 /* We are asking a bridge for its own descriptor. That doesn't need
169 /* Assume all other bridge stuff needs anonymity. */
170 return 1; /* if no circuits yet, this might break bootstrapping, but it's
171 * needed to be safe. */
176 case DIR_PURPOSE_UPLOAD_DIR
:
177 case DIR_PURPOSE_UPLOAD_VOTE
:
178 case DIR_PURPOSE_UPLOAD_SIGNATURES
:
179 case DIR_PURPOSE_FETCH_STATUS_VOTE
:
180 case DIR_PURPOSE_FETCH_DETACHED_SIGNATURES
:
181 case DIR_PURPOSE_FETCH_CONSENSUS
:
182 case DIR_PURPOSE_FETCH_CERTIFICATE
:
183 case DIR_PURPOSE_FETCH_SERVERDESC
:
184 case DIR_PURPOSE_FETCH_EXTRAINFO
:
185 case DIR_PURPOSE_FETCH_MICRODESC
:
187 case DIR_PURPOSE_HAS_FETCHED_HSDESC
:
188 case DIR_PURPOSE_HAS_FETCHED_RENDDESC_V2
:
189 case DIR_PURPOSE_UPLOAD_RENDDESC_V2
:
190 case DIR_PURPOSE_FETCH_RENDDESC_V2
:
191 case DIR_PURPOSE_FETCH_HSDESC
:
192 case DIR_PURPOSE_UPLOAD_HSDESC
:
194 case DIR_PURPOSE_SERVER
:
196 log_warn(LD_BUG
, "Called with dir_purpose=%d, router_purpose=%d",
197 dir_purpose
, router_purpose
);
198 tor_assert_nonfatal_unreached();
199 return 1; /* Assume it needs anonymity; better safe than sorry. */
203 /** Return a newly allocated string describing <b>auth</b>. Only describes
204 * authority features. */
206 authdir_type_to_string(dirinfo_type_t auth
)
209 smartlist_t
*lst
= smartlist_new();
210 if (auth
& V3_DIRINFO
)
211 smartlist_add(lst
, (void*)"V3");
212 if (auth
& BRIDGE_DIRINFO
)
213 smartlist_add(lst
, (void*)"Bridge");
214 if (smartlist_len(lst
)) {
215 result
= smartlist_join_strings(lst
, ", ", 0, NULL
);
217 result
= tor_strdup("[Not an authority]");
223 /** Return a string describing a given directory connection purpose. */
225 dir_conn_purpose_to_string(int purpose
)
229 case DIR_PURPOSE_UPLOAD_DIR
:
230 return "server descriptor upload";
231 case DIR_PURPOSE_UPLOAD_VOTE
:
232 return "server vote upload";
233 case DIR_PURPOSE_UPLOAD_SIGNATURES
:
234 return "consensus signature upload";
235 case DIR_PURPOSE_FETCH_SERVERDESC
:
236 return "server descriptor fetch";
237 case DIR_PURPOSE_FETCH_EXTRAINFO
:
238 return "extra-info fetch";
239 case DIR_PURPOSE_FETCH_CONSENSUS
:
240 return "consensus network-status fetch";
241 case DIR_PURPOSE_FETCH_CERTIFICATE
:
242 return "authority cert fetch";
243 case DIR_PURPOSE_FETCH_STATUS_VOTE
:
244 return "status vote fetch";
245 case DIR_PURPOSE_FETCH_DETACHED_SIGNATURES
:
246 return "consensus signature fetch";
247 case DIR_PURPOSE_FETCH_RENDDESC_V2
:
248 return "hidden-service v2 descriptor fetch";
249 case DIR_PURPOSE_UPLOAD_RENDDESC_V2
:
250 return "hidden-service v2 descriptor upload";
251 case DIR_PURPOSE_FETCH_HSDESC
:
252 return "hidden-service descriptor fetch";
253 case DIR_PURPOSE_UPLOAD_HSDESC
:
254 return "hidden-service descriptor upload";
255 case DIR_PURPOSE_FETCH_MICRODESC
:
256 return "microdescriptor fetch";
259 log_warn(LD_BUG
, "Called with unknown purpose %d", purpose
);
263 /** Return the requisite directory information types. */
264 STATIC dirinfo_type_t
265 dir_fetch_type(int dir_purpose
, int router_purpose
, const char *resource
)
268 switch (dir_purpose
) {
269 case DIR_PURPOSE_FETCH_EXTRAINFO
:
270 type
= EXTRAINFO_DIRINFO
;
271 if (router_purpose
== ROUTER_PURPOSE_BRIDGE
)
272 type
|= BRIDGE_DIRINFO
;
276 case DIR_PURPOSE_FETCH_SERVERDESC
:
277 if (router_purpose
== ROUTER_PURPOSE_BRIDGE
)
278 type
= BRIDGE_DIRINFO
;
282 case DIR_PURPOSE_FETCH_STATUS_VOTE
:
283 case DIR_PURPOSE_FETCH_DETACHED_SIGNATURES
:
284 case DIR_PURPOSE_FETCH_CERTIFICATE
:
287 case DIR_PURPOSE_FETCH_CONSENSUS
:
289 if (resource
&& !strcmp(resource
, "microdesc"))
290 type
|= MICRODESC_DIRINFO
;
292 case DIR_PURPOSE_FETCH_MICRODESC
:
293 type
= MICRODESC_DIRINFO
;
296 log_warn(LD_BUG
, "Unexpected purpose %d", (int)dir_purpose
);
303 /** Return true iff <b>identity_digest</b> is the digest of a router which
304 * says that it caches extrainfos. (If <b>is_authority</b> we always
305 * believe that to be true.) */
307 router_supports_extrainfo(const char *identity_digest
, int is_authority
)
309 const node_t
*node
= node_get_by_id(identity_digest
);
311 if (node
&& node
->ri
) {
312 if (node
->ri
->caches_extra_info
)
321 /** Return true iff any trusted directory authority has accepted our
324 * We consider any authority sufficient because waiting for all of
325 * them means it never happens while any authority is down; we don't
326 * go for something more complex in the middle (like \>1/3 or \>1/2 or
327 * \>=1/2) because that doesn't seem necessary yet.
330 directories_have_accepted_server_descriptor(void)
332 const smartlist_t
*servers
= router_get_trusted_dir_servers();
333 const or_options_t
*options
= get_options();
334 SMARTLIST_FOREACH(servers
, dir_server_t
*, d
, {
335 if ((d
->type
& options
->PublishServerDescriptor_
) &&
336 d
->has_accepted_serverdesc
) {
343 /** Start a connection to every suitable directory authority, using
344 * connection purpose <b>dir_purpose</b> and uploading <b>payload</b>
345 * (of length <b>payload_len</b>). The dir_purpose should be one of
346 * 'DIR_PURPOSE_UPLOAD_{DIR|VOTE|SIGNATURES}'.
348 * <b>router_purpose</b> describes the type of descriptor we're
349 * publishing, if we're publishing a descriptor -- e.g. general or bridge.
351 * <b>type</b> specifies what sort of dir authorities (V3,
352 * BRIDGE, etc) we should upload to.
354 * If <b>extrainfo_len</b> is nonzero, the first <b>payload_len</b> bytes of
355 * <b>payload</b> hold a router descriptor, and the next <b>extrainfo_len</b>
356 * bytes of <b>payload</b> hold an extra-info document. Upload the descriptor
357 * to all authorities, and the extra-info document to all authorities that
361 directory_post_to_dirservers(uint8_t dir_purpose
, uint8_t router_purpose
,
364 size_t payload_len
, size_t extrainfo_len
)
366 const or_options_t
*options
= get_options();
367 dir_indirection_t indirection
;
368 const smartlist_t
*dirservers
= router_get_trusted_dir_servers();
370 const int exclude_self
= (dir_purpose
== DIR_PURPOSE_UPLOAD_VOTE
||
371 dir_purpose
== DIR_PURPOSE_UPLOAD_SIGNATURES
);
372 tor_assert(dirservers
);
373 /* This tries dirservers which we believe to be down, but ultimately, that's
374 * harmless, and we may as well err on the side of getting things uploaded.
376 SMARTLIST_FOREACH_BEGIN(dirservers
, dir_server_t
*, ds
) {
377 routerstatus_t
*rs
= &(ds
->fake_status
);
378 size_t upload_len
= payload_len
;
380 if ((type
& ds
->type
) == 0)
383 if (exclude_self
&& router_digest_is_me(ds
->digest
)) {
384 /* we don't upload to ourselves, but at least there's now at least
385 * one authority of this type that has what we wanted to upload. */
390 if (options
->StrictNodes
&&
391 routerset_contains_routerstatus(options
->ExcludeNodes
, rs
, -1)) {
392 log_warn(LD_DIR
, "Wanted to contact authority '%s' for %s, but "
393 "it's in our ExcludedNodes list and StrictNodes is set. "
396 dir_conn_purpose_to_string(dir_purpose
));
400 found
= 1; /* at least one authority of this type was listed */
401 if (dir_purpose
== DIR_PURPOSE_UPLOAD_DIR
)
402 ds
->has_accepted_serverdesc
= 0;
404 if (extrainfo_len
&& router_supports_extrainfo(ds
->digest
, 1)) {
405 upload_len
+= extrainfo_len
;
406 log_info(LD_DIR
, "Uploading an extrainfo too (length %d)",
407 (int) extrainfo_len
);
409 if (purpose_needs_anonymity(dir_purpose
, router_purpose
, NULL
)) {
410 indirection
= DIRIND_ANONYMOUS
;
411 } else if (!fascist_firewall_allows_dir_server(ds
,
412 FIREWALL_DIR_CONNECTION
,
414 if (fascist_firewall_allows_dir_server(ds
, FIREWALL_OR_CONNECTION
, 0))
415 indirection
= DIRIND_ONEHOP
;
417 indirection
= DIRIND_ANONYMOUS
;
419 indirection
= DIRIND_DIRECT_CONN
;
422 directory_request_t
*req
= directory_request_new(dir_purpose
);
423 directory_request_set_routerstatus(req
, rs
);
424 directory_request_set_router_purpose(req
, router_purpose
);
425 directory_request_set_indirection(req
, indirection
);
426 directory_request_set_payload(req
, payload
, upload_len
);
427 directory_initiate_request(req
);
428 directory_request_free(req
);
429 } SMARTLIST_FOREACH_END(ds
);
431 char *s
= authdir_type_to_string(type
);
432 log_warn(LD_DIR
, "Publishing server descriptor to directory authorities "
433 "of type '%s', but no authorities of that type listed!", s
);
438 /** Return true iff, according to the values in <b>options</b>, we should be
439 * using directory guards for direct downloads of directory information. */
441 should_use_directory_guards(const or_options_t
*options
)
443 /* Public (non-bridge) servers never use directory guards. */
444 if (public_server_mode(options
))
446 /* If guards are disabled, we can't use directory guards.
448 if (!options
->UseEntryGuards
)
450 /* If we're configured to fetch directory info aggressively or of a
451 * nonstandard type, don't use directory guards. */
452 if (options
->DownloadExtraInfo
|| options
->FetchDirInfoEarly
||
453 options
->FetchDirInfoExtraEarly
|| options
->FetchUselessDescriptors
)
458 /** Pick an unconstrained directory server from among our guards, the latest
459 * networkstatus, or the fallback dirservers, for use in downloading
460 * information of type <b>type</b>, and return its routerstatus. */
461 static const routerstatus_t
*
462 directory_pick_generic_dirserver(dirinfo_type_t type
, int pds_flags
,
464 circuit_guard_state_t
**guard_state_out
)
466 const routerstatus_t
*rs
= NULL
;
467 const or_options_t
*options
= get_options();
469 if (options
->UseBridges
)
470 log_warn(LD_BUG
, "Called when we have UseBridges set.");
472 if (should_use_directory_guards(options
)) {
473 const node_t
*node
= guards_choose_dirguard(dir_purpose
, guard_state_out
);
477 /* anybody with a non-zero dirport will do */
478 rs
= router_pick_directory_server(type
, pds_flags
);
481 log_info(LD_DIR
, "No router found for %s; falling back to "
482 "dirserver list.", dir_conn_purpose_to_string(dir_purpose
));
483 rs
= router_pick_fallback_dirserver(type
, pds_flags
);
490 * Set the extra fields in <b>req</b> that are used when requesting a
491 * consensus of type <b>resource</b>.
493 * Right now, these fields are if-modified-since and x-or-diff-from-consensus.
496 dir_consensus_request_set_additional_headers(directory_request_t
*req
,
497 const char *resource
)
499 time_t if_modified_since
= 0;
500 uint8_t or_diff_from
[DIGEST256_LEN
];
501 int or_diff_from_is_set
= 0;
503 /* DEFAULT_IF_MODIFIED_SINCE_DELAY is 1/20 of the default consensus
506 const int DEFAULT_IF_MODIFIED_SINCE_DELAY
= 180;
507 const int32_t DEFAULT_TRY_DIFF_FOR_CONSENSUS_NEWER
= 72;
508 const int32_t MIN_TRY_DIFF_FOR_CONSENSUS_NEWER
= 0;
509 const int32_t MAX_TRY_DIFF_FOR_CONSENSUS_NEWER
= 8192;
510 const char TRY_DIFF_FOR_CONSENSUS_NEWER_NAME
[] =
511 "try-diff-for-consensus-newer-than";
515 flav
= networkstatus_parse_flavor_name(resource
);
517 int32_t max_age_for_diff
= 3600 *
518 networkstatus_get_param(NULL
,
519 TRY_DIFF_FOR_CONSENSUS_NEWER_NAME
,
520 DEFAULT_TRY_DIFF_FOR_CONSENSUS_NEWER
,
521 MIN_TRY_DIFF_FOR_CONSENSUS_NEWER
,
522 MAX_TRY_DIFF_FOR_CONSENSUS_NEWER
);
525 /* IF we have a parsed consensus of this type, we can do an
526 * if-modified-time based on it. */
528 v
= networkstatus_get_latest_consensus_by_flavor(flav
);
530 /* In networks with particularly short V3AuthVotingIntervals,
531 * ask for the consensus if it's been modified since half the
532 * V3AuthVotingInterval of the most recent consensus. */
533 time_t ims_delay
= DEFAULT_IF_MODIFIED_SINCE_DELAY
;
534 if (v
->fresh_until
> v
->valid_after
535 && ims_delay
> (v
->fresh_until
- v
->valid_after
)/2) {
536 ims_delay
= (v
->fresh_until
- v
->valid_after
)/2;
538 if_modified_since
= v
->valid_after
+ ims_delay
;
539 if (v
->valid_after
>= approx_time() - max_age_for_diff
) {
540 memcpy(or_diff_from
, v
->digest_sha3_as_signed
, DIGEST256_LEN
);
541 or_diff_from_is_set
= 1;
545 /* Otherwise it might be a consensus we don't parse, but which we
546 * do cache. Look at the cached copy, perhaps. */
547 cached_dir_t
*cd
= dirserv_get_consensus(resource
);
548 /* We have no method of determining the voting interval from an
549 * unparsed consensus, so we use the default. */
551 if_modified_since
= cd
->published
+ DEFAULT_IF_MODIFIED_SINCE_DELAY
;
552 if (cd
->published
>= approx_time() - max_age_for_diff
) {
553 memcpy(or_diff_from
, cd
->digest_sha3_as_signed
, DIGEST256_LEN
);
554 or_diff_from_is_set
= 1;
559 if (if_modified_since
> 0)
560 directory_request_set_if_modified_since(req
, if_modified_since
);
561 if (or_diff_from_is_set
) {
562 char hex
[HEX_DIGEST256_LEN
+ 1];
563 base16_encode(hex
, sizeof(hex
),
564 (const char*)or_diff_from
, sizeof(or_diff_from
));
565 directory_request_add_header(req
, X_OR_DIFF_FROM_CONSENSUS_HEADER
, hex
);
569 /** Start a connection to a random running directory server, using
570 * connection purpose <b>dir_purpose</b>, intending to fetch descriptors
571 * of purpose <b>router_purpose</b>, and requesting <b>resource</b>.
572 * Use <b>pds_flags</b> as arguments to router_pick_directory_server()
573 * or router_pick_trusteddirserver().
576 directory_get_from_dirserver
,(
578 uint8_t router_purpose
,
579 const char *resource
,
581 download_want_authority_t want_authority
))
583 const routerstatus_t
*rs
= NULL
;
584 const or_options_t
*options
= get_options();
585 int prefer_authority
= (directory_fetches_from_authorities(options
)
586 || want_authority
== DL_WANT_AUTHORITY
);
587 int require_authority
= 0;
588 int get_via_tor
= purpose_needs_anonymity(dir_purpose
, router_purpose
,
590 dirinfo_type_t type
= dir_fetch_type(dir_purpose
, router_purpose
, resource
);
592 if (type
== NO_DIRINFO
)
595 if (!options
->FetchServerDescriptors
)
598 circuit_guard_state_t
*guard_state
= NULL
;
600 if (options
->UseBridges
&& !(type
& BRIDGE_DIRINFO
)) {
601 /* We want to ask a running bridge for which we have a descriptor.
603 * When we ask choose_random_entry() for a bridge, we specify what
604 * sort of dir fetch we'll be doing, so it won't return a bridge
605 * that can't answer our question.
607 const node_t
*node
= guards_choose_dirguard(dir_purpose
, &guard_state
);
608 if (node
&& node
->ri
) {
609 /* every bridge has a routerinfo. */
610 routerinfo_t
*ri
= node
->ri
;
611 /* clients always make OR connections to bridges */
612 tor_addr_port_t or_ap
;
613 directory_request_t
*req
= directory_request_new(dir_purpose
);
614 /* we are willing to use a non-preferred address if we need to */
615 fascist_firewall_choose_address_node(node
, FIREWALL_OR_CONNECTION
, 0,
617 directory_request_set_or_addr_port(req
, &or_ap
);
618 directory_request_set_directory_id_digest(req
,
619 ri
->cache_info
.identity_digest
);
620 directory_request_set_router_purpose(req
, router_purpose
);
621 directory_request_set_resource(req
, resource
);
622 if (dir_purpose
== DIR_PURPOSE_FETCH_CONSENSUS
)
623 dir_consensus_request_set_additional_headers(req
, resource
);
624 directory_request_set_guard_state(req
, guard_state
);
625 directory_initiate_request(req
);
626 directory_request_free(req
);
629 entry_guard_cancel(&guard_state
);
631 log_notice(LD_DIR
, "Ignoring directory request, since no bridge "
632 "nodes are available yet.");
637 if (prefer_authority
|| (type
& BRIDGE_DIRINFO
)) {
638 /* only ask authdirservers, and don't ask myself */
639 rs
= router_pick_trusteddirserver(type
, pds_flags
);
640 if (rs
== NULL
&& (pds_flags
& (PDS_NO_EXISTING_SERVERDESC_FETCH
|
641 PDS_NO_EXISTING_MICRODESC_FETCH
))) {
642 /* We don't want to fetch from any authorities that we're currently
643 * fetching server descriptors from, and we got no match. Did we
644 * get no match because all the authorities have connections
645 * fetching server descriptors (in which case we should just
646 * return,) or because all the authorities are down or on fire or
647 * unreachable or something (in which case we should go on with
648 * our fallback code)? */
649 pds_flags
&= ~(PDS_NO_EXISTING_SERVERDESC_FETCH
|
650 PDS_NO_EXISTING_MICRODESC_FETCH
);
651 rs
= router_pick_trusteddirserver(type
, pds_flags
);
653 log_debug(LD_DIR
, "Deferring serverdesc fetch: all authorities "
658 if (rs
== NULL
&& require_authority
) {
659 log_info(LD_DIR
, "No authorities were available for %s: will try "
660 "later.", dir_conn_purpose_to_string(dir_purpose
));
664 if (!rs
&& !(type
& BRIDGE_DIRINFO
)) {
665 rs
= directory_pick_generic_dirserver(type
, pds_flags
,
669 get_via_tor
= 1; /* last resort: try routing it via Tor */
675 /* Never use fascistfirewall; we're going via Tor. */
676 pds_flags
|= PDS_IGNORE_FASCISTFIREWALL
;
677 rs
= router_pick_directory_server(type
, pds_flags
);
680 /* If we have any hope of building an indirect conn, we know some router
681 * descriptors. If (rs==NULL), we can't build circuits anyway, so
682 * there's no point in falling back to the authorities in this case. */
684 const dir_indirection_t indirection
=
685 get_via_tor
? DIRIND_ANONYMOUS
: DIRIND_ONEHOP
;
686 directory_request_t
*req
= directory_request_new(dir_purpose
);
687 directory_request_set_routerstatus(req
, rs
);
688 directory_request_set_router_purpose(req
, router_purpose
);
689 directory_request_set_indirection(req
, indirection
);
690 directory_request_set_resource(req
, resource
);
691 if (dir_purpose
== DIR_PURPOSE_FETCH_CONSENSUS
)
692 dir_consensus_request_set_additional_headers(req
, resource
);
694 directory_request_set_guard_state(req
, guard_state
);
695 directory_initiate_request(req
);
696 directory_request_free(req
);
699 "While fetching directory info, "
700 "no running dirservers known. Will try again later. "
701 "(purpose %d)", dir_purpose
);
702 if (!purpose_needs_anonymity(dir_purpose
, router_purpose
, resource
)) {
703 /* remember we tried them all and failed. */
704 directory_all_unreachable(time(NULL
));
709 /** As directory_get_from_dirserver, but initiates a request to <i>every</i>
710 * directory authority other than ourself. Only for use by authorities when
711 * searching for missing information while voting. */
713 directory_get_from_all_authorities(uint8_t dir_purpose
,
714 uint8_t router_purpose
,
715 const char *resource
)
717 tor_assert(dir_purpose
== DIR_PURPOSE_FETCH_STATUS_VOTE
||
718 dir_purpose
== DIR_PURPOSE_FETCH_DETACHED_SIGNATURES
);
720 SMARTLIST_FOREACH_BEGIN(router_get_trusted_dir_servers(),
721 dir_server_t
*, ds
) {
722 if (router_digest_is_me(ds
->digest
))
724 if (!(ds
->type
& V3_DIRINFO
))
726 const routerstatus_t
*rs
= &ds
->fake_status
;
727 directory_request_t
*req
= directory_request_new(dir_purpose
);
728 directory_request_set_routerstatus(req
, rs
);
729 directory_request_set_router_purpose(req
, router_purpose
);
730 directory_request_set_resource(req
, resource
);
731 directory_initiate_request(req
);
732 directory_request_free(req
);
733 } SMARTLIST_FOREACH_END(ds
);
736 /** Return true iff <b>ind</b> requires a multihop circuit. */
738 dirind_is_anon(dir_indirection_t ind
)
740 return ind
== DIRIND_ANON_DIRPORT
|| ind
== DIRIND_ANONYMOUS
;
743 /* Choose reachable OR and Dir addresses and ports from status, copying them
744 * into use_or_ap and use_dir_ap. If indirection is anonymous, then we're
745 * connecting via another relay, so choose the primary IPv4 address and ports.
747 * status should have at least one reachable address, if we can't choose a
748 * reachable address, warn and return -1. Otherwise, return 0.
751 directory_choose_address_routerstatus(const routerstatus_t
*status
,
752 dir_indirection_t indirection
,
753 tor_addr_port_t
*use_or_ap
,
754 tor_addr_port_t
*use_dir_ap
)
756 tor_assert(status
!= NULL
);
757 tor_assert(use_or_ap
!= NULL
);
758 tor_assert(use_dir_ap
!= NULL
);
760 const or_options_t
*options
= get_options();
761 int have_or
= 0, have_dir
= 0;
763 /* We expect status to have at least one reachable address if we're
764 * connecting to it directly.
766 * Therefore, we can simply use the other address if the one we want isn't
767 * allowed by the firewall.
769 * (When Tor uploads and downloads a hidden service descriptor, it uses
770 * DIRIND_ANONYMOUS, except for Tor2Web, which uses DIRIND_ONEHOP.
771 * So this code will only modify the address for Tor2Web's HS descriptor
772 * fetches. Even Single Onion Servers (NYI) use DIRIND_ANONYMOUS, to avoid
773 * HSDirs denying service by rejecting descriptors.)
776 /* Initialise the OR / Dir addresses */
777 tor_addr_make_null(&use_or_ap
->addr
, AF_UNSPEC
);
779 tor_addr_make_null(&use_dir_ap
->addr
, AF_UNSPEC
);
780 use_dir_ap
->port
= 0;
782 /* ORPort connections */
783 if (indirection
== DIRIND_ANONYMOUS
) {
785 /* Since we're going to build a 3-hop circuit and ask the 2nd relay
786 * to extend to this address, always use the primary (IPv4) OR address */
787 tor_addr_from_ipv4h(&use_or_ap
->addr
, status
->addr
);
788 use_or_ap
->port
= status
->or_port
;
791 } else if (indirection
== DIRIND_ONEHOP
) {
792 /* We use an IPv6 address if we have one and we prefer it.
793 * Use the preferred address and port if they are reachable, otherwise,
794 * use the alternate address and port (if any).
796 have_or
= fascist_firewall_choose_address_rs(status
,
797 FIREWALL_OR_CONNECTION
, 0,
801 /* DirPort connections
802 * DIRIND_ONEHOP uses ORPort, but may fall back to the DirPort on relays */
803 if (indirection
== DIRIND_DIRECT_CONN
||
804 indirection
== DIRIND_ANON_DIRPORT
||
805 (indirection
== DIRIND_ONEHOP
806 && !directory_must_use_begindir(options
))) {
807 have_dir
= fascist_firewall_choose_address_rs(status
,
808 FIREWALL_DIR_CONNECTION
, 0,
812 /* We rejected all addresses in the relay's status. This means we can't
814 if (!have_or
&& !have_dir
) {
815 static int logged_backtrace
= 0;
816 log_info(LD_BUG
, "Rejected all OR and Dir addresses from %s when "
817 "launching an outgoing directory connection to: IPv4 %s OR %d "
818 "Dir %d IPv6 %s OR %d Dir %d", routerstatus_describe(status
),
819 fmt_addr32(status
->addr
), status
->or_port
,
820 status
->dir_port
, fmt_addr(&status
->ipv6_addr
),
821 status
->ipv6_orport
, status
->dir_port
);
822 if (!logged_backtrace
) {
823 log_backtrace(LOG_INFO
, LD_BUG
, "Addresses came from");
824 logged_backtrace
= 1;
832 /** Return true iff <b>conn</b> is the client side of a directory connection
833 * we launched to ourself in order to determine the reachability of our
836 directory_conn_is_self_reachability_test(dir_connection_t
*conn
)
838 if (conn
->requested_resource
&&
839 !strcmpstart(conn
->requested_resource
,"authority")) {
840 const routerinfo_t
*me
= router_get_my_routerinfo();
842 router_digest_is_me(conn
->identity_digest
) &&
843 tor_addr_eq_ipv4h(&conn
->base_
.addr
, me
->addr
) && /*XXXX prop 118*/
844 me
->dir_port
== conn
->base_
.port
)
850 /** Called when we are unable to complete the client's request to a directory
851 * server due to a network error: Mark the router as down and try again if
855 connection_dir_request_failed(dir_connection_t
*conn
)
857 if (conn
->guard_state
) {
858 /* We haven't seen a success on this guard state, so consider it to have
860 entry_guard_failed(&conn
->guard_state
);
862 if (directory_conn_is_self_reachability_test(conn
)) {
863 return; /* this was a test fetch. don't retry. */
865 if (!entry_list_is_constrained(get_options()))
866 router_set_status(conn
->identity_digest
, 0); /* don't try this one again */
867 if (conn
->base_
.purpose
== DIR_PURPOSE_FETCH_SERVERDESC
||
868 conn
->base_
.purpose
== DIR_PURPOSE_FETCH_EXTRAINFO
) {
869 log_info(LD_DIR
, "Giving up on serverdesc/extrainfo fetch from "
870 "directory server at '%s'; retrying",
871 conn
->base_
.address
);
872 if (conn
->router_purpose
== ROUTER_PURPOSE_BRIDGE
)
873 connection_dir_bridge_routerdesc_failed(conn
);
874 connection_dir_download_routerdesc_failed(conn
);
875 } else if (conn
->base_
.purpose
== DIR_PURPOSE_FETCH_CONSENSUS
) {
876 if (conn
->requested_resource
)
877 networkstatus_consensus_download_failed(0, conn
->requested_resource
);
878 } else if (conn
->base_
.purpose
== DIR_PURPOSE_FETCH_CERTIFICATE
) {
879 log_info(LD_DIR
, "Giving up on certificate fetch from directory server "
881 conn
->base_
.address
);
882 connection_dir_download_cert_failed(conn
, 0);
883 } else if (conn
->base_
.purpose
== DIR_PURPOSE_FETCH_DETACHED_SIGNATURES
) {
884 log_info(LD_DIR
, "Giving up downloading detached signatures from '%s'",
885 conn
->base_
.address
);
886 } else if (conn
->base_
.purpose
== DIR_PURPOSE_FETCH_STATUS_VOTE
) {
887 log_info(LD_DIR
, "Giving up downloading votes from '%s'",
888 conn
->base_
.address
);
889 } else if (conn
->base_
.purpose
== DIR_PURPOSE_FETCH_MICRODESC
) {
890 log_info(LD_DIR
, "Giving up on downloading microdescriptors from "
891 "directory server at '%s'; will retry", conn
->base_
.address
);
892 connection_dir_download_routerdesc_failed(conn
);
896 /** Helper: Attempt to fetch directly the descriptors of each bridge
897 * listed in <b>failed</b>.
900 connection_dir_retry_bridges(smartlist_t
*descs
)
902 char digest
[DIGEST_LEN
];
903 SMARTLIST_FOREACH(descs
, const char *, cp
,
905 if (base16_decode(digest
, DIGEST_LEN
, cp
, strlen(cp
)) != DIGEST_LEN
) {
906 log_warn(LD_BUG
, "Malformed fingerprint in list: %s",
910 retry_bridge_descriptor_fetch_directly(digest
);
914 /** Called when an attempt to download one or more router descriptors
915 * or extra-info documents on connection <b>conn</b> failed.
918 connection_dir_download_routerdesc_failed(dir_connection_t
*conn
)
920 /* No need to increment the failure count for routerdescs, since
921 * it's not their fault. */
923 /* No need to relaunch descriptor downloads here: we already do it
924 * every 10 or 60 seconds (FOO_DESCRIPTOR_RETRY_INTERVAL) in main.c. */
925 tor_assert(conn
->base_
.purpose
== DIR_PURPOSE_FETCH_SERVERDESC
||
926 conn
->base_
.purpose
== DIR_PURPOSE_FETCH_EXTRAINFO
||
927 conn
->base_
.purpose
== DIR_PURPOSE_FETCH_MICRODESC
);
932 /** Called when an attempt to download a bridge's routerdesc from
933 * one of the authorities failed due to a network error. If
934 * possible attempt to download descriptors from the bridge directly.
937 connection_dir_bridge_routerdesc_failed(dir_connection_t
*conn
)
939 smartlist_t
*which
= NULL
;
941 /* Requests for bridge descriptors are in the form 'fp/', so ignore
943 if (!conn
->requested_resource
|| strcmpstart(conn
->requested_resource
,"fp/"))
946 which
= smartlist_new();
947 dir_split_resource_into_fingerprints(conn
->requested_resource
951 tor_assert(conn
->base_
.purpose
!= DIR_PURPOSE_FETCH_EXTRAINFO
);
952 if (smartlist_len(which
)) {
953 connection_dir_retry_bridges(which
);
954 SMARTLIST_FOREACH(which
, char *, cp
, tor_free(cp
));
956 smartlist_free(which
);
959 /** Called when an attempt to fetch a certificate fails. */
961 connection_dir_download_cert_failed(dir_connection_t
*conn
, int status
)
963 const char *fp_pfx
= "fp/";
964 const char *fpsk_pfx
= "fp-sk/";
966 tor_assert(conn
->base_
.purpose
== DIR_PURPOSE_FETCH_CERTIFICATE
);
968 if (!conn
->requested_resource
)
970 failed
= smartlist_new();
972 * We have two cases download by fingerprint (resource starts
973 * with "fp/") or download by fingerprint/signing key pair
974 * (resource starts with "fp-sk/").
976 if (!strcmpstart(conn
->requested_resource
, fp_pfx
)) {
977 /* Download by fingerprint case */
978 dir_split_resource_into_fingerprints(conn
->requested_resource
+
980 failed
, NULL
, DSR_HEX
);
981 SMARTLIST_FOREACH_BEGIN(failed
, char *, cp
) {
982 /* Null signing key digest indicates download by fp only */
983 authority_cert_dl_failed(cp
, NULL
, status
);
985 } SMARTLIST_FOREACH_END(cp
);
986 } else if (!strcmpstart(conn
->requested_resource
, fpsk_pfx
)) {
987 /* Download by (fp,sk) pairs */
988 dir_split_resource_into_fingerprint_pairs(conn
->requested_resource
+
989 strlen(fpsk_pfx
), failed
);
990 SMARTLIST_FOREACH_BEGIN(failed
, fp_pair_t
*, cp
) {
991 authority_cert_dl_failed(cp
->first
, cp
->second
, status
);
993 } SMARTLIST_FOREACH_END(cp
);
996 "Don't know what to do with failure for cert fetch %s",
997 conn
->requested_resource
);
1000 smartlist_free(failed
);
1002 update_certificate_downloads(time(NULL
));
1005 /* Should this tor instance only use begindir for all its directory requests?
1008 directory_must_use_begindir(const or_options_t
*options
)
1010 /* Clients, onion services, and bridges must use begindir,
1011 * relays and authorities do not have to */
1012 return !public_server_mode(options
);
1015 /** Evaluate the situation and decide if we should use an encrypted
1016 * "begindir-style" connection for this directory request.
1017 * 0) If there is no DirPort, yes.
1018 * 1) If or_port is 0, or it's a direct conn and or_port is firewalled
1019 * or we're a dir mirror, no.
1020 * 2) If we prefer to avoid begindir conns, and we're not fetching or
1021 * publishing a bridge relay descriptor, no.
1023 * If returning 0, return in *reason why we can't use begindir.
1024 * reason must not be NULL.
1027 directory_command_should_use_begindir(const or_options_t
*options
,
1028 const directory_request_t
*req
,
1029 const char **reason
)
1031 const tor_addr_t
*or_addr
= &req
->or_addr_port
.addr
;
1032 //const tor_addr_t *dir_addr = &req->dir_addr_port.addr;
1033 const int or_port
= req
->or_addr_port
.port
;
1034 const int dir_port
= req
->dir_addr_port
.port
;
1036 const dir_indirection_t indirection
= req
->indirection
;
1041 /* Reasons why we must use begindir */
1043 *reason
= "(using begindir - directory with no DirPort)";
1044 return 1; /* We don't know a DirPort -- must begindir. */
1046 /* Reasons why we can't possibly use begindir */
1048 *reason
= "directory with unknown ORPort";
1049 return 0; /* We don't know an ORPort -- no chance. */
1051 if (indirection
== DIRIND_DIRECT_CONN
||
1052 indirection
== DIRIND_ANON_DIRPORT
) {
1053 *reason
= "DirPort connection";
1056 if (indirection
== DIRIND_ONEHOP
) {
1057 /* We're firewalled and want a direct OR connection */
1058 if (!fascist_firewall_allows_address_addr(or_addr
, or_port
,
1059 FIREWALL_OR_CONNECTION
, 0, 0)) {
1060 *reason
= "ORPort not reachable";
1064 /* Reasons why we want to avoid using begindir */
1065 if (indirection
== DIRIND_ONEHOP
) {
1066 if (!directory_must_use_begindir(options
)) {
1067 *reason
= "in relay mode";
1071 /* DIRIND_ONEHOP on a client, or DIRIND_ANONYMOUS
1073 *reason
= "(using begindir)";
1078 * Create and return a new directory_request_t with purpose
1079 * <b>dir_purpose</b>.
1081 directory_request_t
*
1082 directory_request_new(uint8_t dir_purpose
)
1084 tor_assert(dir_purpose
>= DIR_PURPOSE_MIN_
);
1085 tor_assert(dir_purpose
<= DIR_PURPOSE_MAX_
);
1086 tor_assert(dir_purpose
!= DIR_PURPOSE_SERVER
);
1087 tor_assert(dir_purpose
!= DIR_PURPOSE_HAS_FETCHED_RENDDESC_V2
);
1088 tor_assert(dir_purpose
!= DIR_PURPOSE_HAS_FETCHED_HSDESC
);
1090 directory_request_t
*result
= tor_malloc_zero(sizeof(*result
));
1091 tor_addr_make_null(&result
->or_addr_port
.addr
, AF_INET
);
1092 result
->or_addr_port
.port
= 0;
1093 tor_addr_make_null(&result
->dir_addr_port
.addr
, AF_INET
);
1094 result
->dir_addr_port
.port
= 0;
1095 result
->dir_purpose
= dir_purpose
;
1096 result
->router_purpose
= ROUTER_PURPOSE_GENERAL
;
1097 result
->indirection
= DIRIND_ONEHOP
;
1101 * Release all resources held by <b>req</b>.
1104 directory_request_free(directory_request_t
*req
)
1108 config_free_lines(req
->additional_headers
);
1112 * Set the address and OR port to use for this directory request. If there is
1113 * no OR port, we'll have to connect over the dirport. (If there are both,
1114 * the indirection setting determins which to use.)
1117 directory_request_set_or_addr_port(directory_request_t
*req
,
1118 const tor_addr_port_t
*p
)
1120 memcpy(&req
->or_addr_port
, p
, sizeof(*p
));
1123 * Set the address and dirport to use for this directory request. If there
1124 * is no dirport, we'll have to connect over the OR port. (If there are both,
1125 * the indirection setting determins which to use.)
1128 directory_request_set_dir_addr_port(directory_request_t
*req
,
1129 const tor_addr_port_t
*p
)
1131 memcpy(&req
->dir_addr_port
, p
, sizeof(*p
));
1134 * Set the RSA identity digest of the directory to use for this directory
1138 directory_request_set_directory_id_digest(directory_request_t
*req
,
1141 memcpy(req
->digest
, digest
, DIGEST_LEN
);
1144 * Set the router purpose associated with uploaded and downloaded router
1145 * descriptors and extrainfo documents in this directory request. The purpose
1146 * must be one of ROUTER_PURPOSE_GENERAL (the default) or
1147 * ROUTER_PURPOSE_BRIDGE.
1150 directory_request_set_router_purpose(directory_request_t
*req
,
1151 uint8_t router_purpose
)
1153 tor_assert(router_purpose
== ROUTER_PURPOSE_GENERAL
||
1154 router_purpose
== ROUTER_PURPOSE_BRIDGE
);
1155 // assert that it actually makes sense to set this purpose, given
1157 req
->router_purpose
= router_purpose
;
1160 * Set the indirection to be used for the directory request. The indirection
1161 * parameter configures whether to connect to a DirPort or ORPort, and whether
1162 * to anonymize the connection. DIRIND_ONEHOP (use ORPort, don't anonymize)
1163 * is the default. See dir_indirection_t for more information.
1166 directory_request_set_indirection(directory_request_t
*req
,
1167 dir_indirection_t indirection
)
1169 req
->indirection
= indirection
;
1173 * Set a pointer to the resource to request from a directory. Different
1174 * request types use resources to indicate different components of their URL.
1175 * Note that only an alias to <b>resource</b> is stored, so the
1176 * <b>resource</b> must outlive the request.
1179 directory_request_set_resource(directory_request_t
*req
,
1180 const char *resource
)
1182 req
->resource
= resource
;
1185 * Set a pointer to the payload to include with this directory request, along
1186 * with its length. Note that only an alias to <b>payload</b> is stored, so
1187 * the <b>payload</b> must outlive the request.
1190 directory_request_set_payload(directory_request_t
*req
,
1191 const char *payload
,
1194 tor_assert(DIR_PURPOSE_IS_UPLOAD(req
->dir_purpose
));
1196 req
->payload
= payload
;
1197 req
->payload_len
= payload_len
;
1200 * Set an if-modified-since date to send along with the request. The
1201 * default is 0 (meaning, send no if-modified-since header).
1204 directory_request_set_if_modified_since(directory_request_t
*req
,
1205 time_t if_modified_since
)
1207 req
->if_modified_since
= if_modified_since
;
1210 /** Include a header of name <b>key</b> with content <b>val</b> in the
1211 * request. Neither may include newlines or other odd characters. Their
1212 * ordering is not currently guaranteed.
1214 * Note that, as elsewhere in this module, header keys include a trailing
1218 directory_request_add_header(directory_request_t
*req
,
1222 config_line_prepend(&req
->additional_headers
, key
, val
);
1225 * Set an object containing HS data to be associated with this request. Note
1226 * that only an alias to <b>query</b> is stored, so the <b>query</b> object
1227 * must outlive the request.
1230 directory_request_set_rend_query(directory_request_t
*req
,
1231 const rend_data_t
*query
)
1234 tor_assert(req
->dir_purpose
== DIR_PURPOSE_FETCH_RENDDESC_V2
||
1235 req
->dir_purpose
== DIR_PURPOSE_UPLOAD_RENDDESC_V2
);
1237 req
->rend_query
= query
;
1240 * Set an object containing HS connection identifier to be associated with
1241 * this request. Note that only an alias to <b>ident</b> is stored, so the
1242 * <b>ident</b> object must outlive the request.
1245 directory_request_upload_set_hs_ident(directory_request_t
*req
,
1246 const hs_ident_dir_conn_t
*ident
)
1249 tor_assert(req
->dir_purpose
== DIR_PURPOSE_UPLOAD_HSDESC
);
1251 req
->hs_ident
= ident
;
1254 * Set an object containing HS connection identifier to be associated with
1255 * this fetch request. Note that only an alias to <b>ident</b> is stored, so
1256 * the <b>ident</b> object must outlive the request.
1259 directory_request_fetch_set_hs_ident(directory_request_t
*req
,
1260 const hs_ident_dir_conn_t
*ident
)
1263 tor_assert(req
->dir_purpose
== DIR_PURPOSE_FETCH_HSDESC
);
1265 req
->hs_ident
= ident
;
1267 /** Set a static circuit_guard_state_t object to affliate with the request in
1268 * <b>req</b>. This object will receive notification when the attempt to
1269 * connect to the guard either succeeds or fails. */
1271 directory_request_set_guard_state(directory_request_t
*req
,
1272 circuit_guard_state_t
*state
)
1274 req
->guard_state
= state
;
1278 * Internal: Return true if any information for contacting the directory in
1279 * <b>req</b> has been set, other than by the routerstatus. */
1281 directory_request_dir_contact_info_specified(const directory_request_t
*req
)
1283 /* We only check for ports here, since we don't use an addr unless the port
1285 return (req
->or_addr_port
.port
||
1286 req
->dir_addr_port
.port
||
1287 ! tor_digest_is_zero(req
->digest
));
1291 * Set the routerstatus to use for the directory associated with this
1292 * request. If this option is set, then no other function to set the
1293 * directory's address or identity should be called.
1296 directory_request_set_routerstatus(directory_request_t
*req
,
1297 const routerstatus_t
*status
)
1299 req
->routerstatus
= status
;
1302 * Helper: update the addresses, ports, and identities in <b>req</b>
1303 * from the routerstatus object in <b>req</b>. Return 0 on success.
1304 * On failure, warn and return -1.
1307 directory_request_set_dir_from_routerstatus(directory_request_t
*req
)
1310 const routerstatus_t
*status
= req
->routerstatus
;
1311 if (BUG(status
== NULL
))
1313 const or_options_t
*options
= get_options();
1315 tor_addr_port_t use_or_ap
, use_dir_ap
;
1316 const int anonymized_connection
= dirind_is_anon(req
->indirection
);
1318 tor_assert(status
!= NULL
);
1320 node
= node_get_by_id(status
->identity_digest
);
1322 /* XXX The below check is wrong: !node means it's not in the consensus,
1323 * but we haven't checked if we have a descriptor for it -- and also,
1324 * we only care about the descriptor if it's a begindir-style anonymized
1326 if (!node
&& anonymized_connection
) {
1327 log_info(LD_DIR
, "Not sending anonymized request to directory '%s'; we "
1328 "don't have its router descriptor.",
1329 routerstatus_describe(status
));
1333 if (options
->ExcludeNodes
&& options
->StrictNodes
&&
1334 routerset_contains_routerstatus(options
->ExcludeNodes
, status
, -1)) {
1335 log_warn(LD_DIR
, "Wanted to contact directory mirror %s for %s, but "
1336 "it's in our ExcludedNodes list and StrictNodes is set. "
1337 "Skipping. This choice might make your Tor not work.",
1338 routerstatus_describe(status
),
1339 dir_conn_purpose_to_string(req
->dir_purpose
));
1343 /* At this point, if we are a client making a direct connection to a
1344 * directory server, we have selected a server that has at least one address
1345 * allowed by ClientUseIPv4/6 and Reachable{"",OR,Dir}Addresses. This
1346 * selection uses the preference in ClientPreferIPv6{OR,Dir}Port, if
1347 * possible. (If UseBridges is set, clients always use IPv6, and prefer it
1350 * Now choose an address that we can use to connect to the directory server.
1352 if (directory_choose_address_routerstatus(status
,
1353 req
->indirection
, &use_or_ap
,
1358 directory_request_set_or_addr_port(req
, &use_or_ap
);
1359 directory_request_set_dir_addr_port(req
, &use_dir_ap
);
1360 directory_request_set_directory_id_digest(req
, status
->identity_digest
);
1365 * Launch the provided directory request, configured in <b>request</b>.
1366 * After this function is called, you can free <b>request</b>.
1369 directory_initiate_request
,(directory_request_t
*request
))
1371 tor_assert(request
);
1372 if (request
->routerstatus
) {
1373 tor_assert_nonfatal(
1374 ! directory_request_dir_contact_info_specified(request
));
1375 if (directory_request_set_dir_from_routerstatus(request
) < 0) {
1380 const tor_addr_port_t
*or_addr_port
= &request
->or_addr_port
;
1381 const tor_addr_port_t
*dir_addr_port
= &request
->dir_addr_port
;
1382 const char *digest
= request
->digest
;
1383 const uint8_t dir_purpose
= request
->dir_purpose
;
1384 const uint8_t router_purpose
= request
->router_purpose
;
1385 const dir_indirection_t indirection
= request
->indirection
;
1386 const char *resource
= request
->resource
;
1387 const rend_data_t
*rend_query
= request
->rend_query
;
1388 const hs_ident_dir_conn_t
*hs_ident
= request
->hs_ident
;
1389 circuit_guard_state_t
*guard_state
= request
->guard_state
;
1391 tor_assert(or_addr_port
->port
|| dir_addr_port
->port
);
1394 dir_connection_t
*conn
;
1395 const or_options_t
*options
= get_options();
1396 int socket_error
= 0;
1397 const char *begindir_reason
= NULL
;
1398 /* Should the connection be to a relay's OR port (and inside that we will
1399 * send our directory request)? */
1400 const int use_begindir
=
1401 directory_command_should_use_begindir(options
, request
, &begindir_reason
);
1403 /* Will the connection go via a three-hop Tor circuit? Note that this
1404 * is separate from whether it will use_begindir. */
1405 const int anonymized_connection
= dirind_is_anon(indirection
);
1407 /* What is the address we want to make the directory request to? If
1408 * we're making a begindir request this is the ORPort of the relay
1409 * we're contacting; if not a begindir request, this is its DirPort.
1410 * Note that if anonymized_connection is true, we won't be initiating
1411 * a connection directly to this address. */
1413 tor_addr_copy(&addr
, &(use_begindir
? or_addr_port
: dir_addr_port
)->addr
);
1414 uint16_t port
= (use_begindir
? or_addr_port
: dir_addr_port
)->port
;
1416 log_debug(LD_DIR
, "anonymized %d, use_begindir %d.",
1417 anonymized_connection
, use_begindir
);
1419 log_debug(LD_DIR
, "Initiating %s", dir_conn_purpose_to_string(dir_purpose
));
1421 if (purpose_needs_anonymity(dir_purpose
, router_purpose
, resource
)) {
1422 tor_assert(anonymized_connection
||
1423 rend_non_anonymous_mode_enabled(options
));
1426 /* use encrypted begindir connections for everything except relays
1427 * this provides better protection for directory fetches */
1428 if (!use_begindir
&& directory_must_use_begindir(options
)) {
1429 log_warn(LD_BUG
, "Client could not use begindir connection: %s",
1430 begindir_reason
? begindir_reason
: "(NULL)");
1434 /* ensure that we don't make direct connections when a SOCKS server is
1436 if (!anonymized_connection
&& !use_begindir
&& !options
->HTTPProxy
&&
1437 (options
->Socks4Proxy
|| options
->Socks5Proxy
)) {
1438 log_warn(LD_DIR
, "Cannot connect to a directory server through a "
1443 /* Make sure that the destination addr and port we picked is viable. */
1444 if (!port
|| tor_addr_is_null(&addr
)) {
1445 static int logged_backtrace
= 0;
1447 "Cannot make an outgoing %sconnection without a remote %sPort.",
1448 use_begindir
? "begindir " : "",
1449 use_begindir
? "OR" : "Dir");
1450 if (!logged_backtrace
) {
1451 log_backtrace(LOG_INFO
, LD_BUG
, "Address came from");
1452 logged_backtrace
= 1;
1457 conn
= dir_connection_new(tor_addr_family(&addr
));
1459 /* set up conn so it's got all the data we need to remember */
1460 tor_addr_copy(&conn
->base_
.addr
, &addr
);
1461 conn
->base_
.port
= port
;
1462 conn
->base_
.address
= tor_addr_to_str_dup(&addr
);
1463 memcpy(conn
->identity_digest
, digest
, DIGEST_LEN
);
1465 conn
->base_
.purpose
= dir_purpose
;
1466 conn
->router_purpose
= router_purpose
;
1468 /* give it an initial state */
1469 conn
->base_
.state
= DIR_CONN_STATE_CONNECTING
;
1471 /* decide whether we can learn our IP address from this conn */
1472 /* XXXX This is a bad name for this field now. */
1473 conn
->dirconn_direct
= !anonymized_connection
;
1475 /* copy rendezvous data, if any */
1477 /* We can't have both v2 and v3+ identifier. */
1478 tor_assert_nonfatal(!hs_ident
);
1479 conn
->rend_data
= rend_data_dup(rend_query
);
1482 /* We can't have both v2 and v3+ identifier. */
1483 tor_assert_nonfatal(!rend_query
);
1484 conn
->hs_ident
= hs_ident_dir_conn_dup(hs_ident
);
1487 if (!anonymized_connection
&& !use_begindir
) {
1488 /* then we want to connect to dirport directly */
1490 if (options
->HTTPProxy
) {
1491 tor_addr_copy(&addr
, &options
->HTTPProxyAddr
);
1492 port
= options
->HTTPProxyPort
;
1495 // In this case we should not have picked a directory guard.
1496 if (BUG(guard_state
)) {
1497 entry_guard_cancel(&guard_state
);
1500 switch (connection_connect(TO_CONN(conn
), conn
->base_
.address
, &addr
,
1501 port
, &socket_error
)) {
1503 connection_mark_for_close(TO_CONN(conn
));
1506 /* start flushing conn */
1507 conn
->base_
.state
= DIR_CONN_STATE_CLIENT_SENDING
;
1510 /* queue the command on the outbuf */
1511 directory_send_command(conn
, 1, request
);
1512 connection_watch_events(TO_CONN(conn
), READ_EVENT
| WRITE_EVENT
);
1513 /* writable indicates finish, readable indicates broken link,
1514 error indicates broken link in windowsland. */
1517 /* We will use a Tor circuit (maybe 1-hop, maybe 3-hop, maybe with
1518 * begindir, maybe not with begindir) */
1520 entry_connection_t
*linked_conn
;
1522 /* Anonymized tunneled connections can never share a circuit.
1523 * One-hop directory connections can share circuits with each other
1524 * but nothing else. */
1525 int iso_flags
= anonymized_connection
? ISO_STREAM
: ISO_SESSIONGRP
;
1527 /* If it's an anonymized connection, remember the fact that we
1528 * wanted it for later: maybe we'll want it again soon. */
1529 if (anonymized_connection
&& use_begindir
)
1530 rep_hist_note_used_internal(time(NULL
), 0, 1);
1531 else if (anonymized_connection
&& !use_begindir
)
1532 rep_hist_note_used_port(time(NULL
), conn
->base_
.port
);
1534 // In this case we should not have a directory guard; we'll
1535 // get a regular guard later when we build the circuit.
1536 if (BUG(anonymized_connection
&& guard_state
)) {
1537 entry_guard_cancel(&guard_state
);
1540 conn
->guard_state
= guard_state
;
1542 /* make an AP connection
1543 * populate it and add it at the right state
1544 * hook up both sides
1547 connection_ap_make_link(TO_CONN(conn
),
1548 conn
->base_
.address
, conn
->base_
.port
,
1550 SESSION_GROUP_DIRCONN
, iso_flags
,
1551 use_begindir
, !anonymized_connection
);
1553 log_warn(LD_NET
,"Making tunnel to dirserver failed.");
1554 connection_mark_for_close(TO_CONN(conn
));
1558 if (connection_add(TO_CONN(conn
)) < 0) {
1559 log_warn(LD_NET
,"Unable to add connection for link to dirserver.");
1560 connection_mark_for_close(TO_CONN(conn
));
1563 conn
->base_
.state
= DIR_CONN_STATE_CLIENT_SENDING
;
1564 /* queue the command on the outbuf */
1565 directory_send_command(conn
, 0, request
);
1567 connection_watch_events(TO_CONN(conn
), READ_EVENT
|WRITE_EVENT
);
1568 connection_start_reading(ENTRY_TO_CONN(linked_conn
));
1572 /** Return true iff anything we say on <b>conn</b> is being encrypted before
1573 * we send it to the client/server. */
1575 connection_dir_is_encrypted(const dir_connection_t
*conn
)
1577 /* Right now it's sufficient to see if conn is or has been linked, since
1578 * the only thing it could be linked to is an edge connection on a
1579 * circuit, and the only way it could have been unlinked is at the edge
1580 * connection getting closed.
1582 return TO_CONN(conn
)->linked
;
1585 /** Helper for sorting
1587 * sort strings alphabetically
1590 compare_strs_(const void **a
, const void **b
)
1592 const char *s1
= *a
, *s2
= *b
;
1593 return strcmp(s1
, s2
);
1596 #define CONDITIONAL_CONSENSUS_FPR_LEN 3
1597 #if (CONDITIONAL_CONSENSUS_FPR_LEN > DIGEST_LEN)
1598 #error "conditional consensus fingerprint length is larger than digest length"
1601 /** Return the URL we should use for a consensus download.
1603 * Use the "conditional consensus downloading" feature described in
1604 * dir-spec.txt, i.e.
1605 * GET .../consensus/<b>fpr</b>+<b>fpr</b>+<b>fpr</b>
1607 * If 'resource' is provided, it is the name of a consensus flavor to request.
1610 directory_get_consensus_url(const char *resource
)
1613 const char *hyphen
, *flavor
;
1614 if (resource
==NULL
|| strcmp(resource
, "ns")==0) {
1615 flavor
= ""; /* Request ns consensuses as "", so older servers will work*/
1623 char *authority_id_list
;
1624 smartlist_t
*authority_digests
= smartlist_new();
1626 SMARTLIST_FOREACH_BEGIN(router_get_trusted_dir_servers(),
1627 dir_server_t
*, ds
) {
1629 if (!(ds
->type
& V3_DIRINFO
))
1632 hex
= tor_malloc(2*CONDITIONAL_CONSENSUS_FPR_LEN
+1);
1633 base16_encode(hex
, 2*CONDITIONAL_CONSENSUS_FPR_LEN
+1,
1634 ds
->v3_identity_digest
, CONDITIONAL_CONSENSUS_FPR_LEN
);
1635 smartlist_add(authority_digests
, hex
);
1636 } SMARTLIST_FOREACH_END(ds
);
1637 smartlist_sort(authority_digests
, compare_strs_
);
1638 authority_id_list
= smartlist_join_strings(authority_digests
,
1641 tor_asprintf(&url
, "/tor/status-vote/current/consensus%s%s/%s.z",
1642 hyphen
, flavor
, authority_id_list
);
1644 SMARTLIST_FOREACH(authority_digests
, char *, cp
, tor_free(cp
));
1645 smartlist_free(authority_digests
);
1646 tor_free(authority_id_list
);
1652 * Copies the ipv6 from source to destination, subject to buffer size limit
1653 * size. If decorate is true, makes sure the copied address is decorated.
1656 copy_ipv6_address(char* destination
, const char* source
, size_t len
,
1658 tor_assert(destination
);
1661 if (decorate
&& source
[0] != '[') {
1662 tor_snprintf(destination
, len
, "[%s]", source
);
1664 strlcpy(destination
, source
, len
);
1668 /** Queue an appropriate HTTP command for <b>request</b> on
1669 * <b>conn</b>-\>outbuf. If <b>direct</b> is true, we're making a
1670 * non-anonymized connection to the dirport.
1673 directory_send_command(dir_connection_t
*conn
,
1675 const directory_request_t
*req
)
1678 const int purpose
= req
->dir_purpose
;
1679 const char *resource
= req
->resource
;
1680 const char *payload
= req
->payload
;
1681 const size_t payload_len
= req
->payload_len
;
1682 const time_t if_modified_since
= req
->if_modified_since
;
1683 const int anonymized_connection
= dirind_is_anon(req
->indirection
);
1685 char proxystring
[256];
1686 char hoststring
[128];
1687 /* NEEDS to be the same size hoststring.
1688 Will be decorated with brackets around it if it is ipv6. */
1689 char decorated_address
[128];
1690 smartlist_t
*headers
= smartlist_new();
1692 char *accept_encoding
;
1695 size_t request_len
, total_request_len
= 0;
1696 const char *httpcommand
= NULL
;
1699 tor_assert(conn
->base_
.type
== CONN_TYPE_DIR
);
1701 tor_free(conn
->requested_resource
);
1703 conn
->requested_resource
= tor_strdup(resource
);
1705 /* decorate the ip address if it is ipv6 */
1706 if (strchr(conn
->base_
.address
, ':')) {
1707 copy_ipv6_address(decorated_address
, conn
->base_
.address
,
1708 sizeof(decorated_address
), 1);
1710 strlcpy(decorated_address
, conn
->base_
.address
, sizeof(decorated_address
));
1713 /* come up with a string for which Host: we want */
1714 if (conn
->base_
.port
== 80) {
1715 strlcpy(hoststring
, decorated_address
, sizeof(hoststring
));
1717 tor_snprintf(hoststring
, sizeof(hoststring
), "%s:%d",
1718 decorated_address
, conn
->base_
.port
);
1721 /* Format if-modified-since */
1722 if (if_modified_since
) {
1723 char b
[RFC1123_TIME_LEN
+1];
1724 format_rfc1123_time(b
, if_modified_since
);
1725 smartlist_add_asprintf(headers
, "If-Modified-Since: %s\r\n", b
);
1728 /* come up with some proxy lines, if we're using one. */
1729 if (direct
&& get_options()->HTTPProxy
) {
1730 char *base64_authenticator
=NULL
;
1731 const char *authenticator
= get_options()->HTTPProxyAuthenticator
;
1733 tor_snprintf(proxystring
, sizeof(proxystring
),"http://%s", hoststring
);
1734 if (authenticator
) {
1735 base64_authenticator
= alloc_http_authenticator(authenticator
);
1736 if (!base64_authenticator
)
1737 log_warn(LD_BUG
, "Encoding http authenticator failed");
1739 if (base64_authenticator
) {
1740 smartlist_add_asprintf(headers
,
1741 "Proxy-Authorization: Basic %s\r\n",
1742 base64_authenticator
);
1743 tor_free(base64_authenticator
);
1749 if (! anonymized_connection
) {
1750 /* Add Accept-Encoding. */
1751 accept_encoding
= accept_encoding_header();
1752 smartlist_add_asprintf(headers
, "Accept-Encoding: %s\r\n",
1754 tor_free(accept_encoding
);
1757 /* Add additional headers, if any */
1760 for (h
= req
->additional_headers
; h
; h
= h
->next
) {
1761 smartlist_add_asprintf(headers
, "%s%s\r\n", h
->key
, h
->value
);
1766 case DIR_PURPOSE_FETCH_CONSENSUS
:
1767 /* resource is optional. If present, it's a flavor name */
1768 tor_assert(!payload
);
1769 httpcommand
= "GET";
1770 url
= directory_get_consensus_url(resource
);
1771 log_info(LD_DIR
, "Downloading consensus from %s using %s",
1774 case DIR_PURPOSE_FETCH_CERTIFICATE
:
1775 tor_assert(resource
);
1776 tor_assert(!payload
);
1777 httpcommand
= "GET";
1778 tor_asprintf(&url
, "/tor/keys/%s", resource
);
1780 case DIR_PURPOSE_FETCH_STATUS_VOTE
:
1781 tor_assert(resource
);
1782 tor_assert(!payload
);
1783 httpcommand
= "GET";
1784 tor_asprintf(&url
, "/tor/status-vote/next/%s.z", resource
);
1786 case DIR_PURPOSE_FETCH_DETACHED_SIGNATURES
:
1787 tor_assert(!resource
);
1788 tor_assert(!payload
);
1789 httpcommand
= "GET";
1790 url
= tor_strdup("/tor/status-vote/next/consensus-signatures.z");
1792 case DIR_PURPOSE_FETCH_SERVERDESC
:
1793 tor_assert(resource
);
1794 httpcommand
= "GET";
1795 tor_asprintf(&url
, "/tor/server/%s", resource
);
1797 case DIR_PURPOSE_FETCH_EXTRAINFO
:
1798 tor_assert(resource
);
1799 httpcommand
= "GET";
1800 tor_asprintf(&url
, "/tor/extra/%s", resource
);
1802 case DIR_PURPOSE_FETCH_MICRODESC
:
1803 tor_assert(resource
);
1804 httpcommand
= "GET";
1805 tor_asprintf(&url
, "/tor/micro/%s", resource
);
1807 case DIR_PURPOSE_UPLOAD_DIR
: {
1808 const char *why
= router_get_descriptor_gen_reason();
1809 tor_assert(!resource
);
1810 tor_assert(payload
);
1811 httpcommand
= "POST";
1812 url
= tor_strdup("/tor/");
1814 smartlist_add_asprintf(headers
, "X-Desc-Gen-Reason: %s\r\n", why
);
1818 case DIR_PURPOSE_UPLOAD_VOTE
:
1819 tor_assert(!resource
);
1820 tor_assert(payload
);
1821 httpcommand
= "POST";
1822 url
= tor_strdup("/tor/post/vote");
1824 case DIR_PURPOSE_UPLOAD_SIGNATURES
:
1825 tor_assert(!resource
);
1826 tor_assert(payload
);
1827 httpcommand
= "POST";
1828 url
= tor_strdup("/tor/post/consensus-signature");
1830 case DIR_PURPOSE_FETCH_RENDDESC_V2
:
1831 tor_assert(resource
);
1832 tor_assert(strlen(resource
) <= REND_DESC_ID_V2_LEN_BASE32
);
1833 tor_assert(!payload
);
1834 httpcommand
= "GET";
1835 tor_asprintf(&url
, "/tor/rendezvous2/%s", resource
);
1837 case DIR_PURPOSE_FETCH_HSDESC
:
1838 tor_assert(resource
);
1839 tor_assert(strlen(resource
) <= ED25519_BASE64_LEN
);
1840 tor_assert(!payload
);
1841 httpcommand
= "GET";
1842 tor_asprintf(&url
, "/tor/hs/3/%s", resource
);
1844 case DIR_PURPOSE_UPLOAD_RENDDESC_V2
:
1845 tor_assert(!resource
);
1846 tor_assert(payload
);
1847 httpcommand
= "POST";
1848 url
= tor_strdup("/tor/rendezvous2/publish");
1850 case DIR_PURPOSE_UPLOAD_HSDESC
:
1851 tor_assert(resource
);
1852 tor_assert(payload
);
1853 httpcommand
= "POST";
1854 tor_asprintf(&url
, "/tor/hs/%s/publish", resource
);
1861 /* warn in the non-tunneled case */
1862 if (direct
&& (strlen(proxystring
) + strlen(url
) >= 4096)) {
1864 "Squid does not like URLs longer than 4095 bytes, and this "
1865 "one is %d bytes long: %s%s",
1866 (int)(strlen(proxystring
) + strlen(url
)), proxystring
, url
);
1869 tor_snprintf(request
, sizeof(request
), "%s %s", httpcommand
, proxystring
);
1871 request_len
= strlen(request
);
1872 total_request_len
+= request_len
;
1873 connection_buf_add(request
, request_len
, TO_CONN(conn
));
1875 url_len
= strlen(url
);
1876 total_request_len
+= url_len
;
1877 connection_buf_add(url
, url_len
, TO_CONN(conn
));
1880 if (!strcmp(httpcommand
, "POST") || payload
) {
1881 smartlist_add_asprintf(headers
, "Content-Length: %lu\r\n",
1882 payload
? (unsigned long)payload_len
: 0);
1886 char *header
= smartlist_join_strings(headers
, "", 0, NULL
);
1887 tor_snprintf(request
, sizeof(request
), " HTTP/1.0\r\nHost: %s\r\n%s\r\n",
1888 hoststring
, header
);
1892 request_len
= strlen(request
);
1893 total_request_len
+= request_len
;
1894 connection_buf_add(request
, request_len
, TO_CONN(conn
));
1897 /* then send the payload afterwards too */
1898 connection_buf_add(payload
, payload_len
, TO_CONN(conn
));
1899 total_request_len
+= payload_len
;
1902 SMARTLIST_FOREACH(headers
, char *, h
, tor_free(h
));
1903 smartlist_free(headers
);
1906 "Sent request to directory server '%s:%d': "
1907 "(purpose: %d, request size: " U64_FORMAT
", "
1908 "payload size: " U64_FORMAT
")",
1909 conn
->base_
.address
, conn
->base_
.port
,
1910 conn
->base_
.purpose
,
1911 U64_PRINTF_ARG(total_request_len
),
1912 U64_PRINTF_ARG(payload
? payload_len
: 0));
1915 /** Parse an HTTP request string <b>headers</b> of the form
1917 * "\%s [http[s]://]\%s HTTP/1..."
1919 * If it's well-formed, strdup the second \%s into *<b>url</b>, and
1920 * nul-terminate it. If the url doesn't start with "/tor/", rewrite it
1921 * so it does. Return 0.
1922 * Otherwise, return -1.
1925 parse_http_url(const char *headers
, char **url
)
1927 char *command
= NULL
;
1928 if (parse_http_command(headers
, &command
, url
) < 0) {
1931 if (strcmpstart(*url
, "/tor/")) {
1932 char *new_url
= NULL
;
1933 tor_asprintf(&new_url
, "/tor%s%s",
1934 *url
[0] == '/' ? "" : "/",
1943 /** Parse an HTTP request line at the start of a headers string. On failure,
1944 * return -1. On success, set *<b>command_out</b> to a copy of the HTTP
1945 * command ("get", "post", etc), set *<b>url_out</b> to a copy of the URL, and
1948 parse_http_command(const char *headers
, char **command_out
, char **url_out
)
1950 const char *command
, *end_of_command
;
1951 char *s
, *start
, *tmp
;
1953 s
= (char *)eat_whitespace_no_nl(headers
);
1956 s
= (char *)find_whitespace(s
); /* get past GET/POST */
1959 s
= (char *)eat_whitespace_no_nl(s
);
1961 start
= s
; /* this is the URL, assuming it's valid */
1962 s
= (char *)find_whitespace(start
);
1965 /* tolerate the http[s] proxy style of putting the hostname in the url */
1966 if (s
-start
>= 4 && !strcmpstart(start
,"http")) {
1970 if (s
-tmp
>= 3 && !strcmpstart(tmp
,"://")) {
1971 tmp
= strchr(tmp
+3, '/');
1972 if (tmp
&& tmp
< s
) {
1973 log_debug(LD_DIR
,"Skipping over 'http[s]://hostname/' string");
1979 /* Check if the header is well formed (next sequence
1980 * should be HTTP/1.X\r\n). Assumes we're supporting 1.0? */
1984 char *e
= (char *)eat_whitespace_no_nl(s
);
1985 if (2 != tor_sscanf(e
, "HTTP/1.%u%c", &minor_ver
, &ch
)) {
1992 *url_out
= tor_memdup_nulterm(start
, s
-start
);
1993 *command_out
= tor_memdup_nulterm(command
, end_of_command
- command
);
1997 /** Return a copy of the first HTTP header in <b>headers</b> whose key is
1998 * <b>which</b>. The key should be given with a terminating colon and space;
1999 * this function copies everything after, up to but not including the
2000 * following \\r\\n. */
2002 http_get_header(const char *headers
, const char *which
)
2004 const char *cp
= headers
;
2006 if (!strcasecmpstart(cp
, which
)) {
2008 cp
+= strlen(which
);
2009 if ((eos
= strchr(cp
,'\r')))
2010 return tor_strndup(cp
, eos
-cp
);
2012 return tor_strdup(cp
);
2014 cp
= strchr(cp
, '\n');
2021 /** If <b>headers</b> indicates that a proxy was involved, then rewrite
2022 * <b>conn</b>-\>address to describe our best guess of the address that
2023 * originated this HTTP request. */
2025 http_set_address_origin(const char *headers
, connection_t
*conn
)
2029 fwd
= http_get_header(headers
, "Forwarded-For: ");
2031 fwd
= http_get_header(headers
, "X-Forwarded-For: ");
2034 if (tor_addr_parse(&toraddr
,fwd
) == -1 ||
2035 tor_addr_is_internal(&toraddr
,0)) {
2036 log_debug(LD_DIR
, "Ignoring local/internal IP %s", escaped(fwd
));
2041 tor_free(conn
->address
);
2042 conn
->address
= tor_strdup(fwd
);
2047 /** Parse an HTTP response string <b>headers</b> of the form
2049 * "HTTP/1.\%d \%d\%s\r\n...".
2052 * If it's well-formed, assign the status code to *<b>code</b> and
2053 * return 0. Otherwise, return -1.
2055 * On success: If <b>date</b> is provided, set *date to the Date
2056 * header in the http headers, or 0 if no such header is found. If
2057 * <b>compression</b> is provided, set *<b>compression</b> to the
2058 * compression method given in the Content-Encoding header, or 0 if no
2059 * such header is found, or -1 if the value of the header is not
2060 * recognized. If <b>reason</b> is provided, strdup the reason string
2064 parse_http_response(const char *headers
, int *code
, time_t *date
,
2065 compress_method_t
*compression
, char **reason
)
2068 char datestr
[RFC1123_TIME_LEN
+1];
2069 smartlist_t
*parsed_headers
;
2070 tor_assert(headers
);
2073 while (TOR_ISSPACE(*headers
)) headers
++; /* tolerate leading whitespace */
2075 if (tor_sscanf(headers
, "HTTP/1.%u %u", &n1
, &n2
) < 2 ||
2076 (n1
!= 0 && n1
!= 1) ||
2077 (n2
< 100 || n2
>= 600)) {
2078 log_warn(LD_HTTP
,"Failed to parse header %s",escaped(headers
));
2083 parsed_headers
= smartlist_new();
2084 smartlist_split_string(parsed_headers
, headers
, "\n",
2085 SPLIT_SKIP_SPACE
|SPLIT_IGNORE_BLANK
, -1);
2087 smartlist_t
*status_line_elements
= smartlist_new();
2088 tor_assert(smartlist_len(parsed_headers
));
2089 smartlist_split_string(status_line_elements
,
2090 smartlist_get(parsed_headers
, 0),
2091 " ", SPLIT_SKIP_SPACE
|SPLIT_IGNORE_BLANK
, 3);
2092 tor_assert(smartlist_len(status_line_elements
) <= 3);
2093 if (smartlist_len(status_line_elements
) == 3) {
2094 *reason
= smartlist_get(status_line_elements
, 2);
2095 smartlist_set(status_line_elements
, 2, NULL
); /* Prevent free */
2097 SMARTLIST_FOREACH(status_line_elements
, char *, cp
, tor_free(cp
));
2098 smartlist_free(status_line_elements
);
2102 SMARTLIST_FOREACH(parsed_headers
, const char *, s
,
2103 if (!strcmpstart(s
, "Date: ")) {
2104 strlcpy(datestr
, s
+6, sizeof(datestr
));
2105 /* This will do nothing on failure, so we don't need to check
2106 the result. We shouldn't warn, since there are many other valid
2107 date formats besides the one we use. */
2108 parse_rfc1123_time(datestr
, date
);
2113 const char *enc
= NULL
;
2114 SMARTLIST_FOREACH(parsed_headers
, const char *, s
,
2115 if (!strcmpstart(s
, "Content-Encoding: ")) {
2120 *compression
= NO_METHOD
;
2122 *compression
= compression_method_get_by_name(enc
);
2124 if (*compression
== UNKNOWN_METHOD
)
2125 log_info(LD_HTTP
, "Unrecognized content encoding: %s. Trying to deal.",
2129 SMARTLIST_FOREACH(parsed_headers
, char *, s
, tor_free(s
));
2130 smartlist_free(parsed_headers
);
2135 /** Return true iff <b>body</b> doesn't start with a plausible router or
2136 * network-status or microdescriptor opening. This is a sign of possible
2139 body_is_plausible(const char *body
, size_t len
, int purpose
)
2143 return 1; /* empty bodies don't need decompression */
2146 if (purpose
== DIR_PURPOSE_FETCH_MICRODESC
) {
2147 return (!strcmpstart(body
,"onion-key"));
2150 if (!strcmpstart(body
,"router") ||
2151 !strcmpstart(body
,"network-status"))
2153 for (i
=0;i
<32;++i
) {
2154 if (!TOR_ISPRINT(body
[i
]) && !TOR_ISSPACE(body
[i
]))
2161 /** Called when we've just fetched a bunch of router descriptors in
2162 * <b>body</b>. The list <b>which</b>, if present, holds digests for
2163 * descriptors we requested: descriptor digests if <b>descriptor_digests</b>
2164 * is true, or identity digests otherwise. Parse the descriptors, validate
2165 * them, and annotate them as having purpose <b>purpose</b> and as having been
2166 * downloaded from <b>source</b>.
2168 * Return the number of routers actually added. */
2170 load_downloaded_routers(const char *body
, smartlist_t
*which
,
2171 int descriptor_digests
,
2176 char time_buf
[ISO_TIME_LEN
+1];
2178 int general
= router_purpose
== ROUTER_PURPOSE_GENERAL
;
2179 format_iso_time(time_buf
, time(NULL
));
2182 if (tor_snprintf(buf
, sizeof(buf
),
2183 "@downloaded-at %s\n"
2185 "%s%s%s", time_buf
, escaped(source
),
2186 !general
? "@purpose " : "",
2187 !general
? router_purpose_to_string(router_purpose
) : "",
2188 !general
? "\n" : "")<0)
2191 added
= router_load_routers_from_string(body
, NULL
, SAVED_NOWHERE
, which
,
2192 descriptor_digests
, buf
);
2193 if (added
&& general
)
2194 control_event_bootstrap(BOOTSTRAP_STATUS_LOADING_DESCRIPTORS
,
2195 count_loading_descriptors_progress());
2199 static int handle_response_fetch_consensus(dir_connection_t
*,
2200 const response_handler_args_t
*);
2201 static int handle_response_fetch_certificate(dir_connection_t
*,
2202 const response_handler_args_t
*);
2203 static int handle_response_fetch_status_vote(dir_connection_t
*,
2204 const response_handler_args_t
*);
2205 static int handle_response_fetch_detached_signatures(dir_connection_t
*,
2206 const response_handler_args_t
*);
2207 static int handle_response_fetch_desc(dir_connection_t
*,
2208 const response_handler_args_t
*);
2209 static int handle_response_upload_dir(dir_connection_t
*,
2210 const response_handler_args_t
*);
2211 static int handle_response_upload_vote(dir_connection_t
*,
2212 const response_handler_args_t
*);
2213 static int handle_response_upload_signatures(dir_connection_t
*,
2214 const response_handler_args_t
*);
2215 static int handle_response_fetch_renddesc_v2(dir_connection_t
*,
2216 const response_handler_args_t
*);
2217 static int handle_response_upload_renddesc_v2(dir_connection_t
*,
2218 const response_handler_args_t
*);
2219 static int handle_response_upload_hsdesc(dir_connection_t
*,
2220 const response_handler_args_t
*);
2223 dir_client_decompress_response_body(char **bodyp
, size_t *bodylenp
,
2224 dir_connection_t
*conn
,
2225 compress_method_t compression
,
2226 int anonymized_connection
)
2229 const char *body
= *bodyp
;
2230 size_t body_len
= *bodylenp
;
2231 int allow_partial
= (conn
->base_
.purpose
== DIR_PURPOSE_FETCH_SERVERDESC
||
2232 conn
->base_
.purpose
== DIR_PURPOSE_FETCH_EXTRAINFO
||
2233 conn
->base_
.purpose
== DIR_PURPOSE_FETCH_MICRODESC
);
2235 int plausible
= body_is_plausible(body
, body_len
, conn
->base_
.purpose
);
2237 if (plausible
&& compression
== NO_METHOD
) {
2241 int severity
= LOG_DEBUG
;
2242 char *new_body
= NULL
;
2244 const char *description1
, *description2
;
2245 int want_to_try_both
= 0;
2247 compress_method_t guessed
= detect_compression_method(body
, body_len
);
2249 description1
= compression_method_get_human_name(compression
);
2251 if (BUG(description1
== NULL
))
2252 description1
= compression_method_get_human_name(UNKNOWN_METHOD
);
2254 if (guessed
== UNKNOWN_METHOD
&& !plausible
)
2255 description2
= "confusing binary junk";
2257 description2
= compression_method_get_human_name(guessed
);
2259 /* Tell the user if we don't believe what we're told about compression.*/
2260 want_to_try_both
= (compression
== UNKNOWN_METHOD
||
2261 guessed
!= compression
);
2262 if (want_to_try_both
) {
2263 severity
= LOG_PROTOCOL_WARN
;
2266 tor_log(severity
, LD_HTTP
,
2267 "HTTP body from server '%s:%d' was labeled as %s, "
2268 "%s it seems to be %s.%s",
2269 conn
->base_
.address
, conn
->base_
.port
, description1
,
2270 guessed
!= compression
?"but":"and",
2272 (compression
>0 && guessed
>0 && want_to_try_both
)?
2273 " Trying both.":"");
2275 /* Try declared compression first if we can.
2276 * tor_compress_supports_method() also returns true for NO_METHOD.
2277 * Ensure that the server is not sending us data compressed using a
2278 * compression method that is not allowed for anonymous connections. */
2279 if (anonymized_connection
&&
2280 ! allowed_anonymous_connection_compression_method(compression
)) {
2281 warn_disallowed_anonymous_compression_method(compression
);
2286 if (tor_compress_supports_method(compression
)) {
2287 tor_uncompress(&new_body
, &new_len
, body
, body_len
, compression
,
2288 !allow_partial
, LOG_PROTOCOL_WARN
);
2290 /* We succeeded with the declared compression method. Great! */
2296 /* Okay, if that didn't work, and we think that it was compressed
2297 * differently, try that. */
2298 if (anonymized_connection
&&
2299 ! allowed_anonymous_connection_compression_method(guessed
)) {
2300 warn_disallowed_anonymous_compression_method(guessed
);
2305 if (tor_compress_supports_method(guessed
) &&
2306 compression
!= guessed
) {
2307 tor_uncompress(&new_body
, &new_len
, body
, body_len
, guessed
,
2308 !allow_partial
, LOG_INFO
);
2311 /* If we're pretty sure that we have a compressed directory, and
2312 * we didn't manage to uncompress it, then warn and bail. */
2313 if (!plausible
&& !new_body
) {
2314 log_fn(LOG_PROTOCOL_WARN
, LD_HTTP
,
2315 "Unable to decompress HTTP body (tried %s%s%s, server '%s:%d').",
2317 tried_both
?" and ":"",
2318 tried_both
?description2
:"",
2319 conn
->base_
.address
, conn
->base_
.port
);
2330 *bodylenp
= new_len
;
2339 /** We are a client, and we've finished reading the server's
2340 * response. Parse it and act appropriately.
2342 * If we're still happy with using this directory server in the future, return
2343 * 0. Otherwise return -1; and the caller should consider trying the request
2346 * The caller will take care of marking the connection for close.
2349 connection_dir_client_reached_eof(dir_connection_t
*conn
)
2352 char *headers
= NULL
;
2353 char *reason
= NULL
;
2354 size_t body_len
= 0;
2356 time_t date_header
= 0;
2358 compress_method_t compression
;
2361 int allow_partial
= (conn
->base_
.purpose
== DIR_PURPOSE_FETCH_SERVERDESC
||
2362 conn
->base_
.purpose
== DIR_PURPOSE_FETCH_EXTRAINFO
||
2363 conn
->base_
.purpose
== DIR_PURPOSE_FETCH_MICRODESC
);
2364 size_t received_bytes
;
2365 const int anonymized_connection
=
2366 purpose_needs_anonymity(conn
->base_
.purpose
,
2367 conn
->router_purpose
,
2368 conn
->requested_resource
);
2370 received_bytes
= connection_get_inbuf_len(TO_CONN(conn
));
2372 switch (connection_fetch_from_buf_http(TO_CONN(conn
),
2373 &headers
, MAX_HEADERS_SIZE
,
2374 &body
, &body_len
, MAX_DIR_DL_SIZE
,
2376 case -1: /* overflow */
2377 log_warn(LD_PROTOCOL
,
2378 "'fetch' response too large (server '%s:%d'). Closing.",
2379 conn
->base_
.address
, conn
->base_
.port
);
2383 "'fetch' response not all here, but we're at eof. Closing.");
2385 /* case 1, fall through */
2388 if (parse_http_response(headers
, &status_code
, &date_header
,
2389 &compression
, &reason
) < 0) {
2390 log_warn(LD_HTTP
,"Unparseable headers (server '%s:%d'). Closing.",
2391 conn
->base_
.address
, conn
->base_
.port
);
2396 if (!reason
) reason
= tor_strdup("[no reason given]");
2398 tor_log(LOG_DEBUG
, LD_DIR
,
2399 "Received response from directory server '%s:%d': %d %s "
2400 "(purpose: %d, response size: " U64_FORMAT
2401 #ifdef MEASUREMENTS_21206
2402 ", data cells received: %d, data cells sent: %d"
2404 ", compression: %d)",
2405 conn
->base_
.address
, conn
->base_
.port
, status_code
,
2406 escaped(reason
), conn
->base_
.purpose
,
2407 U64_PRINTF_ARG(received_bytes
),
2408 #ifdef MEASUREMENTS_21206
2409 conn
->data_cells_received
, conn
->data_cells_sent
,
2413 if (conn
->guard_state
) {
2414 /* we count the connection as successful once we can read from it. We do
2415 * not, however, delay use of the circuit here, since it's just for a
2416 * one-hop directory request. */
2417 /* XXXXprop271 note that this will not do the right thing for other
2418 * waiting circuits that would be triggered by this circuit becoming
2419 * complete/usable. But that's ok, I think.
2421 entry_guard_succeeded(&conn
->guard_state
);
2422 circuit_guard_state_free(conn
->guard_state
);
2423 conn
->guard_state
= NULL
;
2426 /* now check if it's got any hints for us about our IP address. */
2427 if (conn
->dirconn_direct
) {
2428 char *guess
= http_get_header(headers
, X_ADDRESS_HEADER
);
2430 router_new_address_suggestion(guess
, conn
);
2435 if (date_header
> 0) {
2436 /* The date header was written very soon after we sent our request,
2437 * so compute the skew as the difference between sending the request
2438 * and the date header. (We used to check now-date_header, but that's
2439 * inaccurate if we spend a lot of time downloading.)
2441 apparent_skew
= conn
->base_
.timestamp_lastwritten
- date_header
;
2442 if (labs(apparent_skew
)>ALLOW_DIRECTORY_TIME_SKEW
) {
2443 int trusted
= router_digest_is_trusted_dir(conn
->identity_digest
);
2444 clock_skew_warning(TO_CONN(conn
), apparent_skew
, trusted
, LD_HTTP
,
2445 "directory", "DIRSERV");
2446 skewed
= 1; /* don't check the recommended-versions line */
2448 log_debug(LD_HTTP
, "Time on received directory is within tolerance; "
2449 "we are %ld seconds skewed. (That's okay.)", apparent_skew
);
2452 (void) skewed
; /* skewed isn't used yet. */
2454 if (status_code
== 503) {
2457 const char *id_digest
= conn
->identity_digest
;
2458 log_info(LD_DIR
,"Received http status code %d (%s) from server "
2459 "'%s:%d'. I'll try again soon.",
2460 status_code
, escaped(reason
), conn
->base_
.address
,
2462 time_t now
= approx_time();
2463 if ((rs
= router_get_mutable_consensus_status_by_id(id_digest
)))
2464 rs
->last_dir_503_at
= now
;
2465 if ((ds
= router_get_fallback_dirserver_by_digest(id_digest
)))
2466 ds
->fake_status
.last_dir_503_at
= now
;
2472 if (dir_client_decompress_response_body(&body
, &body_len
,
2473 conn
, compression
, anonymized_connection
) < 0) {
2478 response_handler_args_t args
;
2479 memset(&args
, 0, sizeof(args
));
2480 args
.status_code
= status_code
;
2481 args
.reason
= reason
;
2483 args
.body_len
= body_len
;
2484 args
.headers
= headers
;
2486 switch (conn
->base_
.purpose
) {
2487 case DIR_PURPOSE_FETCH_CONSENSUS
:
2488 rv
= handle_response_fetch_consensus(conn
, &args
);
2490 case DIR_PURPOSE_FETCH_CERTIFICATE
:
2491 rv
= handle_response_fetch_certificate(conn
, &args
);
2493 case DIR_PURPOSE_FETCH_STATUS_VOTE
:
2494 rv
= handle_response_fetch_status_vote(conn
, &args
);
2496 case DIR_PURPOSE_FETCH_DETACHED_SIGNATURES
:
2497 rv
= handle_response_fetch_detached_signatures(conn
, &args
);
2499 case DIR_PURPOSE_FETCH_SERVERDESC
:
2500 case DIR_PURPOSE_FETCH_EXTRAINFO
:
2501 rv
= handle_response_fetch_desc(conn
, &args
);
2503 case DIR_PURPOSE_FETCH_MICRODESC
:
2504 rv
= handle_response_fetch_microdesc(conn
, &args
);
2506 case DIR_PURPOSE_FETCH_RENDDESC_V2
:
2507 rv
= handle_response_fetch_renddesc_v2(conn
, &args
);
2509 case DIR_PURPOSE_UPLOAD_DIR
:
2510 rv
= handle_response_upload_dir(conn
, &args
);
2512 case DIR_PURPOSE_UPLOAD_SIGNATURES
:
2513 rv
= handle_response_upload_signatures(conn
, &args
);
2515 case DIR_PURPOSE_UPLOAD_VOTE
:
2516 rv
= handle_response_upload_vote(conn
, &args
);
2518 case DIR_PURPOSE_UPLOAD_RENDDESC_V2
:
2519 rv
= handle_response_upload_renddesc_v2(conn
, &args
);
2521 case DIR_PURPOSE_UPLOAD_HSDESC
:
2522 rv
= handle_response_upload_hsdesc(conn
, &args
);
2524 case DIR_PURPOSE_FETCH_HSDESC
:
2525 rv
= handle_response_fetch_hsdesc_v3(conn
, &args
);
2528 tor_assert_nonfatal_unreached();
2541 * Handler function: processes a response to a request for a networkstatus
2542 * consensus document by checking the consensus, storing it, and marking
2543 * router requests as reachable.
2546 handle_response_fetch_consensus(dir_connection_t
*conn
,
2547 const response_handler_args_t
*args
)
2549 tor_assert(conn
->base_
.purpose
== DIR_PURPOSE_FETCH_CONSENSUS
);
2550 const int status_code
= args
->status_code
;
2551 const char *body
= args
->body
;
2552 const size_t body_len
= args
->body_len
;
2553 const char *reason
= args
->reason
;
2554 const time_t now
= approx_time();
2556 const char *consensus
;
2557 char *new_consensus
= NULL
;
2558 const char *sourcename
;
2561 const char *flavname
= conn
->requested_resource
;
2562 if (status_code
!= 200) {
2563 int severity
= (status_code
== 304) ? LOG_INFO
: LOG_WARN
;
2564 tor_log(severity
, LD_DIR
,
2565 "Received http status code %d (%s) from server "
2566 "'%s:%d' while fetching consensus directory.",
2567 status_code
, escaped(reason
), conn
->base_
.address
,
2569 networkstatus_consensus_download_failed(status_code
, flavname
);
2573 if (looks_like_a_consensus_diff(body
, body_len
)) {
2574 /* First find our previous consensus. Maybe it's in ram, maybe not. */
2575 cached_dir_t
*cd
= dirserv_get_consensus(flavname
);
2576 const char *consensus_body
;
2577 char *owned_consensus
= NULL
;
2579 consensus_body
= cd
->dir
;
2581 owned_consensus
= networkstatus_read_cached_consensus(flavname
);
2582 consensus_body
= owned_consensus
;
2584 if (!consensus_body
) {
2585 log_warn(LD_DIR
, "Received a consensus diff, but we can't find "
2586 "any %s-flavored consensus in our current cache.",flavname
);
2587 networkstatus_consensus_download_failed(0, flavname
);
2588 // XXXX if this happens too much, see below
2592 new_consensus
= consensus_diff_apply(consensus_body
, body
);
2593 tor_free(owned_consensus
);
2594 if (new_consensus
== NULL
) {
2595 log_warn(LD_DIR
, "Could not apply consensus diff received from server "
2596 "'%s:%d'", conn
->base_
.address
, conn
->base_
.port
);
2597 // XXXX If this happens too many times, we should maybe not use
2598 // XXXX this directory for diffs any more?
2599 networkstatus_consensus_download_failed(0, flavname
);
2602 log_info(LD_DIR
, "Applied consensus diff (size %d) from server "
2603 "'%s:%d', resulting in a new consensus document (size %d).",
2604 (int)body_len
, conn
->base_
.address
, conn
->base_
.port
,
2605 (int)strlen(new_consensus
));
2606 consensus
= new_consensus
;
2607 sourcename
= "generated based on a diff";
2609 log_info(LD_DIR
,"Received consensus directory (body size %d) from server "
2610 "'%s:%d'", (int)body_len
, conn
->base_
.address
, conn
->base_
.port
);
2612 sourcename
= "downloaded";
2615 if ((r
=networkstatus_set_current_consensus(consensus
, flavname
, 0,
2616 conn
->identity_digest
))<0) {
2617 log_fn(r
<-1?LOG_WARN
:LOG_INFO
, LD_DIR
,
2618 "Unable to load %s consensus directory %s from "
2619 "server '%s:%d'. I'll try again soon.",
2620 flavname
, sourcename
, conn
->base_
.address
, conn
->base_
.port
);
2621 networkstatus_consensus_download_failed(0, flavname
);
2622 tor_free(new_consensus
);
2626 /* If we launched other fetches for this consensus, cancel them. */
2627 connection_dir_close_consensus_fetches(conn
, flavname
);
2629 /* update the list of routers and directory guards */
2630 routers_update_all_from_networkstatus(now
, 3);
2631 update_microdescs_from_networkstatus(now
);
2632 directory_info_has_arrived(now
, 0, 0);
2634 if (authdir_mode_v3(get_options())) {
2635 sr_act_post_consensus(
2636 networkstatus_get_latest_consensus_by_flavor(FLAV_NS
));
2638 log_info(LD_DIR
, "Successfully loaded consensus.");
2640 tor_free(new_consensus
);
2645 * Handler function: processes a response to a request for one or more
2646 * authority certificates
2649 handle_response_fetch_certificate(dir_connection_t
*conn
,
2650 const response_handler_args_t
*args
)
2652 tor_assert(conn
->base_
.purpose
== DIR_PURPOSE_FETCH_CERTIFICATE
);
2653 const int status_code
= args
->status_code
;
2654 const char *reason
= args
->reason
;
2655 const char *body
= args
->body
;
2656 const size_t body_len
= args
->body_len
;
2658 if (status_code
!= 200) {
2660 "Received http status code %d (%s) from server "
2661 "'%s:%d' while fetching \"/tor/keys/%s\".",
2662 status_code
, escaped(reason
), conn
->base_
.address
,
2663 conn
->base_
.port
, conn
->requested_resource
);
2664 connection_dir_download_cert_failed(conn
, status_code
);
2667 log_info(LD_DIR
,"Received authority certificates (body size %d) from "
2669 (int)body_len
, conn
->base_
.address
, conn
->base_
.port
);
2672 * Tell trusted_dirs_load_certs_from_string() whether it was by fp
2676 if (!strcmpstart(conn
->requested_resource
, "fp/")) {
2677 src_code
= TRUSTED_DIRS_CERTS_SRC_DL_BY_ID_DIGEST
;
2678 } else if (!strcmpstart(conn
->requested_resource
, "fp-sk/")) {
2679 src_code
= TRUSTED_DIRS_CERTS_SRC_DL_BY_ID_SK_DIGEST
;
2682 if (src_code
!= -1) {
2683 if (trusted_dirs_load_certs_from_string(body
, src_code
, 1,
2684 conn
->identity_digest
)<0) {
2685 log_warn(LD_DIR
, "Unable to parse fetched certificates");
2686 /* if we fetched more than one and only some failed, the successful
2687 * ones got flushed to disk so it's safe to call this on them */
2688 connection_dir_download_cert_failed(conn
, status_code
);
2690 time_t now
= approx_time();
2691 directory_info_has_arrived(now
, 0, 0);
2692 log_info(LD_DIR
, "Successfully loaded certificates from fetch.");
2696 "Couldn't figure out what to do with fetched certificates for "
2697 "unknown resource %s",
2698 conn
->requested_resource
);
2699 connection_dir_download_cert_failed(conn
, status_code
);
2705 * Handler function: processes a response to a request for an authority's
2706 * current networkstatus vote.
2709 handle_response_fetch_status_vote(dir_connection_t
*conn
,
2710 const response_handler_args_t
*args
)
2712 tor_assert(conn
->base_
.purpose
== DIR_PURPOSE_FETCH_STATUS_VOTE
);
2713 const int status_code
= args
->status_code
;
2714 const char *reason
= args
->reason
;
2715 const char *body
= args
->body
;
2716 const size_t body_len
= args
->body_len
;
2720 log_info(LD_DIR
,"Got votes (body size %d) from server %s:%d",
2721 (int)body_len
, conn
->base_
.address
, conn
->base_
.port
);
2722 if (status_code
!= 200) {
2724 "Received http status code %d (%s) from server "
2725 "'%s:%d' while fetching \"/tor/status-vote/next/%s.z\".",
2726 status_code
, escaped(reason
), conn
->base_
.address
,
2727 conn
->base_
.port
, conn
->requested_resource
);
2730 dirvote_add_vote(body
, &msg
, &st
);
2732 log_warn(LD_DIR
, "Error adding retrieved vote: %s", msg
);
2734 log_info(LD_DIR
, "Added vote(s) successfully [msg: %s]", msg
);
2741 * Handler function: processes a response to a request for the signatures
2742 * that an authority knows about on a given consensus.
2745 handle_response_fetch_detached_signatures(dir_connection_t
*conn
,
2746 const response_handler_args_t
*args
)
2748 tor_assert(conn
->base_
.purpose
== DIR_PURPOSE_FETCH_DETACHED_SIGNATURES
);
2749 const int status_code
= args
->status_code
;
2750 const char *reason
= args
->reason
;
2751 const char *body
= args
->body
;
2752 const size_t body_len
= args
->body_len
;
2754 const char *msg
= NULL
;
2755 log_info(LD_DIR
,"Got detached signatures (body size %d) from server %s:%d",
2756 (int)body_len
, conn
->base_
.address
, conn
->base_
.port
);
2757 if (status_code
!= 200) {
2759 "Received http status code %d (%s) from server '%s:%d' while fetching "
2760 "\"/tor/status-vote/next/consensus-signatures.z\".",
2761 status_code
, escaped(reason
), conn
->base_
.address
,
2765 if (dirvote_add_signatures(body
, conn
->base_
.address
, &msg
)<0) {
2766 log_warn(LD_DIR
, "Problem adding detached signatures from %s:%d: %s",
2767 conn
->base_
.address
, conn
->base_
.port
, msg
?msg
:"???");
2774 * Handler function: processes a response to a request for a group of server
2775 * descriptors or an extrainfo documents.
2778 handle_response_fetch_desc(dir_connection_t
*conn
,
2779 const response_handler_args_t
*args
)
2781 tor_assert(conn
->base_
.purpose
== DIR_PURPOSE_FETCH_SERVERDESC
||
2782 conn
->base_
.purpose
== DIR_PURPOSE_FETCH_EXTRAINFO
);
2783 const int status_code
= args
->status_code
;
2784 const char *reason
= args
->reason
;
2785 const char *body
= args
->body
;
2786 const size_t body_len
= args
->body_len
;
2788 int was_ei
= conn
->base_
.purpose
== DIR_PURPOSE_FETCH_EXTRAINFO
;
2789 smartlist_t
*which
= NULL
;
2790 int n_asked_for
= 0;
2791 int descriptor_digests
= conn
->requested_resource
&&
2792 !strcmpstart(conn
->requested_resource
,"d/");
2793 log_info(LD_DIR
,"Received %s (body size %d) from server '%s:%d'",
2794 was_ei
? "extra server info" : "server info",
2795 (int)body_len
, conn
->base_
.address
, conn
->base_
.port
);
2796 if (conn
->requested_resource
&&
2797 (!strcmpstart(conn
->requested_resource
,"d/") ||
2798 !strcmpstart(conn
->requested_resource
,"fp/"))) {
2799 which
= smartlist_new();
2800 dir_split_resource_into_fingerprints(conn
->requested_resource
+
2801 (descriptor_digests
? 2 : 3),
2803 n_asked_for
= smartlist_len(which
);
2805 if (status_code
!= 200) {
2806 int dir_okay
= status_code
== 404 ||
2807 (status_code
== 400 && !strcmp(reason
, "Servers unavailable."));
2808 /* 404 means that it didn't have them; no big deal.
2809 * Older (pre-0.1.1.8) servers said 400 Servers unavailable instead. */
2810 log_fn(dir_okay
? LOG_INFO
: LOG_WARN
, LD_DIR
,
2811 "Received http status code %d (%s) from server '%s:%d' "
2812 "while fetching \"/tor/server/%s\". I'll try again soon.",
2813 status_code
, escaped(reason
), conn
->base_
.address
,
2814 conn
->base_
.port
, conn
->requested_resource
);
2816 connection_dir_download_routerdesc_failed(conn
);
2818 dir_routerdesc_download_failed(which
, status_code
,
2819 conn
->router_purpose
,
2820 was_ei
, descriptor_digests
);
2821 SMARTLIST_FOREACH(which
, char *, cp
, tor_free(cp
));
2822 smartlist_free(which
);
2824 return dir_okay
? 0 : -1;
2826 /* Learn the routers, assuming we requested by fingerprint or "all"
2829 * We use "authority" to fetch our own descriptor for
2830 * testing, and to fetch bridge descriptors for bootstrapping. Ignore
2831 * the output of "authority" requests unless we are using bridges,
2832 * since otherwise they'll be the response from reachability tests,
2833 * and we don't really want to add that to our routerlist. */
2834 if (which
|| (conn
->requested_resource
&&
2835 (!strcmpstart(conn
->requested_resource
, "all") ||
2836 (!strcmpstart(conn
->requested_resource
, "authority") &&
2837 get_options()->UseBridges
)))) {
2838 /* as we learn from them, we remove them from 'which' */
2840 router_load_extrainfo_from_string(body
, NULL
, SAVED_NOWHERE
, which
,
2841 descriptor_digests
);
2843 //router_load_routers_from_string(body, NULL, SAVED_NOWHERE, which,
2844 // descriptor_digests, conn->router_purpose);
2845 if (load_downloaded_routers(body
, which
, descriptor_digests
,
2846 conn
->router_purpose
,
2847 conn
->base_
.address
)) {
2848 time_t now
= approx_time();
2849 directory_info_has_arrived(now
, 0, 1);
2853 if (which
) { /* mark remaining ones as failed */
2854 log_info(LD_DIR
, "Received %d/%d %s requested from %s:%d",
2855 n_asked_for
-smartlist_len(which
), n_asked_for
,
2856 was_ei
? "extra-info documents" : "router descriptors",
2857 conn
->base_
.address
, (int)conn
->base_
.port
);
2858 if (smartlist_len(which
)) {
2859 dir_routerdesc_download_failed(which
, status_code
,
2860 conn
->router_purpose
,
2861 was_ei
, descriptor_digests
);
2863 SMARTLIST_FOREACH(which
, char *, cp
, tor_free(cp
));
2864 smartlist_free(which
);
2866 if (directory_conn_is_self_reachability_test(conn
))
2867 router_dirport_found_reachable();
2873 * Handler function: processes a response to a request for a group of
2877 handle_response_fetch_microdesc(dir_connection_t
*conn
,
2878 const response_handler_args_t
*args
)
2880 tor_assert(conn
->base_
.purpose
== DIR_PURPOSE_FETCH_MICRODESC
);
2881 const int status_code
= args
->status_code
;
2882 const char *reason
= args
->reason
;
2883 const char *body
= args
->body
;
2884 const size_t body_len
= args
->body_len
;
2886 smartlist_t
*which
= NULL
;
2887 log_info(LD_DIR
,"Received answer to microdescriptor request (status %d, "
2888 "body size %d) from server '%s:%d'",
2889 status_code
, (int)body_len
, conn
->base_
.address
,
2891 tor_assert(conn
->requested_resource
&&
2892 !strcmpstart(conn
->requested_resource
, "d/"));
2893 tor_assert_nonfatal(!tor_mem_is_zero(conn
->identity_digest
, DIGEST_LEN
));
2894 which
= smartlist_new();
2895 dir_split_resource_into_fingerprints(conn
->requested_resource
+2,
2897 DSR_DIGEST256
|DSR_BASE64
);
2898 if (status_code
!= 200) {
2899 log_info(LD_DIR
, "Received status code %d (%s) from server "
2900 "'%s:%d' while fetching \"/tor/micro/%s\". I'll try again "
2902 status_code
, escaped(reason
), conn
->base_
.address
,
2903 (int)conn
->base_
.port
, conn
->requested_resource
);
2904 dir_microdesc_download_failed(which
, status_code
, conn
->identity_digest
);
2905 SMARTLIST_FOREACH(which
, char *, cp
, tor_free(cp
));
2906 smartlist_free(which
);
2910 time_t now
= approx_time();
2911 mds
= microdescs_add_to_cache(get_microdesc_cache(),
2912 body
, body
+body_len
, SAVED_NOWHERE
, 0,
2914 if (smartlist_len(which
)) {
2915 /* Mark remaining ones as failed. */
2916 dir_microdesc_download_failed(which
, status_code
, conn
->identity_digest
);
2918 if (mds
&& smartlist_len(mds
)) {
2919 control_event_bootstrap(BOOTSTRAP_STATUS_LOADING_DESCRIPTORS
,
2920 count_loading_descriptors_progress());
2921 directory_info_has_arrived(now
, 0, 1);
2923 SMARTLIST_FOREACH(which
, char *, cp
, tor_free(cp
));
2924 smartlist_free(which
);
2925 smartlist_free(mds
);
2932 * Handler function: processes a response to a POST request to upload our
2933 * router descriptor.
2936 handle_response_upload_dir(dir_connection_t
*conn
,
2937 const response_handler_args_t
*args
)
2939 tor_assert(conn
->base_
.purpose
== DIR_PURPOSE_UPLOAD_DIR
);
2940 const int status_code
= args
->status_code
;
2941 const char *reason
= args
->reason
;
2942 const char *headers
= args
->headers
;
2944 switch (status_code
) {
2947 router_get_trusteddirserver_by_digest(conn
->identity_digest
);
2948 char *rejected_hdr
= http_get_header(headers
,
2949 "X-Descriptor-Not-New: ");
2951 if (!strcmp(rejected_hdr
, "Yes")) {
2952 log_info(LD_GENERAL
,
2953 "Authority '%s' declined our descriptor (not new)",
2955 /* XXXX use this information; be sure to upload next one
2957 /* XXXX++ On further thought, the task above implies that we're
2958 * basing our regenerate-descriptor time on when we uploaded the
2959 * last descriptor, not on the published time of the last
2960 * descriptor. If those are different, that's a bad thing to
2963 tor_free(rejected_hdr
);
2965 log_info(LD_GENERAL
,"eof (status 200) after uploading server "
2966 "descriptor: finished.");
2967 control_event_server_status(
2968 LOG_NOTICE
, "ACCEPTED_SERVER_DESCRIPTOR DIRAUTH=%s:%d",
2969 conn
->base_
.address
, conn
->base_
.port
);
2971 ds
->has_accepted_serverdesc
= 1;
2972 if (directories_have_accepted_server_descriptor())
2973 control_event_server_status(LOG_NOTICE
, "GOOD_SERVER_DESCRIPTOR");
2977 log_warn(LD_GENERAL
,"http status 400 (%s) response from "
2978 "dirserver '%s:%d'. Please correct.",
2979 escaped(reason
), conn
->base_
.address
, conn
->base_
.port
);
2980 control_event_server_status(LOG_WARN
,
2981 "BAD_SERVER_DESCRIPTOR DIRAUTH=%s:%d REASON=\"%s\"",
2982 conn
->base_
.address
, conn
->base_
.port
, escaped(reason
));
2985 log_warn(LD_GENERAL
,
2986 "HTTP status %d (%s) was unexpected while uploading "
2987 "descriptor to server '%s:%d'. Possibly the server is "
2989 status_code
, escaped(reason
), conn
->base_
.address
,
2993 /* return 0 in all cases, since we don't want to mark any
2994 * dirservers down just because they don't like us. */
3000 * Handler function: processes a response to POST request to upload our
3001 * own networkstatus vote.
3004 handle_response_upload_vote(dir_connection_t
*conn
,
3005 const response_handler_args_t
*args
)
3007 tor_assert(conn
->base_
.purpose
== DIR_PURPOSE_UPLOAD_VOTE
);
3008 const int status_code
= args
->status_code
;
3009 const char *reason
= args
->reason
;
3011 switch (status_code
) {
3013 log_notice(LD_DIR
,"Uploaded a vote to dirserver %s:%d",
3014 conn
->base_
.address
, conn
->base_
.port
);
3018 log_warn(LD_DIR
,"http status 400 (%s) response after uploading "
3019 "vote to dirserver '%s:%d'. Please correct.",
3020 escaped(reason
), conn
->base_
.address
, conn
->base_
.port
);
3023 log_warn(LD_GENERAL
,
3024 "HTTP status %d (%s) was unexpected while uploading "
3025 "vote to server '%s:%d'.",
3026 status_code
, escaped(reason
), conn
->base_
.address
,
3030 /* return 0 in all cases, since we don't want to mark any
3031 * dirservers down just because they don't like us. */
3036 * Handler function: processes a response to POST request to upload our
3037 * view of the signatures on the current consensus.
3040 handle_response_upload_signatures(dir_connection_t
*conn
,
3041 const response_handler_args_t
*args
)
3043 tor_assert(conn
->base_
.purpose
== DIR_PURPOSE_UPLOAD_SIGNATURES
);
3044 const int status_code
= args
->status_code
;
3045 const char *reason
= args
->reason
;
3047 switch (status_code
) {
3049 log_notice(LD_DIR
,"Uploaded signature(s) to dirserver %s:%d",
3050 conn
->base_
.address
, conn
->base_
.port
);
3054 log_warn(LD_DIR
,"http status 400 (%s) response after uploading "
3055 "signatures to dirserver '%s:%d'. Please correct.",
3056 escaped(reason
), conn
->base_
.address
, conn
->base_
.port
);
3059 log_warn(LD_GENERAL
,
3060 "HTTP status %d (%s) was unexpected while uploading "
3061 "signatures to server '%s:%d'.",
3062 status_code
, escaped(reason
), conn
->base_
.address
,
3066 /* return 0 in all cases, since we don't want to mark any
3067 * dirservers down just because they don't like us. */
3073 * Handler function: processes a response to a request for a v3 hidden service
3077 handle_response_fetch_hsdesc_v3(dir_connection_t
*conn
,
3078 const response_handler_args_t
*args
)
3080 const int status_code
= args
->status_code
;
3081 const char *reason
= args
->reason
;
3082 const char *body
= args
->body
;
3083 const size_t body_len
= args
->body_len
;
3085 tor_assert(conn
->hs_ident
);
3087 log_info(LD_REND
,"Received v3 hsdesc (body size %d, status %d (%s))",
3088 (int)body_len
, status_code
, escaped(reason
));
3090 switch (status_code
) {
3092 /* We got something: Try storing it in the cache. */
3093 if (hs_cache_store_as_client(body
, &conn
->hs_ident
->identity_pk
) < 0) {
3094 log_warn(LD_REND
, "Failed to store hidden service descriptor");
3096 log_info(LD_REND
, "Stored hidden service descriptor successfully.");
3097 TO_CONN(conn
)->purpose
= DIR_PURPOSE_HAS_FETCHED_HSDESC
;
3098 hs_client_desc_has_arrived(conn
->hs_ident
);
3102 /* Not there. We'll retry when connection_about_to_close_connection()
3103 * tries to clean this conn up. */
3104 log_info(LD_REND
, "Fetching hidden service v3 descriptor not found: "
3105 "Retrying at another directory.");
3106 /* TODO: Inform the control port */
3109 log_warn(LD_REND
, "Fetching v3 hidden service descriptor failed: "
3110 "http status 400 (%s). Dirserver didn't like our "
3111 "query? Retrying at another directory.",
3115 log_warn(LD_REND
, "Fetching v3 hidden service descriptor failed: "
3116 "http status %d (%s) response unexpected from HSDir server "
3117 "'%s:%d'. Retrying at another directory.",
3118 status_code
, escaped(reason
), TO_CONN(conn
)->address
,
3119 TO_CONN(conn
)->port
);
3127 * Handler function: processes a response to a request for a v2 hidden service
3131 handle_response_fetch_renddesc_v2(dir_connection_t
*conn
,
3132 const response_handler_args_t
*args
)
3134 tor_assert(conn
->base_
.purpose
== DIR_PURPOSE_FETCH_RENDDESC_V2
);
3135 const int status_code
= args
->status_code
;
3136 const char *reason
= args
->reason
;
3137 const char *body
= args
->body
;
3138 const size_t body_len
= args
->body_len
;
3140 #define SEND_HS_DESC_FAILED_EVENT(reason) \
3141 (control_event_hs_descriptor_failed(conn->rend_data, \
3142 conn->identity_digest, \
3144 #define SEND_HS_DESC_FAILED_CONTENT() \
3145 (control_event_hs_descriptor_content( \
3146 rend_data_get_address(conn->rend_data), \
3147 conn->requested_resource, \
3148 conn->identity_digest, \
3151 tor_assert(conn
->rend_data
);
3152 log_info(LD_REND
,"Received rendezvous descriptor (body size %d, status %d "
3154 (int)body_len
, status_code
, escaped(reason
));
3155 switch (status_code
) {
3158 rend_cache_entry_t
*entry
= NULL
;
3160 if (rend_cache_store_v2_desc_as_client(body
,
3161 conn
->requested_resource
,
3162 conn
->rend_data
, &entry
) < 0) {
3163 log_warn(LD_REND
,"Fetching v2 rendezvous descriptor failed. "
3164 "Retrying at another directory.");
3165 /* We'll retry when connection_about_to_close_connection()
3166 * cleans this dir conn up. */
3167 SEND_HS_DESC_FAILED_EVENT("BAD_DESC");
3168 SEND_HS_DESC_FAILED_CONTENT();
3170 char service_id
[REND_SERVICE_ID_LEN_BASE32
+ 1];
3171 /* Should never be NULL here if we found the descriptor. */
3173 rend_get_service_id(entry
->parsed
->pk
, service_id
);
3175 /* success. notify pending connections about this. */
3176 log_info(LD_REND
, "Successfully fetched v2 rendezvous "
3178 control_event_hs_descriptor_received(service_id
,
3180 conn
->identity_digest
);
3181 control_event_hs_descriptor_content(service_id
,
3182 conn
->requested_resource
,
3183 conn
->identity_digest
,
3185 conn
->base_
.purpose
= DIR_PURPOSE_HAS_FETCHED_RENDDESC_V2
;
3186 rend_client_desc_trynow(service_id
);
3187 memwipe(service_id
, 0, sizeof(service_id
));
3192 /* Not there. We'll retry when
3193 * connection_about_to_close_connection() cleans this conn up. */
3194 log_info(LD_REND
,"Fetching v2 rendezvous descriptor failed: "
3195 "Retrying at another directory.");
3196 SEND_HS_DESC_FAILED_EVENT("NOT_FOUND");
3197 SEND_HS_DESC_FAILED_CONTENT();
3200 log_warn(LD_REND
, "Fetching v2 rendezvous descriptor failed: "
3201 "http status 400 (%s). Dirserver didn't like our "
3202 "v2 rendezvous query? Retrying at another directory.",
3204 SEND_HS_DESC_FAILED_EVENT("QUERY_REJECTED");
3205 SEND_HS_DESC_FAILED_CONTENT();
3208 log_warn(LD_REND
, "Fetching v2 rendezvous descriptor failed: "
3209 "http status %d (%s) response unexpected while "
3210 "fetching v2 hidden service descriptor (server '%s:%d'). "
3211 "Retrying at another directory.",
3212 status_code
, escaped(reason
), conn
->base_
.address
,
3214 SEND_HS_DESC_FAILED_EVENT("UNEXPECTED");
3215 SEND_HS_DESC_FAILED_CONTENT();
3223 * Handler function: processes a response to a POST request to upload a v2
3224 * hidden service descriptor.
3227 handle_response_upload_renddesc_v2(dir_connection_t
*conn
,
3228 const response_handler_args_t
*args
)
3230 tor_assert(conn
->base_
.purpose
== DIR_PURPOSE_UPLOAD_RENDDESC_V2
);
3231 const int status_code
= args
->status_code
;
3232 const char *reason
= args
->reason
;
3234 #define SEND_HS_DESC_UPLOAD_FAILED_EVENT(reason) \
3235 (control_event_hs_descriptor_upload_failed( \
3236 conn->identity_digest, \
3237 rend_data_get_address(conn->rend_data), \
3240 log_info(LD_REND
,"Uploaded rendezvous descriptor (status %d "
3242 status_code
, escaped(reason
));
3243 /* Without the rend data, we'll have a problem identifying what has been
3244 * uploaded for which service. */
3245 tor_assert(conn
->rend_data
);
3246 switch (status_code
) {
3249 "Uploading rendezvous descriptor: finished with status "
3250 "200 (%s)", escaped(reason
));
3251 control_event_hs_descriptor_uploaded(conn
->identity_digest
,
3252 rend_data_get_address(conn
->rend_data
));
3253 rend_service_desc_has_uploaded(conn
->rend_data
);
3256 log_warn(LD_REND
,"http status 400 (%s) response from dirserver "
3257 "'%s:%d'. Malformed rendezvous descriptor?",
3258 escaped(reason
), conn
->base_
.address
, conn
->base_
.port
);
3259 SEND_HS_DESC_UPLOAD_FAILED_EVENT("UPLOAD_REJECTED");
3262 log_warn(LD_REND
,"http status %d (%s) response unexpected (server "
3264 status_code
, escaped(reason
), conn
->base_
.address
,
3266 SEND_HS_DESC_UPLOAD_FAILED_EVENT("UNEXPECTED");
3274 * Handler function: processes a response to a POST request to upload an
3275 * hidden service descriptor.
3278 handle_response_upload_hsdesc(dir_connection_t
*conn
,
3279 const response_handler_args_t
*args
)
3281 const int status_code
= args
->status_code
;
3282 const char *reason
= args
->reason
;
3285 tor_assert(conn
->base_
.purpose
== DIR_PURPOSE_UPLOAD_HSDESC
);
3287 log_info(LD_REND
, "Uploaded hidden service descriptor (status %d "
3289 status_code
, escaped(reason
));
3290 /* For this directory response, it MUST have an hidden service identifier on
3291 * this connection. */
3292 tor_assert(conn
->hs_ident
);
3293 switch (status_code
) {
3295 log_info(LD_REND
, "Uploading hidden service descriptor: "
3296 "finished with status 200 (%s)", escaped(reason
));
3297 /* XXX: Trigger control event. */
3300 log_fn(LOG_PROTOCOL_WARN
, LD_REND
,
3301 "Uploading hidden service descriptor: http "
3302 "status 400 (%s) response from dirserver "
3303 "'%s:%d'. Malformed hidden service descriptor?",
3304 escaped(reason
), conn
->base_
.address
, conn
->base_
.port
);
3305 /* XXX: Trigger control event. */
3308 log_warn(LD_REND
, "Uploading hidden service descriptor: http "
3309 "status %d (%s) response unexpected (server "
3311 status_code
, escaped(reason
), conn
->base_
.address
,
3313 /* XXX: Trigger control event. */
3320 /** Called when a directory connection reaches EOF. */
3322 connection_dir_reached_eof(dir_connection_t
*conn
)
3325 if (conn
->base_
.state
!= DIR_CONN_STATE_CLIENT_READING
) {
3326 log_info(LD_HTTP
,"conn reached eof, not reading. [state=%d] Closing.",
3328 connection_close_immediate(TO_CONN(conn
)); /* error: give up on flushing */
3329 connection_mark_for_close(TO_CONN(conn
));
3333 retval
= connection_dir_client_reached_eof(conn
);
3334 if (retval
== 0) /* success */
3335 conn
->base_
.state
= DIR_CONN_STATE_CLIENT_FINISHED
;
3336 connection_mark_for_close(TO_CONN(conn
));
3340 /** If any directory object is arriving, and it's over 10MB large, we're
3341 * getting DoS'd. (As of 0.1.2.x, raw directories are about 1MB, and we never
3342 * ask for more than 96 router descriptors at a time.)
3344 #define MAX_DIRECTORY_OBJECT_SIZE (10*(1<<20))
3346 #define MAX_VOTE_DL_SIZE (MAX_DIRECTORY_OBJECT_SIZE * 5)
3348 /** Read handler for directory connections. (That's connections <em>to</em>
3349 * directory servers and connections <em>at</em> directory servers.)
3352 connection_dir_process_inbuf(dir_connection_t
*conn
)
3356 tor_assert(conn
->base_
.type
== CONN_TYPE_DIR
);
3358 /* Directory clients write, then read data until they receive EOF;
3359 * directory servers read data until they get an HTTP command, then
3360 * write their response (when it's finished flushing, they mark for
3364 /* If we're on the dirserver side, look for a command. */
3365 if (conn
->base_
.state
== DIR_CONN_STATE_SERVER_COMMAND_WAIT
) {
3366 if (directory_handle_command(conn
) < 0) {
3367 connection_mark_for_close(TO_CONN(conn
));
3374 (TO_CONN(conn
)->purpose
== DIR_PURPOSE_FETCH_STATUS_VOTE
) ?
3375 MAX_VOTE_DL_SIZE
: MAX_DIRECTORY_OBJECT_SIZE
;
3377 if (connection_get_inbuf_len(TO_CONN(conn
)) > max_size
) {
3379 "Too much data received from directory connection (%s): "
3380 "denial of service attempt, or you need to upgrade?",
3381 conn
->base_
.address
);
3382 connection_mark_for_close(TO_CONN(conn
));
3386 if (!conn
->base_
.inbuf_reached_eof
)
3387 log_debug(LD_HTTP
,"Got data, not eof. Leaving on inbuf.");
3391 /** We are closing a dir connection: If <b>dir_conn</b> is a dir connection
3392 * that tried to fetch an HS descriptor, check if it successfuly fetched it,
3393 * or if we need to try again. */
3395 refetch_hsdesc_if_needed(dir_connection_t
*dir_conn
)
3397 connection_t
*conn
= TO_CONN(dir_conn
);
3399 /* If we were trying to fetch a v2 rend desc and did not succeed, retry as
3400 * needed. (If a fetch is successful, the connection state is changed to
3401 * DIR_PURPOSE_HAS_FETCHED_RENDDESC_V2 or DIR_PURPOSE_HAS_FETCHED_HSDESC to
3402 * mark that refetching is unnecessary.) */
3403 if (conn
->purpose
== DIR_PURPOSE_FETCH_RENDDESC_V2
&&
3404 dir_conn
->rend_data
&&
3405 rend_valid_v2_service_id(
3406 rend_data_get_address(dir_conn
->rend_data
))) {
3407 rend_client_refetch_v2_renddesc(dir_conn
->rend_data
);
3410 /* Check for v3 rend desc fetch */
3411 if (conn
->purpose
== DIR_PURPOSE_FETCH_HSDESC
&&
3412 dir_conn
->hs_ident
&&
3413 !ed25519_public_key_is_zero(&dir_conn
->hs_ident
->identity_pk
)) {
3414 hs_client_refetch_hsdesc(&dir_conn
->hs_ident
->identity_pk
);
3418 /** Called when we're about to finally unlink and free a directory connection:
3419 * perform necessary accounting and cleanup */
3421 connection_dir_about_to_close(dir_connection_t
*dir_conn
)
3423 connection_t
*conn
= TO_CONN(dir_conn
);
3425 if (conn
->state
< DIR_CONN_STATE_CLIENT_FINISHED
) {
3426 /* It's a directory connection and connecting or fetching
3427 * failed: forget about this router, and maybe try again. */
3428 connection_dir_request_failed(dir_conn
);
3431 refetch_hsdesc_if_needed(dir_conn
);
3434 /** Create an http response for the client <b>conn</b> out of
3435 * <b>status</b> and <b>reason_phrase</b>. Write it to <b>conn</b>.
3438 write_short_http_response(dir_connection_t
*conn
, int status
,
3439 const char *reason_phrase
)
3442 char *datestring
= NULL
;
3444 IF_BUG_ONCE(!reason_phrase
) { /* bullet-proofing */
3445 reason_phrase
= "unspecified";
3448 if (server_mode(get_options())) {
3449 /* include the Date: header, but only if we're a relay or bridge */
3450 char datebuf
[RFC1123_TIME_LEN
+1];
3451 format_rfc1123_time(datebuf
, time(NULL
));
3452 tor_asprintf(&datestring
, "Date: %s\r\n", datebuf
);
3455 tor_asprintf(&buf
, "HTTP/1.0 %d %s\r\n%s\r\n",
3456 status
, reason_phrase
, datestring
?datestring
:"");
3458 log_debug(LD_DIRSERV
,"Wrote status 'HTTP/1.0 %d %s'", status
, reason_phrase
);
3459 connection_buf_add(buf
, strlen(buf
), TO_CONN(conn
));
3461 tor_free(datestring
);
3465 /** Write the header for an HTTP/1.0 response onto <b>conn</b>-\>outbuf,
3466 * with <b>type</b> as the Content-Type.
3468 * If <b>length</b> is nonnegative, it is the Content-Length.
3469 * If <b>encoding</b> is provided, it is the Content-Encoding.
3470 * If <b>cache_lifetime</b> is greater than 0, the content may be cached for
3471 * up to cache_lifetime seconds. Otherwise, the content may not be cached. */
3473 write_http_response_header_impl(dir_connection_t
*conn
, ssize_t length
,
3474 const char *type
, const char *encoding
,
3475 const char *extra_headers
,
3476 long cache_lifetime
)
3478 char date
[RFC1123_TIME_LEN
+1];
3481 time_t now
= time(NULL
);
3485 format_rfc1123_time(date
, now
);
3487 tor_snprintf(cp
, sizeof(tmp
),
3488 "HTTP/1.0 200 OK\r\nDate: %s\r\n",
3492 tor_snprintf(cp
, sizeof(tmp
)-(cp
-tmp
), "Content-Type: %s\r\n", type
);
3495 if (!is_local_addr(&conn
->base_
.addr
)) {
3496 /* Don't report the source address for a nearby/private connection.
3497 * Otherwise we tend to mis-report in cases where incoming ports are
3498 * being forwarded to a Tor server running behind the firewall. */
3499 tor_snprintf(cp
, sizeof(tmp
)-(cp
-tmp
),
3500 X_ADDRESS_HEADER
"%s\r\n", conn
->base_
.address
);
3504 tor_snprintf(cp
, sizeof(tmp
)-(cp
-tmp
),
3505 "Content-Encoding: %s\r\n", encoding
);
3509 tor_snprintf(cp
, sizeof(tmp
)-(cp
-tmp
),
3510 "Content-Length: %ld\r\n", (long)length
);
3513 if (cache_lifetime
> 0) {
3514 char expbuf
[RFC1123_TIME_LEN
+1];
3515 format_rfc1123_time(expbuf
, (time_t)(now
+ cache_lifetime
));
3516 /* We could say 'Cache-control: max-age=%d' here if we start doing
3518 tor_snprintf(cp
, sizeof(tmp
)-(cp
-tmp
),
3519 "Expires: %s\r\n", expbuf
);
3521 } else if (cache_lifetime
== 0) {
3522 /* We could say 'Cache-control: no-cache' here if we start doing
3524 strlcpy(cp
, "Pragma: no-cache\r\n", sizeof(tmp
)-(cp
-tmp
));
3527 if (extra_headers
) {
3528 strlcpy(cp
, extra_headers
, sizeof(tmp
)-(cp
-tmp
));
3531 if (sizeof(tmp
)-(cp
-tmp
) > 3)
3532 memcpy(cp
, "\r\n", 3);
3535 connection_buf_add(tmp
, strlen(tmp
), TO_CONN(conn
));
3538 /** As write_http_response_header_impl, but sets encoding and content-typed
3539 * based on whether the response will be <b>compressed</b> or not. */
3541 write_http_response_headers(dir_connection_t
*conn
, ssize_t length
,
3542 compress_method_t method
,
3543 const char *extra_headers
, long cache_lifetime
)
3545 const char *methodname
= compression_method_get_name(method
);
3546 const char *doctype
;
3547 if (method
== NO_METHOD
)
3548 doctype
= "text/plain";
3550 doctype
= "application/octet-stream";
3551 write_http_response_header_impl(conn
, length
,
3558 /** As write_http_response_headers, but assumes extra_headers is NULL */
3560 write_http_response_header(dir_connection_t
*conn
, ssize_t length
,
3561 compress_method_t method
,
3562 long cache_lifetime
)
3564 write_http_response_headers(conn
, length
, method
, NULL
, cache_lifetime
);
3567 /** Array of compression methods to use (if supported) for serving
3568 * precompressed data, ordered from best to worst. */
3569 static compress_method_t srv_meth_pref_precompressed
[] = {
3577 /** Array of compression methods to use (if supported) for serving
3578 * streamed data, ordered from best to worst. */
3579 static compress_method_t srv_meth_pref_streaming_compression
[] = {
3586 /** Array of allowed compression methods to use (if supported) when receiving a
3587 * response from a request that was required to be anonymous. */
3588 static compress_method_t client_meth_allowed_anonymous_compression
[] = {
3594 /** Parse the compression methods listed in an Accept-Encoding header <b>h</b>,
3595 * and convert them to a bitfield where compression method x is supported if
3596 * and only if 1 << x is set in the bitfield. */
3598 parse_accept_encoding_header(const char *h
)
3600 unsigned result
= (1u << NO_METHOD
);
3601 smartlist_t
*methods
= smartlist_new();
3602 smartlist_split_string(methods
, h
, ",",
3603 SPLIT_SKIP_SPACE
|SPLIT_STRIP_SPACE
|SPLIT_IGNORE_BLANK
, 0);
3605 SMARTLIST_FOREACH_BEGIN(methods
, const char *, m
) {
3606 compress_method_t method
= compression_method_get_by_name(m
);
3607 if (method
!= UNKNOWN_METHOD
) {
3608 tor_assert(((unsigned)method
) < 8*sizeof(unsigned));
3609 result
|= (1u << method
);
3611 } SMARTLIST_FOREACH_END(m
);
3612 SMARTLIST_FOREACH_BEGIN(methods
, char *, m
) {
3614 } SMARTLIST_FOREACH_END(m
);
3615 smartlist_free(methods
);
3619 /** Array of compression methods to use (if supported) for requesting
3620 * compressed data, ordered from best to worst. */
3621 static compress_method_t client_meth_pref
[] = {
3629 /** Return a newly allocated string containing a comma separated list of
3630 * supported encodings. */
3632 accept_encoding_header(void)
3634 smartlist_t
*methods
= smartlist_new();
3635 char *header
= NULL
;
3636 compress_method_t method
;
3639 for (i
= 0; i
< ARRAY_LENGTH(client_meth_pref
); ++i
) {
3640 method
= client_meth_pref
[i
];
3641 if (tor_compress_supports_method(method
))
3642 smartlist_add(methods
, (char *)compression_method_get_name(method
));
3645 header
= smartlist_join_strings(methods
, ", ", 0, NULL
);
3646 smartlist_free(methods
);
3651 /** Decide whether a client would accept the consensus we have.
3653 * Clients can say they only want a consensus if it's signed by more
3654 * than half the authorities in a list. They pass this list in
3655 * the url as "...consensus/<b>fpr</b>+<b>fpr</b>+<b>fpr</b>".
3657 * <b>fpr</b> may be an abbreviated fingerprint, i.e. only a left substring
3658 * of the full authority identity digest. (Only strings of even length,
3659 * i.e. encodings of full bytes, are handled correctly. In the case
3660 * of an odd number of hex digits the last one is silently ignored.)
3662 * Returns 1 if more than half of the requested authorities signed the
3663 * consensus, 0 otherwise.
3666 client_likes_consensus(const struct consensus_cache_entry_t
*ent
,
3667 const char *want_url
)
3669 smartlist_t
*voters
= smartlist_new();
3673 if (consensus_cache_entry_get_voter_id_digests(ent
, voters
) != 0) {
3674 return 1; // We don't know the voters; assume the client won't mind. */
3677 smartlist_t
*want_authorities
= smartlist_new();
3678 dir_split_resource_into_fingerprints(want_url
, want_authorities
, NULL
, 0);
3679 need_at_least
= smartlist_len(want_authorities
)/2+1;
3681 SMARTLIST_FOREACH_BEGIN(want_authorities
, const char *, want_digest
) {
3683 SMARTLIST_FOREACH_BEGIN(voters
, const char *, digest
) {
3684 if (!strcasecmpstart(digest
, want_digest
)) {
3688 } SMARTLIST_FOREACH_END(digest
);
3690 /* early exit, if we already have enough */
3691 if (have
>= need_at_least
)
3693 } SMARTLIST_FOREACH_END(want_digest
);
3695 SMARTLIST_FOREACH(want_authorities
, char *, d
, tor_free(d
));
3696 smartlist_free(want_authorities
);
3697 SMARTLIST_FOREACH(voters
, char *, cp
, tor_free(cp
));
3698 smartlist_free(voters
);
3699 return (have
>= need_at_least
);
3702 /** Return the compression level we should use for sending a compressed
3703 * response of size <b>n_bytes</b>. */
3704 STATIC compression_level_t
3705 choose_compression_level(ssize_t n_bytes
)
3707 if (! have_been_under_memory_pressure()) {
3708 return HIGH_COMPRESSION
; /* we have plenty of RAM. */
3709 } else if (n_bytes
< 0) {
3710 return HIGH_COMPRESSION
; /* unknown; might be big. */
3711 } else if (n_bytes
< 1024) {
3712 return LOW_COMPRESSION
;
3713 } else if (n_bytes
< 2048) {
3714 return MEDIUM_COMPRESSION
;
3716 return HIGH_COMPRESSION
;
3720 /** Information passed to handle a GET request. */
3721 typedef struct get_handler_args_t
{
3722 /** Bitmask of compression methods that the client said (or implied) it
3724 unsigned compression_supported
;
3725 /** If nonzero, the time included an if-modified-since header with this
3727 time_t if_modified_since
;
3728 /** String containing the requested URL or resource. */
3730 /** String containing the HTTP headers */
3731 const char *headers
;
3732 } get_handler_args_t
;
3734 /** Entry for handling an HTTP GET request.
3736 * This entry matches a request if "string" is equal to the requested
3737 * resource, or if "is_prefix" is true and "string" is a prefix of the
3738 * requested resource.
3740 * The 'handler' function is called to handle the request. It receives
3741 * an arguments structure, and must return 0 on success or -1 if we should
3742 * close the connection.
3744 typedef struct url_table_ent_s
{
3747 int (*handler
)(dir_connection_t
*conn
, const get_handler_args_t
*args
);
3750 static int handle_get_frontpage(dir_connection_t
*conn
,
3751 const get_handler_args_t
*args
);
3752 static int handle_get_current_consensus(dir_connection_t
*conn
,
3753 const get_handler_args_t
*args
);
3754 static int handle_get_status_vote(dir_connection_t
*conn
,
3755 const get_handler_args_t
*args
);
3756 static int handle_get_microdesc(dir_connection_t
*conn
,
3757 const get_handler_args_t
*args
);
3758 static int handle_get_descriptor(dir_connection_t
*conn
,
3759 const get_handler_args_t
*args
);
3760 static int handle_get_keys(dir_connection_t
*conn
,
3761 const get_handler_args_t
*args
);
3762 static int handle_get_hs_descriptor_v2(dir_connection_t
*conn
,
3763 const get_handler_args_t
*args
);
3764 static int handle_get_robots(dir_connection_t
*conn
,
3765 const get_handler_args_t
*args
);
3766 static int handle_get_networkstatus_bridges(dir_connection_t
*conn
,
3767 const get_handler_args_t
*args
);
3769 /** Table for handling GET requests. */
3770 static const url_table_ent_t url_table
[] = {
3771 { "/tor/", 0, handle_get_frontpage
},
3772 { "/tor/status-vote/current/consensus", 1, handle_get_current_consensus
},
3773 { "/tor/status-vote/current/", 1, handle_get_status_vote
},
3774 { "/tor/status-vote/next/", 1, handle_get_status_vote
},
3775 { "/tor/micro/d/", 1, handle_get_microdesc
},
3776 { "/tor/server/", 1, handle_get_descriptor
},
3777 { "/tor/extra/", 1, handle_get_descriptor
},
3778 { "/tor/keys/", 1, handle_get_keys
},
3779 { "/tor/rendezvous2/", 1, handle_get_hs_descriptor_v2
},
3780 { "/tor/hs/3/", 1, handle_get_hs_descriptor_v3
},
3781 { "/tor/robots.txt", 0, handle_get_robots
},
3782 { "/tor/networkstatus-bridges", 0, handle_get_networkstatus_bridges
},
3786 /** Helper function: called when a dirserver gets a complete HTTP GET
3787 * request. Look for a request for a directory or for a rendezvous
3788 * service descriptor. On finding one, write a response into
3789 * conn-\>outbuf. If the request is unrecognized, send a 404.
3790 * Return 0 if we handled this successfully, or -1 if we need to close
3791 * the connection. */
3792 MOCK_IMPL(STATIC
int,
3793 directory_handle_command_get
,(dir_connection_t
*conn
, const char *headers
,
3794 const char *req_body
, size_t req_body_len
))
3796 char *url
, *url_mem
, *header
;
3797 time_t if_modified_since
= 0;
3798 int zlib_compressed_in_url
;
3799 unsigned compression_methods_supported
;
3801 /* We ignore the body of a GET request. */
3805 log_debug(LD_DIRSERV
,"Received GET command.");
3807 conn
->base_
.state
= DIR_CONN_STATE_SERVER_WRITING
;
3809 if (parse_http_url(headers
, &url
) < 0) {
3810 write_short_http_response(conn
, 400, "Bad request");
3813 if ((header
= http_get_header(headers
, "If-Modified-Since: "))) {
3815 if (parse_http_time(header
, &tm
) == 0) {
3816 if (tor_timegm(&tm
, &if_modified_since
)<0) {
3817 if_modified_since
= 0;
3819 log_debug(LD_DIRSERV
, "If-Modified-Since is '%s'.", escaped(header
));
3822 /* The correct behavior on a malformed If-Modified-Since header is to
3823 * act as if no If-Modified-Since header had been given. */
3826 log_debug(LD_DIRSERV
,"rewritten url as '%s'.", escaped(url
));
3830 size_t url_len
= strlen(url
);
3832 zlib_compressed_in_url
= url_len
> 2 && !strcmp(url
+url_len
-2, ".z");
3833 if (zlib_compressed_in_url
) {
3834 url
[url_len
-2] = '\0';
3838 if ((header
= http_get_header(headers
, "Accept-Encoding: "))) {
3839 compression_methods_supported
= parse_accept_encoding_header(header
);
3842 compression_methods_supported
= (1u << NO_METHOD
);
3844 if (zlib_compressed_in_url
) {
3845 compression_methods_supported
|= (1u << ZLIB_METHOD
);
3848 /* Remove all methods that we don't both support. */
3849 compression_methods_supported
&= tor_compress_get_supported_method_bitmask();
3851 get_handler_args_t args
;
3853 args
.headers
= headers
;
3854 args
.if_modified_since
= if_modified_since
;
3855 args
.compression_supported
= compression_methods_supported
;
3858 for (i
= 0; url_table
[i
].string
; ++i
) {
3860 if (url_table
[i
].is_prefix
) {
3861 match
= !strcmpstart(url
, url_table
[i
].string
);
3863 match
= !strcmp(url
, url_table
[i
].string
);
3866 result
= url_table
[i
].handler(conn
, &args
);
3871 /* we didn't recognize the url */
3872 write_short_http_response(conn
, 404, "Not found");
3880 /** Helper function for GET / or GET /tor/
3883 handle_get_frontpage(dir_connection_t
*conn
, const get_handler_args_t
*args
)
3885 (void) args
; /* unused */
3886 const char *frontpage
= get_dirportfrontpage();
3890 dlen
= strlen(frontpage
);
3891 /* Let's return a disclaimer page (users shouldn't use V1 anymore,
3892 and caches don't fetch '/', so this is safe). */
3894 /* [We don't check for write_bucket_low here, since we want to serve
3895 * this page no matter what.] */
3896 write_http_response_header_impl(conn
, dlen
, "text/html", "identity",
3897 NULL
, DIRPORTFRONTPAGE_CACHE_LIFETIME
);
3898 connection_buf_add(frontpage
, dlen
, TO_CONN(conn
));
3900 write_short_http_response(conn
, 404, "Not found");
3905 /** Warn that the cached consensus <b>consensus</b> of type
3906 * <b>flavor</b> is too old and will not be served to clients. Rate-limit the
3907 * warning to avoid logging an entry on every request.
3910 warn_consensus_is_too_old(const struct consensus_cache_entry_t
*consensus
,
3911 const char *flavor
, time_t now
)
3913 #define TOO_OLD_WARNING_INTERVAL (60*60)
3914 static ratelim_t warned
= RATELIM_INIT(TOO_OLD_WARNING_INTERVAL
);
3915 char timestamp
[ISO_TIME_LEN
+1];
3919 if (consensus_cache_entry_get_valid_until(consensus
, &valid_until
))
3922 if ((dupes
= rate_limit_log(&warned
, now
))) {
3923 format_local_iso_time(timestamp
, valid_until
);
3924 log_warn(LD_DIRSERV
, "Our %s%sconsensus is too old, so we will not "
3925 "serve it to clients. It was valid until %s local time and we "
3926 "continued to serve it for up to 24 hours after it expired.%s",
3927 flavor
? flavor
: "", flavor
? " " : "", timestamp
, dupes
);
3933 * Parse a single hex-encoded sha3-256 digest from <b>hex</b> into
3934 * <b>digest</b>. Return 0 on success. On failure, report that the hash came
3935 * from <b>location</b>, report that we are taking <b>action</b> with it, and
3939 parse_one_diff_hash(uint8_t *digest
, const char *hex
, const char *location
,
3942 if (base16_decode((char*)digest
, DIGEST256_LEN
, hex
, strlen(hex
)) ==
3946 log_fn(LOG_PROTOCOL_WARN
, LD_DIR
,
3947 "%s contained bogus digest %s; %s.",
3948 location
, escaped(hex
), action
);
3953 /** If there is an X-Or-Diff-From-Consensus header included in <b>headers</b>,
3954 * set <b>digest_out<b> to a new smartlist containing every 256-bit
3955 * hex-encoded digest listed in that header and return 0. Otherwise return
3958 parse_or_diff_from_header(smartlist_t
**digests_out
, const char *headers
)
3960 char *hdr
= http_get_header(headers
, X_OR_DIFF_FROM_CONSENSUS_HEADER
);
3964 smartlist_t
*hex_digests
= smartlist_new();
3965 *digests_out
= smartlist_new();
3966 smartlist_split_string(hex_digests
, hdr
, " ",
3967 SPLIT_SKIP_SPACE
|SPLIT_IGNORE_BLANK
, -1);
3968 SMARTLIST_FOREACH_BEGIN(hex_digests
, const char *, hex
) {
3969 uint8_t digest
[DIGEST256_LEN
];
3970 if (!parse_one_diff_hash(digest
, hex
, "X-Or-Diff-From-Consensus header",
3972 smartlist_add(*digests_out
, tor_memdup(digest
, sizeof(digest
)));
3974 } SMARTLIST_FOREACH_END(hex
);
3975 SMARTLIST_FOREACH(hex_digests
, char *, cp
, tor_free(cp
));
3976 smartlist_free(hex_digests
);
3981 /** Fallback compression method. The fallback compression method is used in
3982 * case a client requests a non-compressed document. We only store compressed
3983 * documents, so we use this compression method to fetch the document and let
3984 * the spooling system do the streaming decompression.
3986 #define FALLBACK_COMPRESS_METHOD ZLIB_METHOD
3989 * Try to find the best consensus diff possible in order to serve a client
3990 * request for a diff from one of the consensuses in <b>digests</b> to the
3991 * current consensus of flavor <b>flav</b>. The client supports the
3992 * compression methods listed in the <b>compression_methods</b> bitfield:
3993 * place the method chosen (if any) into <b>compression_used_out</b>.
3995 static struct consensus_cache_entry_t
*
3996 find_best_diff(const smartlist_t
*digests
, int flav
,
3997 unsigned compression_methods
,
3998 compress_method_t
*compression_used_out
)
4000 struct consensus_cache_entry_t
*result
= NULL
;
4002 SMARTLIST_FOREACH_BEGIN(digests
, const uint8_t *, diff_from
) {
4004 for (u
= 0; u
< ARRAY_LENGTH(srv_meth_pref_precompressed
); ++u
) {
4005 compress_method_t method
= srv_meth_pref_precompressed
[u
];
4006 if (0 == (compression_methods
& (1u<<method
)))
4007 continue; // client doesn't like this one, or we don't have it.
4008 if (consdiffmgr_find_diff_from(&result
, flav
, DIGEST_SHA3_256
,
4009 diff_from
, DIGEST256_LEN
,
4010 method
) == CONSDIFF_AVAILABLE
) {
4011 tor_assert_nonfatal(result
);
4012 *compression_used_out
= method
;
4016 } SMARTLIST_FOREACH_END(diff_from
);
4018 SMARTLIST_FOREACH_BEGIN(digests
, const uint8_t *, diff_from
) {
4019 if (consdiffmgr_find_diff_from(&result
, flav
, DIGEST_SHA3_256
, diff_from
,
4020 DIGEST256_LEN
, FALLBACK_COMPRESS_METHOD
) == CONSDIFF_AVAILABLE
) {
4021 tor_assert_nonfatal(result
);
4022 *compression_used_out
= FALLBACK_COMPRESS_METHOD
;
4025 } SMARTLIST_FOREACH_END(diff_from
);
4030 /** Lookup the cached consensus document by the flavor found in <b>flav</b>.
4031 * The prefered set of compression methods should be listed in the
4032 * <b>compression_methods</b> bitfield. The compression method chosen (if any)
4033 * is stored in <b>compression_used_out</b>. */
4034 static struct consensus_cache_entry_t
*
4035 find_best_consensus(int flav
,
4036 unsigned compression_methods
,
4037 compress_method_t
*compression_used_out
)
4039 struct consensus_cache_entry_t
*result
= NULL
;
4042 for (u
= 0; u
< ARRAY_LENGTH(srv_meth_pref_precompressed
); ++u
) {
4043 compress_method_t method
= srv_meth_pref_precompressed
[u
];
4045 if (0 == (compression_methods
& (1u<<method
)))
4048 if (consdiffmgr_find_consensus(&result
, flav
,
4049 method
) == CONSDIFF_AVAILABLE
) {
4050 tor_assert_nonfatal(result
);
4051 *compression_used_out
= method
;
4056 if (consdiffmgr_find_consensus(&result
, flav
,
4057 FALLBACK_COMPRESS_METHOD
) == CONSDIFF_AVAILABLE
) {
4058 tor_assert_nonfatal(result
);
4059 *compression_used_out
= FALLBACK_COMPRESS_METHOD
;
4066 /** Try to find the best supported compression method possible from a given
4067 * <b>compression_methods</b>. Return NO_METHOD if no mutually supported
4068 * compression method could be found. */
4069 static compress_method_t
4070 find_best_compression_method(unsigned compression_methods
, int stream
)
4073 compress_method_t
*methods
;
4077 methods
= srv_meth_pref_streaming_compression
;
4078 length
= ARRAY_LENGTH(srv_meth_pref_streaming_compression
);
4080 methods
= srv_meth_pref_precompressed
;
4081 length
= ARRAY_LENGTH(srv_meth_pref_precompressed
);
4084 for (u
= 0; u
< length
; ++u
) {
4085 compress_method_t method
= methods
[u
];
4086 if (compression_methods
& (1u<<method
))
4093 /** Check if any of the digests in <b>digests</b> matches the latest consensus
4094 * flavor (given in <b>flavor</b>) that we have available. */
4096 digest_list_contains_best_consensus(consensus_flavor_t flavor
,
4097 const smartlist_t
*digests
)
4099 const networkstatus_t
*ns
= NULL
;
4101 if (digests
== NULL
)
4104 ns
= networkstatus_get_latest_consensus_by_flavor(flavor
);
4109 SMARTLIST_FOREACH_BEGIN(digests
, const uint8_t *, digest
) {
4110 if (tor_memeq(ns
->digest_sha3_as_signed
, digest
, DIGEST256_LEN
))
4112 } SMARTLIST_FOREACH_END(digest
);
4117 /** Check if the given compression method is allowed for a connection that is
4118 * supposed to be anonymous. Returns 1 if the compression method is allowed,
4121 allowed_anonymous_connection_compression_method(compress_method_t method
)
4125 for (u
= 0; u
< ARRAY_LENGTH(client_meth_allowed_anonymous_compression
);
4127 compress_method_t allowed_method
=
4128 client_meth_allowed_anonymous_compression
[u
];
4130 if (! tor_compress_supports_method(allowed_method
))
4133 if (method
== allowed_method
)
4140 /** Log a warning when a remote server has sent us a document using a
4141 * compression method that is not allowed for anonymous directory requests. */
4143 warn_disallowed_anonymous_compression_method(compress_method_t method
)
4145 log_fn(LOG_PROTOCOL_WARN
, LD_HTTP
,
4146 "Received a %s HTTP response, which is not "
4147 "allowed for anonymous directory requests.",
4148 compression_method_get_human_name(method
));
4151 /** Encodes the results of parsing a consensus request to figure out what
4152 * consensus, and possibly what diffs, the user asked for. */
4154 /** name of the flavor to retrieve. */
4156 /** flavor to retrive, as enum. */
4157 consensus_flavor_t flav
;
4158 /** plus-separated list of authority fingerprints; see
4159 * client_likes_consensus(). Aliases the URL in the request passed to
4160 * parse_consensus_request(). */
4161 const char *want_fps
;
4162 /** Optionally, a smartlist of sha3 digests-as-signed of the consensuses
4163 * to return a diff from. */
4164 smartlist_t
*diff_from_digests
;
4165 /** If true, never send a full consensus. If there is no diff, send
4168 } parsed_consensus_request_t
;
4170 /** Remove all data held in <b>req</b>. Do not free <b>req</b> itself, since
4171 * it is stack-allocated. */
4173 parsed_consensus_request_clear(parsed_consensus_request_t
*req
)
4177 tor_free(req
->flavor
);
4178 if (req
->diff_from_digests
) {
4179 SMARTLIST_FOREACH(req
->diff_from_digests
, uint8_t *, d
, tor_free(d
));
4180 smartlist_free(req
->diff_from_digests
);
4182 memset(req
, 0, sizeof(parsed_consensus_request_t
));
4186 * Parse the URL and relevant headers of <b>args</b> for a current-consensus
4187 * request to learn what flavor of consensus we want, what keys it must be
4188 * signed with, and what diffs we would accept (or demand) instead. Return 0
4189 * on success and -1 on failure.
4192 parse_consensus_request(parsed_consensus_request_t
*out
,
4193 const get_handler_args_t
*args
)
4195 const char *url
= args
->url
;
4196 memset(out
, 0, sizeof(parsed_consensus_request_t
));
4197 out
->flav
= FLAV_NS
;
4199 const char CONSENSUS_URL_PREFIX
[] = "/tor/status-vote/current/consensus/";
4200 const char CONSENSUS_FLAVORED_PREFIX
[] =
4201 "/tor/status-vote/current/consensus-";
4203 /* figure out the flavor if any, and who we wanted to sign the thing */
4204 const char *after_flavor
= NULL
;
4206 if (!strcmpstart(url
, CONSENSUS_FLAVORED_PREFIX
)) {
4208 f
= url
+ strlen(CONSENSUS_FLAVORED_PREFIX
);
4209 cp
= strchr(f
, '/');
4211 after_flavor
= cp
+1;
4212 out
->flavor
= tor_strndup(f
, cp
-f
);
4214 out
->flavor
= tor_strdup(f
);
4216 int flav
= networkstatus_parse_flavor_name(out
->flavor
);
4221 if (!strcmpstart(url
, CONSENSUS_URL_PREFIX
))
4222 after_flavor
= url
+strlen(CONSENSUS_URL_PREFIX
);
4225 /* see whether we've been asked explicitly for a diff from an older
4226 * consensus. (The user might also have said that a diff would be okay,
4227 * via X-Or-Diff-From-Consensus */
4228 const char DIFF_COMPONENT
[] = "diff/";
4229 char *diff_hash_in_url
= NULL
;
4230 if (after_flavor
&& !strcmpstart(after_flavor
, DIFF_COMPONENT
)) {
4231 after_flavor
+= strlen(DIFF_COMPONENT
);
4232 const char *cp
= strchr(after_flavor
, '/');
4234 diff_hash_in_url
= tor_strndup(after_flavor
, cp
-after_flavor
);
4235 out
->want_fps
= cp
+1;
4237 diff_hash_in_url
= tor_strdup(after_flavor
);
4238 out
->want_fps
= NULL
;
4241 out
->want_fps
= after_flavor
;
4244 if (diff_hash_in_url
) {
4245 uint8_t diff_from
[DIGEST256_LEN
];
4246 out
->diff_from_digests
= smartlist_new();
4248 int ok
= !parse_one_diff_hash(diff_from
, diff_hash_in_url
, "URL",
4250 tor_free(diff_hash_in_url
);
4252 smartlist_add(out
->diff_from_digests
,
4253 tor_memdup(diff_from
, DIGEST256_LEN
));
4258 parse_or_diff_from_header(&out
->diff_from_digests
, args
->headers
);
4264 /** Helper function for GET /tor/status-vote/current/consensus
4267 handle_get_current_consensus(dir_connection_t
*conn
,
4268 const get_handler_args_t
*args
)
4270 const compress_method_t compress_method
=
4271 find_best_compression_method(args
->compression_supported
, 0);
4272 const time_t if_modified_since
= args
->if_modified_since
;
4273 int clear_spool
= 0;
4275 /* v3 network status fetch. */
4276 long lifetime
= NETWORKSTATUS_CACHE_LIFETIME
;
4278 time_t now
= time(NULL
);
4279 parsed_consensus_request_t req
;
4281 if (parse_consensus_request(&req
, args
) < 0) {
4282 write_short_http_response(conn
, 404, "Couldn't parse request");
4286 if (digest_list_contains_best_consensus(req
.flav
,
4287 req
.diff_from_digests
)) {
4288 write_short_http_response(conn
, 304, "Not modified");
4289 geoip_note_ns_response(GEOIP_REJECT_NOT_MODIFIED
);
4293 struct consensus_cache_entry_t
*cached_consensus
= NULL
;
4295 compress_method_t compression_used
= NO_METHOD
;
4296 if (req
.diff_from_digests
) {
4297 cached_consensus
= find_best_diff(req
.diff_from_digests
, req
.flav
,
4298 args
->compression_supported
,
4302 if (req
.diff_only
&& !cached_consensus
) {
4303 write_short_http_response(conn
, 404, "No such diff available");
4304 // XXXX warn_consensus_is_too_old(v, req.flavor, now);
4305 geoip_note_ns_response(GEOIP_REJECT_NOT_FOUND
);
4309 if (! cached_consensus
) {
4310 cached_consensus
= find_best_consensus(req
.flav
,
4311 args
->compression_supported
,
4315 time_t fresh_until
, valid_until
;
4316 int have_fresh_until
= 0, have_valid_until
= 0;
4317 if (cached_consensus
) {
4319 !consensus_cache_entry_get_fresh_until(cached_consensus
, &fresh_until
);
4321 !consensus_cache_entry_get_valid_until(cached_consensus
, &valid_until
);
4324 if (cached_consensus
&& have_valid_until
&&
4325 !networkstatus_valid_until_is_reasonably_live(valid_until
, now
)) {
4326 write_short_http_response(conn
, 404, "Consensus is too old");
4327 warn_consensus_is_too_old(cached_consensus
, req
.flavor
, now
);
4328 geoip_note_ns_response(GEOIP_REJECT_NOT_FOUND
);
4332 if (cached_consensus
&& req
.want_fps
&&
4333 !client_likes_consensus(cached_consensus
, req
.want_fps
)) {
4334 write_short_http_response(conn
, 404, "Consensus not signed by sufficient "
4335 "number of requested authorities");
4336 geoip_note_ns_response(GEOIP_REJECT_NOT_ENOUGH_SIGS
);
4340 conn
->spool
= smartlist_new();
4343 spooled_resource_t
*spooled
;
4344 if (cached_consensus
) {
4345 spooled
= spooled_resource_new_from_cache_entry(cached_consensus
);
4346 smartlist_add(conn
->spool
, spooled
);
4350 lifetime
= (have_fresh_until
&& fresh_until
> now
) ? fresh_until
- now
: 0;
4352 size_t size_guess
= 0;
4354 dirserv_spool_remove_missing_and_guess_size(conn
, if_modified_since
,
4355 compress_method
!= NO_METHOD
,
4359 if (!smartlist_len(conn
->spool
) && !n_expired
) {
4360 write_short_http_response(conn
, 404, "Not found");
4361 geoip_note_ns_response(GEOIP_REJECT_NOT_FOUND
);
4363 } else if (!smartlist_len(conn
->spool
)) {
4364 write_short_http_response(conn
, 304, "Not modified");
4365 geoip_note_ns_response(GEOIP_REJECT_NOT_MODIFIED
);
4369 if (global_write_bucket_low(TO_CONN(conn
), size_guess
, 2)) {
4370 log_debug(LD_DIRSERV
,
4371 "Client asked for network status lists, but we've been "
4372 "writing too many bytes lately. Sending 503 Dir busy.");
4373 write_short_http_response(conn
, 503, "Directory busy, try again later");
4374 geoip_note_ns_response(GEOIP_REJECT_BUSY
);
4379 if (tor_addr_parse(&addr
, (TO_CONN(conn
))->address
) >= 0) {
4380 geoip_note_client_seen(GEOIP_CLIENT_NETWORKSTATUS
,
4383 geoip_note_ns_response(GEOIP_SUCCESS
);
4384 /* Note that a request for a network status has started, so that we
4385 * can measure the download time later on. */
4386 if (conn
->dirreq_id
)
4387 geoip_start_dirreq(conn
->dirreq_id
, size_guess
, DIRREQ_TUNNELED
);
4389 geoip_start_dirreq(TO_CONN(conn
)->global_identifier
, size_guess
,
4393 /* Use this header to tell caches that the response depends on the
4394 * X-Or-Diff-From-Consensus header (or lack thereof). */
4395 const char vary_header
[] = "Vary: X-Or-Diff-From-Consensus\r\n";
4399 // The compress_method might have been NO_METHOD, but we store the data
4400 // compressed. Decompress them using `compression_used`. See fallback code in
4401 // find_best_consensus() and find_best_diff().
4402 write_http_response_headers(conn
, -1,
4403 compress_method
== NO_METHOD
?
4404 NO_METHOD
: compression_used
,
4406 smartlist_len(conn
->spool
) == 1 ? lifetime
: 0);
4408 if (compress_method
== NO_METHOD
&& smartlist_len(conn
->spool
))
4409 conn
->compress_state
= tor_compress_new(0, compression_used
,
4412 /* Prime the connection with some data. */
4413 const int initial_flush_result
= connection_dirserv_flushed_some(conn
);
4414 tor_assert_nonfatal(initial_flush_result
== 0);
4418 parsed_consensus_request_clear(&req
);
4420 dir_conn_clear_spool(conn
);
4425 /** Helper function for GET /tor/status-vote/{current,next}/...
4428 handle_get_status_vote(dir_connection_t
*conn
, const get_handler_args_t
*args
)
4430 const char *url
= args
->url
;
4433 ssize_t body_len
= 0;
4434 ssize_t estimated_len
= 0;
4435 /* This smartlist holds strings that we can compress on the fly. */
4436 smartlist_t
*items
= smartlist_new();
4437 /* This smartlist holds cached_dir_t objects that have a precompressed
4438 * deflated version. */
4439 smartlist_t
*dir_items
= smartlist_new();
4440 int lifetime
= 60; /* XXXX?? should actually use vote intervals. */
4441 url
+= strlen("/tor/status-vote/");
4442 current
= !strcmpstart(url
, "current/");
4443 url
= strchr(url
, '/');
4446 if (!strcmp(url
, "consensus")) {
4448 tor_assert(!current
); /* we handle current consensus specially above,
4449 * since it wants to be spooled. */
4450 if ((item
= dirvote_get_pending_consensus(FLAV_NS
)))
4451 smartlist_add(items
, (char*)item
);
4452 } else if (!current
&& !strcmp(url
, "consensus-signatures")) {
4453 /* XXXX the spec says that we should implement
4454 * current/consensus-signatures too. It doesn't seem to be needed,
4457 if ((item
=dirvote_get_pending_detached_signatures()))
4458 smartlist_add(items
, (char*)item
);
4459 } else if (!strcmp(url
, "authority")) {
4460 const cached_dir_t
*d
;
4461 int flags
= DGV_BY_ID
|
4462 (current
? DGV_INCLUDE_PREVIOUS
: DGV_INCLUDE_PENDING
);
4463 if ((d
=dirvote_get_vote(NULL
, flags
)))
4464 smartlist_add(dir_items
, (cached_dir_t
*)d
);
4466 const cached_dir_t
*d
;
4467 smartlist_t
*fps
= smartlist_new();
4469 if (!strcmpstart(url
, "d/")) {
4471 flags
= DGV_INCLUDE_PENDING
| DGV_INCLUDE_PREVIOUS
;
4474 (current
? DGV_INCLUDE_PREVIOUS
: DGV_INCLUDE_PENDING
);
4476 dir_split_resource_into_fingerprints(url
, fps
, NULL
,
4477 DSR_HEX
|DSR_SORT_UNIQ
);
4478 SMARTLIST_FOREACH(fps
, char *, fp
, {
4479 if ((d
= dirvote_get_vote(fp
, flags
)))
4480 smartlist_add(dir_items
, (cached_dir_t
*)d
);
4483 smartlist_free(fps
);
4485 if (!smartlist_len(dir_items
) && !smartlist_len(items
)) {
4486 write_short_http_response(conn
, 404, "Not found");
4490 /* We're sending items from at most one kind of source */
4491 tor_assert_nonfatal(smartlist_len(items
) == 0 ||
4492 smartlist_len(dir_items
) == 0);
4496 if (smartlist_len(items
)) {
4497 /* We're taking strings and compressing them on the fly. */
4501 /* We're taking cached_dir_t objects. We only have them uncompressed
4504 mask
= (1u<<NO_METHOD
) | (1u<<ZLIB_METHOD
);
4506 const compress_method_t compress_method
= find_best_compression_method(
4507 args
->compression_supported
&mask
, streaming
);
4509 SMARTLIST_FOREACH(dir_items
, cached_dir_t
*, d
,
4510 body_len
+= compress_method
!= NO_METHOD
?
4511 d
->dir_compressed_len
: d
->dir_len
);
4512 estimated_len
+= body_len
;
4513 SMARTLIST_FOREACH(items
, const char *, item
, {
4514 size_t ln
= strlen(item
);
4515 if (compress_method
!= NO_METHOD
) {
4516 estimated_len
+= ln
/2;
4518 body_len
+= ln
; estimated_len
+= ln
;
4522 if (global_write_bucket_low(TO_CONN(conn
), estimated_len
, 2)) {
4523 write_short_http_response(conn
, 503, "Directory busy, try again later");
4526 write_http_response_header(conn
, body_len
? body_len
: -1,
4530 if (smartlist_len(items
)) {
4531 if (compress_method
!= NO_METHOD
) {
4532 conn
->compress_state
= tor_compress_new(1, compress_method
,
4533 choose_compression_level(estimated_len
));
4534 SMARTLIST_FOREACH(items
, const char *, c
,
4535 connection_buf_add_compress(c
, strlen(c
), conn
, 0));
4536 connection_buf_add_compress("", 0, conn
, 1);
4538 SMARTLIST_FOREACH(items
, const char *, c
,
4539 connection_buf_add(c
, strlen(c
), TO_CONN(conn
)));
4542 SMARTLIST_FOREACH(dir_items
, cached_dir_t
*, d
,
4543 connection_buf_add(compress_method
!= NO_METHOD
?
4544 d
->dir_compressed
: d
->dir
,
4545 compress_method
!= NO_METHOD
?
4546 d
->dir_compressed_len
: d
->dir_len
,
4550 smartlist_free(items
);
4551 smartlist_free(dir_items
);
4558 /** Helper function for GET /tor/micro/d/...
4561 handle_get_microdesc(dir_connection_t
*conn
, const get_handler_args_t
*args
)
4563 const char *url
= args
->url
;
4564 const compress_method_t compress_method
=
4565 find_best_compression_method(args
->compression_supported
, 1);
4566 int clear_spool
= 1;
4568 conn
->spool
= smartlist_new();
4570 dir_split_resource_into_spoolable(url
+strlen("/tor/micro/d/"),
4571 DIR_SPOOL_MICRODESC
,
4573 DSR_DIGEST256
|DSR_BASE64
|DSR_SORT_UNIQ
);
4575 size_t size_guess
= 0;
4576 dirserv_spool_remove_missing_and_guess_size(conn
, 0,
4577 compress_method
!= NO_METHOD
,
4579 if (smartlist_len(conn
->spool
) == 0) {
4580 write_short_http_response(conn
, 404, "Not found");
4583 if (global_write_bucket_low(TO_CONN(conn
), size_guess
, 2)) {
4584 log_info(LD_DIRSERV
,
4585 "Client asked for server descriptors, but we've been "
4586 "writing too many bytes lately. Sending 503 Dir busy.");
4587 write_short_http_response(conn
, 503, "Directory busy, try again later");
4592 write_http_response_header(conn
, -1,
4594 MICRODESC_CACHE_LIFETIME
);
4596 if (compress_method
!= NO_METHOD
)
4597 conn
->compress_state
= tor_compress_new(1, compress_method
,
4598 choose_compression_level(size_guess
));
4600 const int initial_flush_result
= connection_dirserv_flushed_some(conn
);
4601 tor_assert_nonfatal(initial_flush_result
== 0);
4607 dir_conn_clear_spool(conn
);
4612 /** Helper function for GET /tor/{server,extra}/...
4615 handle_get_descriptor(dir_connection_t
*conn
, const get_handler_args_t
*args
)
4617 const char *url
= args
->url
;
4618 const compress_method_t compress_method
=
4619 find_best_compression_method(args
->compression_supported
, 1);
4620 const or_options_t
*options
= get_options();
4621 int clear_spool
= 1;
4622 if (!strcmpstart(url
,"/tor/server/") ||
4623 (!options
->BridgeAuthoritativeDir
&&
4624 !options
->BridgeRelay
&& !strcmpstart(url
,"/tor/extra/"))) {
4626 const char *msg
= NULL
;
4627 int cache_lifetime
= 0;
4628 int is_extra
= !strcmpstart(url
,"/tor/extra/");
4629 url
+= is_extra
? strlen("/tor/extra/") : strlen("/tor/server/");
4630 dir_spool_source_t source
;
4631 time_t publish_cutoff
= 0;
4632 if (!strcmpstart(url
, "d/")) {
4634 is_extra
? DIR_SPOOL_EXTRA_BY_DIGEST
: DIR_SPOOL_SERVER_BY_DIGEST
;
4637 is_extra
? DIR_SPOOL_EXTRA_BY_FP
: DIR_SPOOL_SERVER_BY_FP
;
4638 /* We only want to apply a publish cutoff when we're requesting
4639 * resources by fingerprint. */
4640 publish_cutoff
= time(NULL
) - ROUTER_MAX_AGE_TO_PUBLISH
;
4643 conn
->spool
= smartlist_new();
4644 res
= dirserv_get_routerdesc_spool(conn
->spool
, url
,
4646 connection_dir_is_encrypted(conn
),
4649 if (!strcmpstart(url
, "all")) {
4650 cache_lifetime
= FULL_DIR_CACHE_LIFETIME
;
4651 } else if (smartlist_len(conn
->spool
) == 1) {
4652 cache_lifetime
= ROUTERDESC_BY_DIGEST_CACHE_LIFETIME
;
4655 size_t size_guess
= 0;
4657 dirserv_spool_remove_missing_and_guess_size(conn
, publish_cutoff
,
4658 compress_method
!= NO_METHOD
,
4659 &size_guess
, &n_expired
);
4661 /* If we are the bridge authority and the descriptor is a bridge
4662 * descriptor, remember that we served this descriptor for desc stats. */
4663 /* XXXX it's a bit of a kludge to have this here. */
4664 if (get_options()->BridgeAuthoritativeDir
&&
4665 source
== DIR_SPOOL_SERVER_BY_FP
) {
4666 SMARTLIST_FOREACH_BEGIN(conn
->spool
, spooled_resource_t
*, spooled
) {
4667 const routerinfo_t
*router
=
4668 router_get_by_id_digest((const char *)spooled
->digest
);
4669 /* router can be NULL here when the bridge auth is asked for its own
4671 if (router
&& router
->purpose
== ROUTER_PURPOSE_BRIDGE
)
4672 rep_hist_note_desc_served(router
->cache_info
.identity_digest
);
4673 } SMARTLIST_FOREACH_END(spooled
);
4676 if (res
< 0 || size_guess
== 0 || smartlist_len(conn
->spool
) == 0) {
4679 write_short_http_response(conn
, 404, msg
);
4681 if (global_write_bucket_low(TO_CONN(conn
), size_guess
, 2)) {
4682 log_info(LD_DIRSERV
,
4683 "Client asked for server descriptors, but we've been "
4684 "writing too many bytes lately. Sending 503 Dir busy.");
4685 write_short_http_response(conn
, 503,
4686 "Directory busy, try again later");
4687 dir_conn_clear_spool(conn
);
4690 write_http_response_header(conn
, -1, compress_method
, cache_lifetime
);
4691 if (compress_method
!= NO_METHOD
)
4692 conn
->compress_state
= tor_compress_new(1, compress_method
,
4693 choose_compression_level(size_guess
));
4695 /* Prime the connection with some data. */
4696 int initial_flush_result
= connection_dirserv_flushed_some(conn
);
4697 tor_assert_nonfatal(initial_flush_result
== 0);
4703 dir_conn_clear_spool(conn
);
4707 /** Helper function for GET /tor/keys/...
4710 handle_get_keys(dir_connection_t
*conn
, const get_handler_args_t
*args
)
4712 const char *url
= args
->url
;
4713 const compress_method_t compress_method
=
4714 find_best_compression_method(args
->compression_supported
, 1);
4715 const time_t if_modified_since
= args
->if_modified_since
;
4717 smartlist_t
*certs
= smartlist_new();
4719 if (!strcmp(url
, "/tor/keys/all")) {
4720 authority_cert_get_all(certs
);
4721 } else if (!strcmp(url
, "/tor/keys/authority")) {
4722 authority_cert_t
*cert
= get_my_v3_authority_cert();
4724 smartlist_add(certs
, cert
);
4725 } else if (!strcmpstart(url
, "/tor/keys/fp/")) {
4726 smartlist_t
*fps
= smartlist_new();
4727 dir_split_resource_into_fingerprints(url
+strlen("/tor/keys/fp/"),
4729 DSR_HEX
|DSR_SORT_UNIQ
);
4730 SMARTLIST_FOREACH(fps
, char *, d
, {
4731 authority_cert_t
*c
= authority_cert_get_newest_by_id(d
);
4732 if (c
) smartlist_add(certs
, c
);
4735 smartlist_free(fps
);
4736 } else if (!strcmpstart(url
, "/tor/keys/sk/")) {
4737 smartlist_t
*fps
= smartlist_new();
4738 dir_split_resource_into_fingerprints(url
+strlen("/tor/keys/sk/"),
4740 DSR_HEX
|DSR_SORT_UNIQ
);
4741 SMARTLIST_FOREACH(fps
, char *, d
, {
4742 authority_cert_t
*c
= authority_cert_get_by_sk_digest(d
);
4743 if (c
) smartlist_add(certs
, c
);
4746 smartlist_free(fps
);
4747 } else if (!strcmpstart(url
, "/tor/keys/fp-sk/")) {
4748 smartlist_t
*fp_sks
= smartlist_new();
4749 dir_split_resource_into_fingerprint_pairs(url
+strlen("/tor/keys/fp-sk/"),
4751 SMARTLIST_FOREACH(fp_sks
, fp_pair_t
*, pair
, {
4752 authority_cert_t
*c
= authority_cert_get_by_digests(pair
->first
,
4754 if (c
) smartlist_add(certs
, c
);
4757 smartlist_free(fp_sks
);
4759 write_short_http_response(conn
, 400, "Bad request");
4762 if (!smartlist_len(certs
)) {
4763 write_short_http_response(conn
, 404, "Not found");
4766 SMARTLIST_FOREACH(certs
, authority_cert_t
*, c
,
4767 if (c
->cache_info
.published_on
< if_modified_since
)
4768 SMARTLIST_DEL_CURRENT(certs
, c
));
4769 if (!smartlist_len(certs
)) {
4770 write_short_http_response(conn
, 304, "Not modified");
4774 SMARTLIST_FOREACH(certs
, authority_cert_t
*, c
,
4775 len
+= c
->cache_info
.signed_descriptor_len
);
4777 if (global_write_bucket_low(TO_CONN(conn
),
4778 compress_method
!= NO_METHOD
? len
/2 : len
,
4780 write_short_http_response(conn
, 503, "Directory busy, try again later");
4784 write_http_response_header(conn
,
4785 compress_method
!= NO_METHOD
? -1 : len
,
4788 if (compress_method
!= NO_METHOD
) {
4789 conn
->compress_state
= tor_compress_new(1, compress_method
,
4790 choose_compression_level(len
));
4791 SMARTLIST_FOREACH(certs
, authority_cert_t
*, c
,
4792 connection_buf_add_compress(
4793 c
->cache_info
.signed_descriptor_body
,
4794 c
->cache_info
.signed_descriptor_len
,
4796 connection_buf_add_compress("", 0, conn
, 1);
4798 SMARTLIST_FOREACH(certs
, authority_cert_t
*, c
,
4799 connection_buf_add(c
->cache_info
.signed_descriptor_body
,
4800 c
->cache_info
.signed_descriptor_len
,
4804 smartlist_free(certs
);
4811 /** Helper function for GET /tor/rendezvous2/
4814 handle_get_hs_descriptor_v2(dir_connection_t
*conn
,
4815 const get_handler_args_t
*args
)
4817 const char *url
= args
->url
;
4818 if (connection_dir_is_encrypted(conn
)) {
4819 /* Handle v2 rendezvous descriptor fetch request. */
4821 const char *query
= url
+ strlen("/tor/rendezvous2/");
4822 if (rend_valid_descriptor_id(query
)) {
4823 log_info(LD_REND
, "Got a v2 rendezvous descriptor request for ID '%s'",
4824 safe_str(escaped(query
)));
4825 switch (rend_cache_lookup_v2_desc_as_dir(query
, &descp
)) {
4827 write_http_response_header(conn
, strlen(descp
), NO_METHOD
, 0);
4828 connection_buf_add(descp
, strlen(descp
), TO_CONN(conn
));
4830 case 0: /* well-formed but not present */
4831 write_short_http_response(conn
, 404, "Not found");
4833 case -1: /* not well-formed */
4834 write_short_http_response(conn
, 400, "Bad request");
4837 } else { /* not well-formed */
4838 write_short_http_response(conn
, 400, "Bad request");
4842 /* Not encrypted! */
4843 write_short_http_response(conn
, 404, "Not found");
4849 /** Helper function for GET /tor/hs/3/<z>. Only for version 3.
4852 handle_get_hs_descriptor_v3(dir_connection_t
*conn
,
4853 const get_handler_args_t
*args
)
4856 const char *desc_str
= NULL
;
4857 const char *pubkey_str
= NULL
;
4858 const char *url
= args
->url
;
4860 /* Reject unencrypted dir connections */
4861 if (!connection_dir_is_encrypted(conn
)) {
4862 write_short_http_response(conn
, 404, "Not found");
4866 /* After the path prefix follows the base64 encoded blinded pubkey which we
4867 * use to get the descriptor from the cache. Skip the prefix and get the
4869 tor_assert(!strcmpstart(url
, "/tor/hs/3/"));
4870 pubkey_str
= url
+ strlen("/tor/hs/3/");
4871 retval
= hs_cache_lookup_as_dir(HS_VERSION_THREE
,
4872 pubkey_str
, &desc_str
);
4873 if (retval
<= 0 || desc_str
== NULL
) {
4874 write_short_http_response(conn
, 404, "Not found");
4878 /* Found requested descriptor! Pass it to this nice client. */
4879 write_http_response_header(conn
, strlen(desc_str
), NO_METHOD
, 0);
4880 connection_buf_add(desc_str
, strlen(desc_str
), TO_CONN(conn
));
4886 /** Helper function for GET /tor/networkstatus-bridges
4889 handle_get_networkstatus_bridges(dir_connection_t
*conn
,
4890 const get_handler_args_t
*args
)
4892 const char *headers
= args
->headers
;
4894 const or_options_t
*options
= get_options();
4895 if (options
->BridgeAuthoritativeDir
&&
4896 options
->BridgePassword_AuthDigest_
&&
4897 connection_dir_is_encrypted(conn
)) {
4899 char digest
[DIGEST256_LEN
];
4901 char *header
= http_get_header(headers
, "Authorization: Basic ");
4903 crypto_digest256(digest
, header
, strlen(header
), DIGEST_SHA256
);
4905 /* now make sure the password is there and right */
4908 options
->BridgePassword_AuthDigest_
, DIGEST256_LEN
)) {
4909 write_short_http_response(conn
, 404, "Not found");
4915 /* all happy now. send an answer. */
4916 status
= networkstatus_getinfo_by_purpose("bridge", time(NULL
));
4917 size_t dlen
= strlen(status
);
4918 write_http_response_header(conn
, dlen
, NO_METHOD
, 0);
4919 connection_buf_add(status
, dlen
, TO_CONN(conn
));
4927 /** Helper function for GET robots.txt or /tor/robots.txt */
4929 handle_get_robots(dir_connection_t
*conn
, const get_handler_args_t
*args
)
4933 const char robots
[] = "User-agent: *\r\nDisallow: /\r\n";
4934 size_t len
= strlen(robots
);
4935 write_http_response_header(conn
, len
, NO_METHOD
, ROBOTS_CACHE_LIFETIME
);
4936 connection_buf_add(robots
, len
, TO_CONN(conn
));
4941 /* Given the <b>url</b> from a POST request, try to extract the version number
4942 * using the provided <b>prefix</b>. The version should be after the prefix and
4943 * ending with the seperator "/". For instance:
4946 * On success, <b>end_pos</b> points to the position right after the version
4947 * was found. On error, it is set to NULL.
4949 * Return version on success else negative value. */
4951 parse_hs_version_from_post(const char *url
, const char *prefix
,
4952 const char **end_pos
)
4955 unsigned long version
;
4961 tor_assert(end_pos
);
4963 /* Check if the prefix does start the url. */
4964 if (strcmpstart(url
, prefix
)) {
4967 /* Move pointer to the end of the prefix string. */
4968 start
= url
+ strlen(prefix
);
4969 /* Try this to be the HS version and if we are still at the separator, next
4970 * will be move to the right value. */
4971 version
= tor_parse_long(start
, 10, 0, INT_MAX
, &ok
, &end
);
4977 return (int) version
;
4983 /* Handle the POST request for a hidden service descripror. The request is in
4984 * <b>url</b>, the body of the request is in <b>body</b>. Return 200 on success
4985 * else return 400 indicating a bad request. */
4987 handle_post_hs_descriptor(const char *url
, const char *body
)
4990 const char *end_pos
;
4995 version
= parse_hs_version_from_post(url
, "/tor/hs/", &end_pos
);
5000 /* We have a valid version number, now make sure it's a publish request. Use
5001 * the end position just after the version and check for the command. */
5002 if (strcmpstart(end_pos
, "/publish")) {
5007 case HS_VERSION_THREE
:
5008 if (hs_cache_store_as_dir(body
) < 0) {
5011 log_info(LD_REND
, "Publish request for HS descriptor handled "
5015 /* Unsupported version, return a bad request. */
5025 /** Helper function: called when a dirserver gets a complete HTTP POST
5026 * request. Look for an uploaded server descriptor or rendezvous
5027 * service descriptor. On finding one, process it and write a
5028 * response into conn-\>outbuf. If the request is unrecognized, send a
5029 * 400. Always return 0. */
5030 MOCK_IMPL(STATIC
int,
5031 directory_handle_command_post
,(dir_connection_t
*conn
, const char *headers
,
5032 const char *body
, size_t body_len
))
5035 const or_options_t
*options
= get_options();
5037 log_debug(LD_DIRSERV
,"Received POST command.");
5039 conn
->base_
.state
= DIR_CONN_STATE_SERVER_WRITING
;
5041 if (!public_server_mode(options
)) {
5042 log_info(LD_DIR
, "Rejected dir post request from %s "
5043 "since we're not a public relay.", conn
->base_
.address
);
5044 write_short_http_response(conn
, 503, "Not acting as a public relay");
5048 if (parse_http_url(headers
, &url
) < 0) {
5049 write_short_http_response(conn
, 400, "Bad request");
5052 log_debug(LD_DIRSERV
,"rewritten url as '%s'.", escaped(url
));
5054 /* Handle v2 rendezvous service publish request. */
5055 if (connection_dir_is_encrypted(conn
) &&
5056 !strcmpstart(url
,"/tor/rendezvous2/publish")) {
5057 if (rend_cache_store_v2_desc_as_dir(body
) < 0) {
5058 log_warn(LD_REND
, "Rejected v2 rend descriptor (body size %d) from %s.",
5059 (int)body_len
, conn
->base_
.address
);
5060 write_short_http_response(conn
, 400,
5061 "Invalid v2 service descriptor rejected");
5063 write_short_http_response(conn
, 200, "Service descriptor (v2) stored");
5064 log_info(LD_REND
, "Handled v2 rendezvous descriptor post: accepted");
5069 /* Handle HS descriptor publish request. */
5070 /* XXX: This should be disabled with a consensus param until we want to
5071 * the prop224 be deployed and thus use. */
5072 if (connection_dir_is_encrypted(conn
) && !strcmpstart(url
, "/tor/hs/")) {
5073 const char *msg
= "HS descriptor stored successfully.";
5075 /* We most probably have a publish request for an HS descriptor. */
5076 int code
= handle_post_hs_descriptor(url
, body
);
5078 msg
= "Invalid HS descriptor. Rejected.";
5080 write_short_http_response(conn
, code
, msg
);
5084 if (!authdir_mode(options
)) {
5085 /* we just provide cached directories; we don't want to
5086 * receive anything. */
5087 write_short_http_response(conn
, 400, "Nonauthoritative directory does not "
5088 "accept posted server descriptors");
5092 if (authdir_mode(options
) &&
5093 !strcmp(url
,"/tor/")) { /* server descriptor post */
5094 const char *msg
= "[None]";
5095 uint8_t purpose
= authdir_mode_bridge(options
) ?
5096 ROUTER_PURPOSE_BRIDGE
: ROUTER_PURPOSE_GENERAL
;
5097 was_router_added_t r
= dirserv_add_multiple_descriptors(body
, purpose
,
5098 conn
->base_
.address
, &msg
);
5101 if (r
== ROUTER_ADDED_SUCCESSFULLY
) {
5102 write_short_http_response(conn
, 200, msg
);
5103 } else if (WRA_WAS_OUTDATED(r
)) {
5104 write_http_response_header_impl(conn
, -1, NULL
, NULL
,
5105 "X-Descriptor-Not-New: Yes\r\n", -1);
5107 log_info(LD_DIRSERV
,
5108 "Rejected router descriptor or extra-info from %s "
5110 conn
->base_
.address
, msg
);
5111 write_short_http_response(conn
, 400, msg
);
5116 if (authdir_mode_v3(options
) &&
5117 !strcmp(url
,"/tor/post/vote")) { /* v3 networkstatus vote */
5118 const char *msg
= "OK";
5120 if (dirvote_add_vote(body
, &msg
, &status
)) {
5121 write_short_http_response(conn
, status
, "Vote stored");
5124 log_warn(LD_DIRSERV
, "Rejected vote from %s (\"%s\").",
5125 conn
->base_
.address
, msg
);
5126 write_short_http_response(conn
, status
, msg
);
5131 if (authdir_mode_v3(options
) &&
5132 !strcmp(url
,"/tor/post/consensus-signature")) { /* sigs on consensus. */
5133 const char *msg
= NULL
;
5134 if (dirvote_add_signatures(body
, conn
->base_
.address
, &msg
)>=0) {
5135 write_short_http_response(conn
, 200, msg
?msg
:"Signatures stored");
5137 log_warn(LD_DIR
, "Unable to store signatures posted by %s: %s",
5138 conn
->base_
.address
, msg
?msg
:"???");
5139 write_short_http_response(conn
, 400,
5140 msg
?msg
:"Unable to store signatures");
5145 /* we didn't recognize the url */
5146 write_short_http_response(conn
, 404, "Not found");
5153 /** Called when a dirserver receives data on a directory connection;
5154 * looks for an HTTP request. If the request is complete, remove it
5155 * from the inbuf, try to process it; otherwise, leave it on the
5156 * buffer. Return a 0 on success, or -1 on error.
5159 directory_handle_command(dir_connection_t
*conn
)
5161 char *headers
=NULL
, *body
=NULL
;
5166 tor_assert(conn
->base_
.type
== CONN_TYPE_DIR
);
5168 switch (connection_fetch_from_buf_http(TO_CONN(conn
),
5169 &headers
, MAX_HEADERS_SIZE
,
5170 &body
, &body_len
, MAX_DIR_UL_SIZE
, 0)) {
5171 case -1: /* overflow */
5172 log_warn(LD_DIRSERV
,
5173 "Request too large from address '%s' to DirPort. Closing.",
5174 safe_str(conn
->base_
.address
));
5177 log_debug(LD_DIRSERV
,"command not all here yet.");
5179 /* case 1, fall through */
5182 http_set_address_origin(headers
, TO_CONN(conn
));
5183 // we should escape headers here as well,
5184 // but we can't call escaped() twice, as it uses the same buffer
5185 //log_debug(LD_DIRSERV,"headers %s, body %s.", headers, escaped(body));
5187 if (!strncasecmp(headers
,"GET",3))
5188 r
= directory_handle_command_get(conn
, headers
, body
, body_len
);
5189 else if (!strncasecmp(headers
,"POST",4))
5190 r
= directory_handle_command_post(conn
, headers
, body
, body_len
);
5192 log_fn(LOG_PROTOCOL_WARN
, LD_PROTOCOL
,
5193 "Got headers %s with unknown command. Closing.",
5198 tor_free(headers
); tor_free(body
);
5202 /** Write handler for directory connections; called when all data has
5203 * been flushed. Close the connection or wait for a response as
5207 connection_dir_finished_flushing(dir_connection_t
*conn
)
5210 tor_assert(conn
->base_
.type
== CONN_TYPE_DIR
);
5212 /* Note that we have finished writing the directory response. For direct
5213 * connections this means we're done; for tunneled connections it's only
5214 * an intermediate step. */
5215 if (conn
->dirreq_id
)
5216 geoip_change_dirreq_state(conn
->dirreq_id
, DIRREQ_TUNNELED
,
5217 DIRREQ_FLUSHING_DIR_CONN_FINISHED
);
5219 geoip_change_dirreq_state(TO_CONN(conn
)->global_identifier
,
5221 DIRREQ_FLUSHING_DIR_CONN_FINISHED
);
5222 switch (conn
->base_
.state
) {
5223 case DIR_CONN_STATE_CONNECTING
:
5224 case DIR_CONN_STATE_CLIENT_SENDING
:
5225 log_debug(LD_DIR
,"client finished sending command.");
5226 conn
->base_
.state
= DIR_CONN_STATE_CLIENT_READING
;
5228 case DIR_CONN_STATE_SERVER_WRITING
:
5230 log_warn(LD_BUG
, "Emptied a dirserv buffer, but it's still spooling!");
5231 connection_mark_for_close(TO_CONN(conn
));
5233 log_debug(LD_DIRSERV
, "Finished writing server response. Closing.");
5234 connection_mark_for_close(TO_CONN(conn
));
5238 log_warn(LD_BUG
,"called in unexpected state %d.",
5240 tor_fragile_assert();
5246 /* We just got a new consensus! If there are other in-progress requests
5247 * for this consensus flavor (for example because we launched several in
5248 * parallel), cancel them.
5250 * We do this check here (not just in
5251 * connection_ap_handshake_attach_circuit()) to handle the edge case where
5252 * a consensus fetch begins and ends before some other one tries to attach to
5253 * a circuit, in which case the other one won't know that we're all happy now.
5255 * Don't mark the conn that just gave us the consensus -- otherwise we
5256 * would end up double-marking it when it cleans itself up.
5259 connection_dir_close_consensus_fetches(dir_connection_t
*except_this_one
,
5260 const char *resource
)
5262 smartlist_t
*conns_to_close
=
5263 connection_dir_list_by_purpose_and_resource(DIR_PURPOSE_FETCH_CONSENSUS
,
5265 SMARTLIST_FOREACH_BEGIN(conns_to_close
, dir_connection_t
*, d
) {
5266 if (d
== except_this_one
)
5268 log_info(LD_DIR
, "Closing consensus fetch (to %s) since one "
5269 "has just arrived.", TO_CONN(d
)->address
);
5270 connection_mark_for_close(TO_CONN(d
));
5271 } SMARTLIST_FOREACH_END(d
);
5272 smartlist_free(conns_to_close
);
5275 /** Connected handler for directory connections: begin sending data to the
5276 * server, and return 0.
5277 * Only used when connections don't immediately connect. */
5279 connection_dir_finished_connecting(dir_connection_t
*conn
)
5282 tor_assert(conn
->base_
.type
== CONN_TYPE_DIR
);
5283 tor_assert(conn
->base_
.state
== DIR_CONN_STATE_CONNECTING
);
5285 log_debug(LD_HTTP
,"Dir connection to router %s:%u established.",
5286 conn
->base_
.address
,conn
->base_
.port
);
5288 /* start flushing conn */
5289 conn
->base_
.state
= DIR_CONN_STATE_CLIENT_SENDING
;
5293 /** Decide which download schedule we want to use based on descriptor type
5294 * in <b>dls</b> and <b>options</b>.
5295 * Then return a list of int pointers defining download delays in seconds.
5296 * Helper function for download_status_increment_failure(),
5297 * download_status_reset(), and download_status_increment_attempt(). */
5298 STATIC
const smartlist_t
*
5299 find_dl_schedule(const download_status_t
*dls
, const or_options_t
*options
)
5301 switch (dls
->schedule
) {
5302 case DL_SCHED_GENERIC
:
5303 /* Any other directory document */
5304 if (dir_server_mode(options
)) {
5305 /* A directory authority or directory mirror */
5306 return options
->TestingServerDownloadSchedule
;
5308 return options
->TestingClientDownloadSchedule
;
5310 case DL_SCHED_CONSENSUS
:
5311 if (!networkstatus_consensus_can_use_multiple_directories(options
)) {
5312 /* A public relay */
5313 return options
->TestingServerConsensusDownloadSchedule
;
5315 /* A client or bridge */
5316 if (networkstatus_consensus_is_bootstrapping(time(NULL
))) {
5317 /* During bootstrapping */
5318 if (!networkstatus_consensus_can_use_extra_fallbacks(options
)) {
5319 /* A bootstrapping client without extra fallback directories */
5321 options
->ClientBootstrapConsensusAuthorityOnlyDownloadSchedule
;
5322 } else if (dls
->want_authority
) {
5323 /* A bootstrapping client with extra fallback directories, but
5324 * connecting to an authority */
5326 options
->ClientBootstrapConsensusAuthorityDownloadSchedule
;
5328 /* A bootstrapping client connecting to extra fallback directories
5331 options
->ClientBootstrapConsensusFallbackDownloadSchedule
;
5334 /* A client with a reasonably live consensus, with or without
5336 return options
->TestingClientConsensusDownloadSchedule
;
5339 case DL_SCHED_BRIDGE
:
5340 /* A bridge client downloading bridge descriptors */
5341 if (options
->UseBridges
&& num_bridges_usable() > 0) {
5342 /* A bridge client with one or more running bridges */
5343 return options
->TestingBridgeDownloadSchedule
;
5345 /* A bridge client with no running bridges */
5346 return options
->TestingBridgeBootstrapDownloadSchedule
;
5352 /* Impossible, but gcc will fail with -Werror without a `return`. */
5356 /** Decide which minimum and maximum delay step we want to use based on
5357 * descriptor type in <b>dls</b> and <b>options</b>.
5358 * Helper function for download_status_schedule_get_delay(). */
5360 find_dl_min_and_max_delay(download_status_t
*dls
, const or_options_t
*options
,
5364 tor_assert(options
);
5369 * For now, just use the existing schedule config stuff and pick the
5370 * first/last entries off to get min/max delay for backoff purposes
5372 const smartlist_t
*schedule
= find_dl_schedule(dls
, options
);
5373 tor_assert(schedule
!= NULL
&& smartlist_len(schedule
) >= 2);
5374 *min
= *((int *)(smartlist_get(schedule
, 0)));
5375 /* Increment on failure schedules always use exponential backoff, but they
5376 * have a smaller limit when they're deterministic */
5377 if (dls
->backoff
== DL_SCHED_DETERMINISTIC
)
5378 *max
= *((int *)((smartlist_get(schedule
, smartlist_len(schedule
) - 1))));
5383 /** As next_random_exponential_delay() below, but does not compute a random
5384 * value. Instead, compute the range of values that
5385 * next_random_exponential_delay() should use when computing its random value.
5386 * Store the low bound into *<b>low_bound_out</b>, and the high bound into
5387 * *<b>high_bound_out</b>. Guarantees that the low bound is strictly less
5388 * than the high bound. */
5390 next_random_exponential_delay_range(int *low_bound_out
,
5391 int *high_bound_out
,
5395 // This is the "decorrelated jitter" approach, from
5396 // https://www.awsarchitectureblog.com/2015/03/backoff.html
5398 // sleep = min(cap, random_between(base, sleep * 3))
5400 const int delay_times_3
= delay
< INT_MAX
/3 ? delay
* 3 : INT_MAX
;
5401 *low_bound_out
= base_delay
;
5402 if (delay_times_3
> base_delay
) {
5403 *high_bound_out
= delay_times_3
;
5405 *high_bound_out
= base_delay
+1;
5409 /** Advance one delay step. The algorithm will generate a random delay,
5410 * such that each failure is possibly (random) longer than the ones before.
5412 * We then clamp that value to be no larger than max_delay, and return it.
5414 * The <b>base_delay</b> parameter is lowest possible delay time (can't be
5415 * zero); the <b>backoff_position</b> parameter is the number of times we've
5416 * generated a delay; and the <b>delay</b> argument is the most recently used
5419 * Requires that delay is less than INT_MAX, and delay is in [0,max_delay].
5422 next_random_exponential_delay(int delay
,
5426 /* Check preconditions */
5427 if (BUG(max_delay
< 0))
5429 if (BUG(delay
> max_delay
))
5437 int low_bound
=0, high_bound
=max_delay
;
5439 next_random_exponential_delay_range(&low_bound
, &high_bound
,
5442 int rand_delay
= crypto_rand_int_range(low_bound
, high_bound
);
5444 return MIN(rand_delay
, max_delay
);
5447 /** Find the current delay for dls based on schedule or min_delay/
5448 * max_delay if we're using exponential backoff. If dls->backoff is
5449 * DL_SCHED_RANDOM_EXPONENTIAL, we must have 0 <= min_delay <= max_delay <=
5450 * INT_MAX, but schedule may be set to NULL; otherwise schedule is required.
5451 * This function sets dls->next_attempt_at based on now, and returns the delay.
5452 * Helper for download_status_increment_failure and
5453 * download_status_increment_attempt. */
5455 download_status_schedule_get_delay(download_status_t
*dls
,
5456 const smartlist_t
*schedule
,
5457 int min_delay
, int max_delay
,
5461 /* We don't need a schedule if we're using random exponential backoff */
5462 tor_assert(dls
->backoff
== DL_SCHED_RANDOM_EXPONENTIAL
||
5464 /* If we're using random exponential backoff, we do need min/max delay */
5465 tor_assert(dls
->backoff
!= DL_SCHED_RANDOM_EXPONENTIAL
||
5466 (min_delay
>= 0 && max_delay
>= min_delay
));
5468 int delay
= INT_MAX
;
5469 uint8_t dls_schedule_position
= (dls
->increment_on
5470 == DL_SCHED_INCREMENT_ATTEMPT
5471 ? dls
->n_download_attempts
5472 : dls
->n_download_failures
);
5474 if (dls
->backoff
== DL_SCHED_DETERMINISTIC
) {
5475 if (dls_schedule_position
< smartlist_len(schedule
))
5476 delay
= *(int *)smartlist_get(schedule
, dls_schedule_position
);
5477 else if (dls_schedule_position
== IMPOSSIBLE_TO_DOWNLOAD
)
5480 delay
= *(int *)smartlist_get(schedule
, smartlist_len(schedule
) - 1);
5481 } else if (dls
->backoff
== DL_SCHED_RANDOM_EXPONENTIAL
) {
5482 /* Check if we missed a reset somehow */
5483 IF_BUG_ONCE(dls
->last_backoff_position
> dls_schedule_position
) {
5484 dls
->last_backoff_position
= 0;
5485 dls
->last_delay_used
= 0;
5488 if (dls_schedule_position
> 0) {
5489 delay
= dls
->last_delay_used
;
5491 while (dls
->last_backoff_position
< dls_schedule_position
) {
5492 /* Do one increment step */
5493 delay
= next_random_exponential_delay(delay
, min_delay
, max_delay
);
5494 /* Update our position */
5495 ++(dls
->last_backoff_position
);
5498 /* If we're just starting out, use the minimum delay */
5502 /* Clamp it within min/max if we have them */
5503 if (min_delay
>= 0 && delay
< min_delay
) delay
= min_delay
;
5504 if (max_delay
!= INT_MAX
&& delay
> max_delay
) delay
= max_delay
;
5506 /* Store it for next time */
5507 dls
->last_backoff_position
= dls_schedule_position
;
5508 dls
->last_delay_used
= delay
;
5511 /* A negative delay makes no sense. Knowing that delay is
5512 * non-negative allows us to safely do the wrapping check below. */
5513 tor_assert(delay
>= 0);
5515 /* Avoid now+delay overflowing TIME_MAX, by comparing with a subtraction
5516 * that won't overflow (since delay is non-negative). */
5517 if (delay
< INT_MAX
&& now
<= TIME_MAX
- delay
) {
5518 dls
->next_attempt_at
= now
+delay
;
5520 dls
->next_attempt_at
= TIME_MAX
;
5526 /* Log a debug message about item, which increments on increment_action, has
5527 * incremented dls_n_download_increments times. The message varies based on
5528 * was_schedule_incremented (if not, not_incremented_response is logged), and
5529 * the values of increment, dls_next_attempt_at, and now.
5530 * Helper for download_status_increment_failure and
5531 * download_status_increment_attempt. */
5533 download_status_log_helper(const char *item
, int was_schedule_incremented
,
5534 const char *increment_action
,
5535 const char *not_incremented_response
,
5536 uint8_t dls_n_download_increments
, int increment
,
5537 time_t dls_next_attempt_at
, time_t now
)
5540 if (!was_schedule_incremented
)
5541 log_debug(LD_DIR
, "%s %s %d time(s); I'll try again %s.",
5542 item
, increment_action
, (int)dls_n_download_increments
,
5543 not_incremented_response
);
5544 else if (increment
== 0)
5545 log_debug(LD_DIR
, "%s %s %d time(s); I'll try again immediately.",
5546 item
, increment_action
, (int)dls_n_download_increments
);
5547 else if (dls_next_attempt_at
< TIME_MAX
)
5548 log_debug(LD_DIR
, "%s %s %d time(s); I'll try again in %d seconds.",
5549 item
, increment_action
, (int)dls_n_download_increments
,
5550 (int)(dls_next_attempt_at
-now
));
5552 log_debug(LD_DIR
, "%s %s %d time(s); Giving up for a while.",
5553 item
, increment_action
, (int)dls_n_download_increments
);
5557 /** Determine when a failed download attempt should be retried.
5558 * Called when an attempt to download <b>dls</b> has failed with HTTP status
5559 * <b>status_code</b>. Increment the failure count (if the code indicates a
5560 * real failure, or if we're a server) and set <b>dls</b>-\>next_attempt_at to
5561 * an appropriate time in the future and return it.
5562 * If <b>dls->increment_on</b> is DL_SCHED_INCREMENT_ATTEMPT, increment the
5563 * failure count, and return a time in the far future for the next attempt (to
5564 * avoid an immediate retry). */
5566 download_status_increment_failure(download_status_t
*dls
, int status_code
,
5567 const char *item
, int server
, time_t now
)
5569 (void) status_code
; // XXXX no longer used.
5570 (void) server
; // XXXX no longer used.
5572 int min_delay
= 0, max_delay
= INT_MAX
;
5576 /* dls wasn't reset before it was used */
5577 if (dls
->next_attempt_at
== 0) {
5578 download_status_reset(dls
);
5581 /* count the failure */
5582 if (dls
->n_download_failures
< IMPOSSIBLE_TO_DOWNLOAD
-1) {
5583 ++dls
->n_download_failures
;
5586 if (dls
->increment_on
== DL_SCHED_INCREMENT_FAILURE
) {
5587 /* We don't find out that a failure-based schedule has attempted a
5588 * connection until that connection fails.
5589 * We'll never find out about successful connections, but this doesn't
5590 * matter, because schedules are reset after a successful download.
5592 if (dls
->n_download_attempts
< IMPOSSIBLE_TO_DOWNLOAD
-1)
5593 ++dls
->n_download_attempts
;
5595 /* only return a failure retry time if this schedule increments on failures
5597 const smartlist_t
*schedule
= find_dl_schedule(dls
, get_options());
5598 find_dl_min_and_max_delay(dls
, get_options(), &min_delay
, &max_delay
);
5599 increment
= download_status_schedule_get_delay(dls
, schedule
,
5600 min_delay
, max_delay
, now
);
5603 download_status_log_helper(item
, !dls
->increment_on
, "failed",
5604 "concurrently", dls
->n_download_failures
,
5606 download_status_get_next_attempt_at(dls
),
5609 if (dls
->increment_on
== DL_SCHED_INCREMENT_ATTEMPT
) {
5610 /* stop this schedule retrying on failure, it will launch concurrent
5611 * connections instead */
5614 return download_status_get_next_attempt_at(dls
);
5618 /** Determine when the next download attempt should be made when using an
5619 * attempt-based (potentially concurrent) download schedule.
5620 * Called when an attempt to download <b>dls</b> is being initiated.
5621 * Increment the attempt count and set <b>dls</b>-\>next_attempt_at to an
5622 * appropriate time in the future and return it.
5623 * If <b>dls->increment_on</b> is DL_SCHED_INCREMENT_FAILURE, don't increment
5624 * the attempts, and return a time in the far future (to avoid launching a
5625 * concurrent attempt). */
5627 download_status_increment_attempt(download_status_t
*dls
, const char *item
,
5631 int min_delay
= 0, max_delay
= INT_MAX
;
5635 /* dls wasn't reset before it was used */
5636 if (dls
->next_attempt_at
== 0) {
5637 download_status_reset(dls
);
5640 if (dls
->increment_on
== DL_SCHED_INCREMENT_FAILURE
) {
5641 /* this schedule should retry on failure, and not launch any concurrent
5643 log_warn(LD_BUG
, "Tried to launch an attempt-based connection on a "
5644 "failure-based schedule.");
5648 if (dls
->n_download_attempts
< IMPOSSIBLE_TO_DOWNLOAD
-1)
5649 ++dls
->n_download_attempts
;
5651 const smartlist_t
*schedule
= find_dl_schedule(dls
, get_options());
5652 find_dl_min_and_max_delay(dls
, get_options(), &min_delay
, &max_delay
);
5653 delay
= download_status_schedule_get_delay(dls
, schedule
,
5654 min_delay
, max_delay
, now
);
5656 download_status_log_helper(item
, dls
->increment_on
, "attempted",
5657 "on failure", dls
->n_download_attempts
,
5658 delay
, download_status_get_next_attempt_at(dls
),
5661 return download_status_get_next_attempt_at(dls
);
5665 download_status_get_initial_delay_from_now(const download_status_t
*dls
)
5667 const smartlist_t
*schedule
= find_dl_schedule(dls
, get_options());
5668 /* We use constant initial delays, even in exponential backoff
5670 return time(NULL
) + *(int *)smartlist_get(schedule
, 0);
5673 /** Reset <b>dls</b> so that it will be considered downloadable
5674 * immediately, and/or to show that we don't need it anymore.
5676 * Must be called to initialise a download schedule, otherwise the zeroth item
5677 * in the schedule will never be used.
5679 * (We find the zeroth element of the download schedule, and set
5680 * next_attempt_at to be the appropriate offset from 'now'. In most
5681 * cases this means setting it to 'now', so the item will be immediately
5682 * downloadable; when using authorities with fallbacks, there is a few seconds'
5685 download_status_reset(download_status_t
*dls
)
5687 if (dls
->n_download_failures
== IMPOSSIBLE_TO_DOWNLOAD
5688 || dls
->n_download_attempts
== IMPOSSIBLE_TO_DOWNLOAD
)
5689 return; /* Don't reset this. */
5691 dls
->n_download_failures
= 0;
5692 dls
->n_download_attempts
= 0;
5693 dls
->next_attempt_at
= download_status_get_initial_delay_from_now(dls
);
5694 dls
->last_backoff_position
= 0;
5695 dls
->last_delay_used
= 0;
5696 /* Don't reset dls->want_authority or dls->increment_on */
5699 /** Return the number of failures on <b>dls</b> since the last success (if
5702 download_status_get_n_failures(const download_status_t
*dls
)
5704 return dls
->n_download_failures
;
5707 /** Return the number of attempts to download <b>dls</b> since the last success
5708 * (if any). This can differ from download_status_get_n_failures() due to
5709 * outstanding concurrent attempts. */
5711 download_status_get_n_attempts(const download_status_t
*dls
)
5713 return dls
->n_download_attempts
;
5716 /** Return the next time to attempt to download <b>dls</b>. */
5718 download_status_get_next_attempt_at(const download_status_t
*dls
)
5720 /* dls wasn't reset before it was used */
5721 if (dls
->next_attempt_at
== 0) {
5722 /* so give the answer we would have given if it had been */
5723 return download_status_get_initial_delay_from_now(dls
);
5726 return dls
->next_attempt_at
;
5729 /** Called when one or more routerdesc (or extrainfo, if <b>was_extrainfo</b>)
5730 * fetches have failed (with uppercase fingerprints listed in <b>failed</b>,
5731 * either as descriptor digests or as identity digests based on
5732 * <b>was_descriptor_digests</b>).
5735 dir_routerdesc_download_failed(smartlist_t
*failed
, int status_code
,
5737 int was_extrainfo
, int was_descriptor_digests
)
5739 char digest
[DIGEST_LEN
];
5740 time_t now
= time(NULL
);
5741 int server
= directory_fetches_from_authorities(get_options());
5742 if (!was_descriptor_digests
) {
5743 if (router_purpose
== ROUTER_PURPOSE_BRIDGE
) {
5744 tor_assert(!was_extrainfo
);
5745 connection_dir_retry_bridges(failed
);
5747 return; /* FFFF should implement for other-than-router-purpose someday */
5749 SMARTLIST_FOREACH_BEGIN(failed
, const char *, cp
) {
5750 download_status_t
*dls
= NULL
;
5751 if (base16_decode(digest
, DIGEST_LEN
, cp
, strlen(cp
)) != DIGEST_LEN
) {
5752 log_warn(LD_BUG
, "Malformed fingerprint in list: %s", escaped(cp
));
5755 if (was_extrainfo
) {
5756 signed_descriptor_t
*sd
=
5757 router_get_by_extrainfo_digest(digest
);
5759 dls
= &sd
->ei_dl_status
;
5761 dls
= router_get_dl_status_by_descriptor_digest(digest
);
5763 if (!dls
|| dls
->n_download_failures
>=
5764 get_options()->TestingDescriptorMaxDownloadTries
)
5766 download_status_increment_failure(dls
, status_code
, cp
, server
, now
);
5767 } SMARTLIST_FOREACH_END(cp
);
5769 /* No need to relaunch descriptor downloads here: we already do it
5770 * every 10 or 60 seconds (FOO_DESCRIPTOR_RETRY_INTERVAL) in main.c. */
5773 /** Called when a connection to download microdescriptors from relay with
5774 * <b>dir_id</b> has failed in whole or in part. <b>failed</b> is a list
5775 * of every microdesc digest we didn't get. <b>status_code</b> is the http
5776 * status code we received. Reschedule the microdesc downloads as
5779 dir_microdesc_download_failed(smartlist_t
*failed
,
5780 int status_code
, const char *dir_id
)
5782 networkstatus_t
*consensus
5783 = networkstatus_get_latest_consensus_by_flavor(FLAV_MICRODESC
);
5785 download_status_t
*dls
;
5786 time_t now
= time(NULL
);
5787 int server
= directory_fetches_from_authorities(get_options());
5792 /* We failed to fetch a microdescriptor from 'dir_id', note it down
5793 * so that we don't try the same relay next time... */
5794 microdesc_note_outdated_dirserver(dir_id
);
5796 SMARTLIST_FOREACH_BEGIN(failed
, const char *, d
) {
5797 rs
= router_get_mutable_consensus_status_by_descriptor_digest(consensus
,d
);
5800 dls
= &rs
->dl_status
;
5801 if (dls
->n_download_failures
>=
5802 get_options()->TestingMicrodescMaxDownloadTries
) {
5806 { /* Increment the failure count for this md fetch */
5807 char buf
[BASE64_DIGEST256_LEN
+1];
5808 digest256_to_base64(buf
, d
);
5809 log_info(LD_DIR
, "Failed to download md %s from %s",
5810 buf
, hex_str(dir_id
, DIGEST_LEN
));
5811 download_status_increment_failure(dls
, status_code
, buf
,
5814 } SMARTLIST_FOREACH_END(d
);
5817 /** Helper. Compare two fp_pair_t objects, and return negative, 0, or
5818 * positive as appropriate. */
5820 compare_pairs_(const void **a
, const void **b
)
5822 const fp_pair_t
*fp1
= *a
, *fp2
= *b
;
5824 if ((r
= fast_memcmp(fp1
->first
, fp2
->first
, DIGEST_LEN
)))
5827 return fast_memcmp(fp1
->second
, fp2
->second
, DIGEST_LEN
);
5830 /** Divide a string <b>res</b> of the form FP1-FP2+FP3-FP4...[.z], where each
5831 * FP is a hex-encoded fingerprint, into a sequence of distinct sorted
5832 * fp_pair_t. Skip malformed pairs. On success, return 0 and add those
5833 * fp_pair_t into <b>pairs_out</b>. On failure, return -1. */
5835 dir_split_resource_into_fingerprint_pairs(const char *res
,
5836 smartlist_t
*pairs_out
)
5838 smartlist_t
*pairs_tmp
= smartlist_new();
5839 smartlist_t
*pairs_result
= smartlist_new();
5841 smartlist_split_string(pairs_tmp
, res
, "+", 0, 0);
5842 if (smartlist_len(pairs_tmp
)) {
5843 char *last
= smartlist_get(pairs_tmp
,smartlist_len(pairs_tmp
)-1);
5844 size_t last_len
= strlen(last
);
5845 if (last_len
> 2 && !strcmp(last
+last_len
-2, ".z")) {
5846 last
[last_len
-2] = '\0';
5849 SMARTLIST_FOREACH_BEGIN(pairs_tmp
, char *, cp
) {
5850 if (strlen(cp
) != HEX_DIGEST_LEN
*2+1) {
5852 "Skipping digest pair %s with non-standard length.", escaped(cp
));
5853 } else if (cp
[HEX_DIGEST_LEN
] != '-') {
5855 "Skipping digest pair %s with missing dash.", escaped(cp
));
5858 if (base16_decode(pair
.first
, DIGEST_LEN
,
5859 cp
, HEX_DIGEST_LEN
) != DIGEST_LEN
||
5860 base16_decode(pair
.second
,DIGEST_LEN
,
5861 cp
+HEX_DIGEST_LEN
+1, HEX_DIGEST_LEN
) != DIGEST_LEN
) {
5862 log_info(LD_DIR
, "Skipping non-decodable digest pair %s", escaped(cp
));
5864 smartlist_add(pairs_result
, tor_memdup(&pair
, sizeof(pair
)));
5868 } SMARTLIST_FOREACH_END(cp
);
5869 smartlist_free(pairs_tmp
);
5872 smartlist_sort(pairs_result
, compare_pairs_
);
5873 smartlist_uniq(pairs_result
, compare_pairs_
, tor_free_
);
5875 smartlist_add_all(pairs_out
, pairs_result
);
5876 smartlist_free(pairs_result
);
5880 /** Given a directory <b>resource</b> request, containing zero
5881 * or more strings separated by plus signs, followed optionally by ".z", store
5882 * the strings, in order, into <b>fp_out</b>. If <b>compressed_out</b> is
5883 * non-NULL, set it to 1 if the resource ends in ".z", else set it to 0.
5885 * If (flags & DSR_HEX), then delete all elements that aren't hex digests, and
5886 * decode the rest. If (flags & DSR_BASE64), then use "-" rather than "+" as
5887 * a separator, delete all the elements that aren't base64-encoded digests,
5888 * and decode the rest. If (flags & DSR_DIGEST256), these digests should be
5889 * 256 bits long; else they should be 160.
5891 * If (flags & DSR_SORT_UNIQ), then sort the list and remove all duplicates.
5894 dir_split_resource_into_fingerprints(const char *resource
,
5895 smartlist_t
*fp_out
, int *compressed_out
,
5898 const int decode_hex
= flags
& DSR_HEX
;
5899 const int decode_base64
= flags
& DSR_BASE64
;
5900 const int digests_are_256
= flags
& DSR_DIGEST256
;
5901 const int sort_uniq
= flags
& DSR_SORT_UNIQ
;
5903 const int digest_len
= digests_are_256
? DIGEST256_LEN
: DIGEST_LEN
;
5904 const int hex_digest_len
= digests_are_256
?
5905 HEX_DIGEST256_LEN
: HEX_DIGEST_LEN
;
5906 const int base64_digest_len
= digests_are_256
?
5907 BASE64_DIGEST256_LEN
: BASE64_DIGEST_LEN
;
5908 smartlist_t
*fp_tmp
= smartlist_new();
5910 tor_assert(!(decode_hex
&& decode_base64
));
5913 smartlist_split_string(fp_tmp
, resource
, decode_base64
?"-":"+", 0, 0);
5915 *compressed_out
= 0;
5916 if (smartlist_len(fp_tmp
)) {
5917 char *last
= smartlist_get(fp_tmp
,smartlist_len(fp_tmp
)-1);
5918 size_t last_len
= strlen(last
);
5919 if (last_len
> 2 && !strcmp(last
+last_len
-2, ".z")) {
5920 last
[last_len
-2] = '\0';
5922 *compressed_out
= 1;
5925 if (decode_hex
|| decode_base64
) {
5926 const size_t encoded_len
= decode_hex
? hex_digest_len
: base64_digest_len
;
5928 char *cp
, *d
= NULL
;
5929 for (i
= 0; i
< smartlist_len(fp_tmp
); ++i
) {
5930 cp
= smartlist_get(fp_tmp
, i
);
5931 if (strlen(cp
) != encoded_len
) {
5933 "Skipping digest %s with non-standard length.", escaped(cp
));
5934 smartlist_del_keeporder(fp_tmp
, i
--);
5937 d
= tor_malloc_zero(digest_len
);
5939 (base16_decode(d
, digest_len
, cp
, hex_digest_len
) != digest_len
) :
5940 (base64_decode(d
, digest_len
, cp
, base64_digest_len
)
5942 log_info(LD_DIR
, "Skipping non-decodable digest %s", escaped(cp
));
5943 smartlist_del_keeporder(fp_tmp
, i
--);
5946 smartlist_set(fp_tmp
, i
, d
);
5954 if (decode_hex
|| decode_base64
) {
5955 if (digests_are_256
) {
5956 smartlist_sort_digests256(fp_tmp
);
5957 smartlist_uniq_digests256(fp_tmp
);
5959 smartlist_sort_digests(fp_tmp
);
5960 smartlist_uniq_digests(fp_tmp
);
5963 smartlist_sort_strings(fp_tmp
);
5964 smartlist_uniq_strings(fp_tmp
);
5967 smartlist_add_all(fp_out
, fp_tmp
);
5968 smartlist_free(fp_tmp
);
5972 /** As dir_split_resource_into_fingerprints, but instead fills
5973 * <b>spool_out</b> with a list of spoolable_resource_t for the resource
5974 * identified through <b>source</b>. */
5976 dir_split_resource_into_spoolable(const char *resource
,
5977 dir_spool_source_t source
,
5978 smartlist_t
*spool_out
,
5979 int *compressed_out
,
5982 smartlist_t
*fingerprints
= smartlist_new();
5984 tor_assert(flags
& (DSR_HEX
|DSR_BASE64
));
5985 const size_t digest_len
=
5986 (flags
& DSR_DIGEST256
) ? DIGEST256_LEN
: DIGEST_LEN
;
5988 int r
= dir_split_resource_into_fingerprints(resource
, fingerprints
,
5989 compressed_out
, flags
);
5990 /* This is not a very efficient implementation XXXX */
5991 SMARTLIST_FOREACH_BEGIN(fingerprints
, uint8_t *, digest
) {
5992 spooled_resource_t
*spooled
=
5993 spooled_resource_new(source
, digest
, digest_len
);
5995 smartlist_add(spool_out
, spooled
);
5997 } SMARTLIST_FOREACH_END(digest
);
5999 smartlist_free(fingerprints
);