1 /* Copyright (c) 2001-2004, Roger Dingledine.
2 * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
3 * Copyright (c) 2007-2016, The Tor Project, Inc. */
4 /* See LICENSE for licensing information */
9 #include "circuitbuild.h"
11 #include "connection.h"
12 #include "connection_edge.h"
14 #include "directory.h"
17 #include "entrynodes.h"
20 #include "microdesc.h"
21 #include "networkstatus.h"
25 #include "rendclient.h"
26 #include "rendcommon.h"
27 #include "rendservice.h"
30 #include "routerlist.h"
31 #include "routerparse.h"
32 #include "routerset.h"
34 #if defined(EXPORTMALLINFO) && defined(HAVE_MALLOC_H) && defined(HAVE_MALLINFO)
42 * \brief Code to send and fetch directories and router
43 * descriptors via HTTP. Directories use dirserv.c to generate the
44 * results; clients use routers.c to parse them.
47 /* In-points to directory.c:
49 * - directory_post_to_dirservers(), called from
50 * router_upload_dir_desc_to_dirservers() in router.c
51 * upload_service_descriptor() in rendservice.c
52 * - directory_get_from_dirserver(), called from
53 * rend_client_refetch_renddesc() in rendclient.c
54 * run_scheduled_events() in main.c
56 * - connection_dir_process_inbuf(), called from
57 * connection_process_inbuf() in connection.c
58 * - connection_dir_finished_flushing(), called from
59 * connection_finished_flushing() in connection.c
60 * - connection_dir_finished_connecting(), called from
61 * connection_finished_connecting() in connection.c
63 static void directory_send_command(dir_connection_t
*conn
,
64 int purpose
, int direct
, const char *resource
,
65 const char *payload
, size_t payload_len
,
66 time_t if_modified_since
);
67 static int directory_handle_command(dir_connection_t
*conn
);
68 static int body_is_plausible(const char *body
, size_t body_len
, int purpose
);
69 static char *http_get_header(const char *headers
, const char *which
);
70 static void http_set_address_origin(const char *headers
, connection_t
*conn
);
71 static void connection_dir_download_routerdesc_failed(dir_connection_t
*conn
);
72 static void connection_dir_bridge_routerdesc_failed(dir_connection_t
*conn
);
73 static void connection_dir_download_cert_failed(
74 dir_connection_t
*conn
, int status_code
);
75 static void connection_dir_retry_bridges(smartlist_t
*descs
);
76 static void dir_routerdesc_download_failed(smartlist_t
*failed
,
80 int was_descriptor_digests
);
81 static void dir_microdesc_download_failed(smartlist_t
*failed
,
83 static void note_client_request(int purpose
, int compressed
, size_t bytes
);
84 static int client_likes_consensus(networkstatus_t
*v
, const char *want_url
);
86 static void directory_initiate_command_rend(
87 const tor_addr_port_t
*or_addr_port
,
88 const tor_addr_port_t
*dir_addr_port
,
91 uint8_t router_purpose
,
92 dir_indirection_t indirection
,
96 time_t if_modified_since
,
97 const rend_data_t
*rend_query
);
99 /********* START VARIABLES **********/
101 /** How far in the future do we allow a directory server to tell us it is
102 * before deciding that one of us has the wrong time? */
103 #define ALLOW_DIRECTORY_TIME_SKEW (30*60)
105 #define X_ADDRESS_HEADER "X-Your-Address-Is: "
107 /** HTTP cache control: how long do we tell proxies they can cache each
108 * kind of document we serve? */
109 #define FULL_DIR_CACHE_LIFETIME (60*60)
110 #define RUNNINGROUTERS_CACHE_LIFETIME (20*60)
111 #define DIRPORTFRONTPAGE_CACHE_LIFETIME (20*60)
112 #define NETWORKSTATUS_CACHE_LIFETIME (5*60)
113 #define ROUTERDESC_CACHE_LIFETIME (30*60)
114 #define ROUTERDESC_BY_DIGEST_CACHE_LIFETIME (48*60*60)
115 #define ROBOTS_CACHE_LIFETIME (24*60*60)
116 #define MICRODESC_CACHE_LIFETIME (48*60*60)
118 /********* END VARIABLES ************/
120 /** Return true iff the directory purpose <b>dir_purpose</b> (and if it's
121 * fetching descriptors, it's fetching them for <b>router_purpose</b>)
122 * must use an anonymous connection to a directory. */
124 purpose_needs_anonymity(uint8_t dir_purpose
, uint8_t router_purpose
)
126 if (get_options()->AllDirActionsPrivate
)
128 if (router_purpose
== ROUTER_PURPOSE_BRIDGE
)
129 return 1; /* if no circuits yet, this might break bootstrapping, but it's
130 * needed to be safe. */
131 if (dir_purpose
== DIR_PURPOSE_UPLOAD_DIR
||
132 dir_purpose
== DIR_PURPOSE_UPLOAD_VOTE
||
133 dir_purpose
== DIR_PURPOSE_UPLOAD_SIGNATURES
||
134 dir_purpose
== DIR_PURPOSE_FETCH_STATUS_VOTE
||
135 dir_purpose
== DIR_PURPOSE_FETCH_DETACHED_SIGNATURES
||
136 dir_purpose
== DIR_PURPOSE_FETCH_CONSENSUS
||
137 dir_purpose
== DIR_PURPOSE_FETCH_CERTIFICATE
||
138 dir_purpose
== DIR_PURPOSE_FETCH_SERVERDESC
||
139 dir_purpose
== DIR_PURPOSE_FETCH_EXTRAINFO
||
140 dir_purpose
== DIR_PURPOSE_FETCH_MICRODESC
)
145 /** Return a newly allocated string describing <b>auth</b>. Only describes
146 * authority features. */
148 authdir_type_to_string(dirinfo_type_t auth
)
151 smartlist_t
*lst
= smartlist_new();
152 if (auth
& V3_DIRINFO
)
153 smartlist_add(lst
, (void*)"V3");
154 if (auth
& BRIDGE_DIRINFO
)
155 smartlist_add(lst
, (void*)"Bridge");
156 if (smartlist_len(lst
)) {
157 result
= smartlist_join_strings(lst
, ", ", 0, NULL
);
159 result
= tor_strdup("[Not an authority]");
165 /** Return a string describing a given directory connection purpose. */
167 dir_conn_purpose_to_string(int purpose
)
171 case DIR_PURPOSE_UPLOAD_DIR
:
172 return "server descriptor upload";
173 case DIR_PURPOSE_UPLOAD_VOTE
:
174 return "server vote upload";
175 case DIR_PURPOSE_UPLOAD_SIGNATURES
:
176 return "consensus signature upload";
177 case DIR_PURPOSE_FETCH_SERVERDESC
:
178 return "server descriptor fetch";
179 case DIR_PURPOSE_FETCH_EXTRAINFO
:
180 return "extra-info fetch";
181 case DIR_PURPOSE_FETCH_CONSENSUS
:
182 return "consensus network-status fetch";
183 case DIR_PURPOSE_FETCH_CERTIFICATE
:
184 return "authority cert fetch";
185 case DIR_PURPOSE_FETCH_STATUS_VOTE
:
186 return "status vote fetch";
187 case DIR_PURPOSE_FETCH_DETACHED_SIGNATURES
:
188 return "consensus signature fetch";
189 case DIR_PURPOSE_FETCH_RENDDESC_V2
:
190 return "hidden-service v2 descriptor fetch";
191 case DIR_PURPOSE_UPLOAD_RENDDESC_V2
:
192 return "hidden-service v2 descriptor upload";
193 case DIR_PURPOSE_FETCH_MICRODESC
:
194 return "microdescriptor fetch";
197 log_warn(LD_BUG
, "Called with unknown purpose %d", purpose
);
201 /** Return the requisite directory information types. */
202 STATIC dirinfo_type_t
203 dir_fetch_type(int dir_purpose
, int router_purpose
, const char *resource
)
206 switch (dir_purpose
) {
207 case DIR_PURPOSE_FETCH_EXTRAINFO
:
208 type
= EXTRAINFO_DIRINFO
;
209 if (router_purpose
== ROUTER_PURPOSE_BRIDGE
)
210 type
|= BRIDGE_DIRINFO
;
214 case DIR_PURPOSE_FETCH_SERVERDESC
:
215 if (router_purpose
== ROUTER_PURPOSE_BRIDGE
)
216 type
= BRIDGE_DIRINFO
;
220 case DIR_PURPOSE_FETCH_STATUS_VOTE
:
221 case DIR_PURPOSE_FETCH_DETACHED_SIGNATURES
:
222 case DIR_PURPOSE_FETCH_CERTIFICATE
:
225 case DIR_PURPOSE_FETCH_CONSENSUS
:
227 if (resource
&& !strcmp(resource
, "microdesc"))
228 type
|= MICRODESC_DIRINFO
;
230 case DIR_PURPOSE_FETCH_MICRODESC
:
231 type
= MICRODESC_DIRINFO
;
234 log_warn(LD_BUG
, "Unexpected purpose %d", (int)dir_purpose
);
241 /** Return true iff <b>identity_digest</b> is the digest of a router which
242 * says that it caches extrainfos. (If <b>is_authority</b> we always
243 * believe that to be true.) */
245 router_supports_extrainfo(const char *identity_digest
, int is_authority
)
247 const node_t
*node
= node_get_by_id(identity_digest
);
249 if (node
&& node
->ri
) {
250 if (node
->ri
->caches_extra_info
)
259 /** Return true iff any trusted directory authority has accepted our
262 * We consider any authority sufficient because waiting for all of
263 * them means it never happens while any authority is down; we don't
264 * go for something more complex in the middle (like \>1/3 or \>1/2 or
265 * \>=1/2) because that doesn't seem necessary yet.
268 directories_have_accepted_server_descriptor(void)
270 const smartlist_t
*servers
= router_get_trusted_dir_servers();
271 const or_options_t
*options
= get_options();
272 SMARTLIST_FOREACH(servers
, dir_server_t
*, d
, {
273 if ((d
->type
& options
->PublishServerDescriptor_
) &&
274 d
->has_accepted_serverdesc
) {
281 /** Start a connection to every suitable directory authority, using
282 * connection purpose <b>dir_purpose</b> and uploading <b>payload</b>
283 * (of length <b>payload_len</b>). The dir_purpose should be one of
284 * 'DIR_PURPOSE_UPLOAD_{DIR|VOTE|SIGNATURES}'.
286 * <b>router_purpose</b> describes the type of descriptor we're
287 * publishing, if we're publishing a descriptor -- e.g. general or bridge.
289 * <b>type</b> specifies what sort of dir authorities (V3,
290 * BRIDGE, etc) we should upload to.
292 * If <b>extrainfo_len</b> is nonzero, the first <b>payload_len</b> bytes of
293 * <b>payload</b> hold a router descriptor, and the next <b>extrainfo_len</b>
294 * bytes of <b>payload</b> hold an extra-info document. Upload the descriptor
295 * to all authorities, and the extra-info document to all authorities that
299 directory_post_to_dirservers(uint8_t dir_purpose
, uint8_t router_purpose
,
302 size_t payload_len
, size_t extrainfo_len
)
304 const or_options_t
*options
= get_options();
305 dir_indirection_t indirection
;
306 const smartlist_t
*dirservers
= router_get_trusted_dir_servers();
308 const int exclude_self
= (dir_purpose
== DIR_PURPOSE_UPLOAD_VOTE
||
309 dir_purpose
== DIR_PURPOSE_UPLOAD_SIGNATURES
);
310 tor_assert(dirservers
);
311 /* This tries dirservers which we believe to be down, but ultimately, that's
312 * harmless, and we may as well err on the side of getting things uploaded.
314 SMARTLIST_FOREACH_BEGIN(dirservers
, dir_server_t
*, ds
) {
315 routerstatus_t
*rs
= &(ds
->fake_status
);
316 size_t upload_len
= payload_len
;
318 if ((type
& ds
->type
) == 0)
321 if (exclude_self
&& router_digest_is_me(ds
->digest
)) {
322 /* we don't upload to ourselves, but at least there's now at least
323 * one authority of this type that has what we wanted to upload. */
328 if (options
->StrictNodes
&&
329 routerset_contains_routerstatus(options
->ExcludeNodes
, rs
, -1)) {
330 log_warn(LD_DIR
, "Wanted to contact authority '%s' for %s, but "
331 "it's in our ExcludedNodes list and StrictNodes is set. "
334 dir_conn_purpose_to_string(dir_purpose
));
338 found
= 1; /* at least one authority of this type was listed */
339 if (dir_purpose
== DIR_PURPOSE_UPLOAD_DIR
)
340 ds
->has_accepted_serverdesc
= 0;
342 if (extrainfo_len
&& router_supports_extrainfo(ds
->digest
, 1)) {
343 upload_len
+= extrainfo_len
;
344 log_info(LD_DIR
, "Uploading an extrainfo too (length %d)",
345 (int) extrainfo_len
);
347 if (purpose_needs_anonymity(dir_purpose
, router_purpose
)) {
348 indirection
= DIRIND_ANONYMOUS
;
349 } else if (!fascist_firewall_allows_dir_server(ds
,
350 FIREWALL_DIR_CONNECTION
,
352 if (fascist_firewall_allows_dir_server(ds
, FIREWALL_OR_CONNECTION
, 0))
353 indirection
= DIRIND_ONEHOP
;
355 indirection
= DIRIND_ANONYMOUS
;
357 indirection
= DIRIND_DIRECT_CONN
;
359 directory_initiate_command_routerstatus(rs
, dir_purpose
,
362 NULL
, payload
, upload_len
, 0);
363 } SMARTLIST_FOREACH_END(ds
);
365 char *s
= authdir_type_to_string(type
);
366 log_warn(LD_DIR
, "Publishing server descriptor to directory authorities "
367 "of type '%s', but no authorities of that type listed!", s
);
372 /** Return true iff, according to the values in <b>options</b>, we should be
373 * using directory guards for direct downloads of directory information. */
375 should_use_directory_guards(const or_options_t
*options
)
377 /* Public (non-bridge) servers never use directory guards. */
378 if (public_server_mode(options
))
380 /* If guards are disabled, or directory guards are disabled, we can't
381 * use directory guards.
383 if (!options
->UseEntryGuards
|| !options
->UseEntryGuardsAsDirGuards
)
385 /* If we're configured to fetch directory info aggressively or of a
386 * nonstandard type, don't use directory guards. */
387 if (options
->DownloadExtraInfo
|| options
->FetchDirInfoEarly
||
388 options
->FetchDirInfoExtraEarly
|| options
->FetchUselessDescriptors
)
393 /** Pick an unconstrained directory server from among our guards, the latest
394 * networkstatus, or the fallback dirservers, for use in downloading
395 * information of type <b>type</b>, and return its routerstatus. */
396 static const routerstatus_t
*
397 directory_pick_generic_dirserver(dirinfo_type_t type
, int pds_flags
,
400 const routerstatus_t
*rs
= NULL
;
401 const or_options_t
*options
= get_options();
403 if (options
->UseBridges
)
404 log_warn(LD_BUG
, "Called when we have UseBridges set.");
406 if (should_use_directory_guards(options
)) {
407 const node_t
*node
= choose_random_dirguard(type
);
411 /* anybody with a non-zero dirport will do */
412 rs
= router_pick_directory_server(type
, pds_flags
);
415 log_info(LD_DIR
, "No router found for %s; falling back to "
416 "dirserver list.", dir_conn_purpose_to_string(dir_purpose
));
417 rs
= router_pick_fallback_dirserver(type
, pds_flags
);
423 /** Start a connection to a random running directory server, using
424 * connection purpose <b>dir_purpose</b>, intending to fetch descriptors
425 * of purpose <b>router_purpose</b>, and requesting <b>resource</b>.
426 * Use <b>pds_flags</b> as arguments to router_pick_directory_server()
427 * or router_pick_trusteddirserver().
429 MOCK_IMPL(void, directory_get_from_dirserver
, (
431 uint8_t router_purpose
,
432 const char *resource
,
434 download_want_authority_t want_authority
))
436 const routerstatus_t
*rs
= NULL
;
437 const or_options_t
*options
= get_options();
438 int prefer_authority
= (directory_fetches_from_authorities(options
)
439 || want_authority
== DL_WANT_AUTHORITY
);
440 int require_authority
= 0;
441 int get_via_tor
= purpose_needs_anonymity(dir_purpose
, router_purpose
);
442 dirinfo_type_t type
= dir_fetch_type(dir_purpose
, router_purpose
, resource
);
443 time_t if_modified_since
= 0;
445 if (type
== NO_DIRINFO
)
448 if (dir_purpose
== DIR_PURPOSE_FETCH_CONSENSUS
) {
452 flav
= networkstatus_parse_flavor_name(resource
);
454 /* DEFAULT_IF_MODIFIED_SINCE_DELAY is 1/20 of the default consensus
457 #define DEFAULT_IF_MODIFIED_SINCE_DELAY (180)
459 /* IF we have a parsed consensus of this type, we can do an
460 * if-modified-time based on it. */
461 v
= networkstatus_get_latest_consensus_by_flavor(flav
);
463 /* In networks with particularly short V3AuthVotingIntervals,
464 * ask for the consensus if it's been modified since half the
465 * V3AuthVotingInterval of the most recent consensus. */
466 time_t ims_delay
= DEFAULT_IF_MODIFIED_SINCE_DELAY
;
467 if (v
->fresh_until
> v
->valid_after
468 && ims_delay
> (v
->fresh_until
- v
->valid_after
)/2) {
469 ims_delay
= (v
->fresh_until
- v
->valid_after
)/2;
471 if_modified_since
= v
->valid_after
+ ims_delay
;
474 /* Otherwise it might be a consensus we don't parse, but which we
475 * do cache. Look at the cached copy, perhaps. */
476 cached_dir_t
*cd
= dirserv_get_consensus(resource
);
477 /* We have no method of determining the voting interval from an
478 * unparsed consensus, so we use the default. */
480 if_modified_since
= cd
->published
+ DEFAULT_IF_MODIFIED_SINCE_DELAY
;
484 if (!options
->FetchServerDescriptors
)
488 if (options
->UseBridges
&& !(type
& BRIDGE_DIRINFO
)) {
489 /* We want to ask a running bridge for which we have a descriptor.
491 * When we ask choose_random_entry() for a bridge, we specify what
492 * sort of dir fetch we'll be doing, so it won't return a bridge
493 * that can't answer our question.
495 /* XXX024 Not all bridges handle conditional consensus downloading,
496 * so, for now, never assume the server supports that. -PP */
497 const node_t
*node
= choose_random_dirguard(type
);
498 if (node
&& node
->ri
) {
499 /* every bridge has a routerinfo. */
500 routerinfo_t
*ri
= node
->ri
;
501 /* clients always make OR connections to bridges */
502 tor_addr_port_t or_ap
;
503 /* we are willing to use a non-preferred address if we need to */
504 fascist_firewall_choose_address_node(node
, FIREWALL_OR_CONNECTION
, 0,
506 directory_initiate_command(&or_ap
.addr
, or_ap
.port
,
507 NULL
, 0, /*no dirport*/
508 ri
->cache_info
.identity_digest
,
512 resource
, NULL
, 0, if_modified_since
);
514 log_notice(LD_DIR
, "Ignoring directory request, since no bridge "
515 "nodes are available yet.");
518 if (prefer_authority
|| (type
& BRIDGE_DIRINFO
)) {
519 /* only ask authdirservers, and don't ask myself */
520 rs
= router_pick_trusteddirserver(type
, pds_flags
);
521 if (rs
== NULL
&& (pds_flags
& (PDS_NO_EXISTING_SERVERDESC_FETCH
|
522 PDS_NO_EXISTING_MICRODESC_FETCH
))) {
523 /* We don't want to fetch from any authorities that we're currently
524 * fetching server descriptors from, and we got no match. Did we
525 * get no match because all the authorities have connections
526 * fetching server descriptors (in which case we should just
527 * return,) or because all the authorities are down or on fire or
528 * unreachable or something (in which case we should go on with
529 * our fallback code)? */
530 pds_flags
&= ~(PDS_NO_EXISTING_SERVERDESC_FETCH
|
531 PDS_NO_EXISTING_MICRODESC_FETCH
);
532 rs
= router_pick_trusteddirserver(type
, pds_flags
);
534 log_debug(LD_DIR
, "Deferring serverdesc fetch: all authorities "
539 if (rs
== NULL
&& require_authority
) {
540 log_info(LD_DIR
, "No authorities were available for %s: will try "
541 "later.", dir_conn_purpose_to_string(dir_purpose
));
545 if (!rs
&& !(type
& BRIDGE_DIRINFO
)) {
547 rs
= directory_pick_generic_dirserver(type
, pds_flags
,
550 get_via_tor
= 1; /* last resort: try routing it via Tor */
556 /* Never use fascistfirewall; we're going via Tor. */
557 pds_flags
|= PDS_IGNORE_FASCISTFIREWALL
;
558 rs
= router_pick_directory_server(type
, pds_flags
);
561 /* If we have any hope of building an indirect conn, we know some router
562 * descriptors. If (rs==NULL), we can't build circuits anyway, so
563 * there's no point in falling back to the authorities in this case. */
565 const dir_indirection_t indirection
=
566 get_via_tor
? DIRIND_ANONYMOUS
: DIRIND_ONEHOP
;
567 directory_initiate_command_routerstatus(rs
, dir_purpose
,
574 "While fetching directory info, "
575 "no running dirservers known. Will try again later. "
576 "(purpose %d)", dir_purpose
);
577 if (!purpose_needs_anonymity(dir_purpose
, router_purpose
)) {
578 /* remember we tried them all and failed. */
579 directory_all_unreachable(time(NULL
));
584 /** As directory_get_from_dirserver, but initiates a request to <i>every</i>
585 * directory authority other than ourself. Only for use by authorities when
586 * searching for missing information while voting. */
588 directory_get_from_all_authorities(uint8_t dir_purpose
,
589 uint8_t router_purpose
,
590 const char *resource
)
592 tor_assert(dir_purpose
== DIR_PURPOSE_FETCH_STATUS_VOTE
||
593 dir_purpose
== DIR_PURPOSE_FETCH_DETACHED_SIGNATURES
);
595 SMARTLIST_FOREACH_BEGIN(router_get_trusted_dir_servers(),
596 dir_server_t
*, ds
) {
598 if (router_digest_is_me(ds
->digest
))
600 if (!(ds
->type
& V3_DIRINFO
))
602 rs
= &ds
->fake_status
;
603 directory_initiate_command_routerstatus(rs
, dir_purpose
, router_purpose
,
604 DIRIND_ONEHOP
, resource
, NULL
,
606 } SMARTLIST_FOREACH_END(ds
);
609 /** Return true iff <b>ind</b> requires a multihop circuit. */
611 dirind_is_anon(dir_indirection_t ind
)
613 return ind
== DIRIND_ANON_DIRPORT
|| ind
== DIRIND_ANONYMOUS
;
616 /* Choose reachable OR and Dir addresses and ports from status, copying them
617 * into use_or_ap and use_dir_ap. If indirection is anonymous, then we're
618 * connecting via another relay, so choose the primary IPv4 address and ports.
620 * status should have at least one reachable address, if we can't choose a
621 * reachable address, warn and return -1. Otherwise, return 0.
624 directory_choose_address_routerstatus(const routerstatus_t
*status
,
625 dir_indirection_t indirection
,
626 tor_addr_port_t
*use_or_ap
,
627 tor_addr_port_t
*use_dir_ap
)
629 tor_assert(status
!= NULL
);
630 tor_assert(use_or_ap
!= NULL
);
631 tor_assert(use_dir_ap
!= NULL
);
633 const int anonymized_connection
= dirind_is_anon(indirection
);
634 int have_or
= 0, have_dir
= 0;
636 /* We expect status to have at least one reachable address if we're
637 * connecting to it directly.
639 * Therefore, we can simply use the other address if the one we want isn't
640 * allowed by the firewall.
642 * (When Tor uploads and downloads a hidden service descriptor, it uses
643 * DIRIND_ANONYMOUS, except for Tor2Web, which uses DIRIND_ONEHOP.
644 * So this code will only modify the address for Tor2Web's HS descriptor
645 * fetches. Even Single Onion Servers (NYI) use DIRIND_ANONYMOUS, to avoid
646 * HSDirs denying service by rejecting descriptors.)
649 /* Initialise the OR / Dir addresses */
650 tor_addr_make_null(&use_or_ap
->addr
, AF_UNSPEC
);
652 tor_addr_make_null(&use_dir_ap
->addr
, AF_UNSPEC
);
653 use_dir_ap
->port
= 0;
655 if (anonymized_connection
) {
656 /* Use the primary (IPv4) OR address if we're making an indirect
658 tor_addr_from_ipv4h(&use_or_ap
->addr
, status
->addr
);
659 use_or_ap
->port
= status
->or_port
;
662 /* We use an IPv6 address if we have one and we prefer it.
663 * Use the preferred address and port if they are reachable, otherwise,
664 * use the alternate address and port (if any).
666 have_or
= fascist_firewall_choose_address_rs(status
,
667 FIREWALL_OR_CONNECTION
, 0,
671 have_dir
= fascist_firewall_choose_address_rs(status
,
672 FIREWALL_DIR_CONNECTION
, 0,
675 /* We rejected all addresses in the relay's status. This means we can't
677 if (!have_or
&& !have_dir
) {
678 static int logged_backtrace
= 0;
679 log_info(LD_BUG
, "Rejected all OR and Dir addresses from %s when "
680 "launching an outgoing directory connection to: IPv4 %s OR %d "
681 "Dir %d IPv6 %s OR %d Dir %d", routerstatus_describe(status
),
682 fmt_addr32(status
->addr
), status
->or_port
,
683 status
->dir_port
, fmt_addr(&status
->ipv6_addr
),
684 status
->ipv6_orport
, status
->dir_port
);
685 if (!logged_backtrace
) {
686 log_backtrace(LOG_INFO
, LD_BUG
, "Addresses came from");
687 logged_backtrace
= 1;
695 /** Same as directory_initiate_command_routerstatus(), but accepts
696 * rendezvous data to fetch a hidden service descriptor. */
698 directory_initiate_command_routerstatus_rend(const routerstatus_t
*status
,
700 uint8_t router_purpose
,
701 dir_indirection_t indirection
,
702 const char *resource
,
705 time_t if_modified_since
,
706 const rend_data_t
*rend_query
)
708 const or_options_t
*options
= get_options();
710 tor_addr_port_t use_or_ap
, use_dir_ap
;
711 const int anonymized_connection
= dirind_is_anon(indirection
);
713 tor_assert(status
!= NULL
);
715 node
= node_get_by_id(status
->identity_digest
);
717 if (!node
&& anonymized_connection
) {
718 log_info(LD_DIR
, "Not sending anonymized request to directory '%s'; we "
719 "don't have its router descriptor.",
720 routerstatus_describe(status
));
724 if (options
->ExcludeNodes
&& options
->StrictNodes
&&
725 routerset_contains_routerstatus(options
->ExcludeNodes
, status
, -1)) {
726 log_warn(LD_DIR
, "Wanted to contact directory mirror %s for %s, but "
727 "it's in our ExcludedNodes list and StrictNodes is set. "
728 "Skipping. This choice might make your Tor not work.",
729 routerstatus_describe(status
),
730 dir_conn_purpose_to_string(dir_purpose
));
734 /* At this point, if we are a clients making a direct connection to a
735 * directory server, we have selected a server that has at least one address
736 * allowed by ClientUseIPv4/6 and Reachable{"",OR,Dir}Addresses. This
737 * selection uses the preference in ClientPreferIPv6{OR,Dir}Port, if
738 * possible. (If UseBridges is set, clients always use IPv6, and prefer it
741 * Now choose an address that we can use to connect to the directory server.
743 if (directory_choose_address_routerstatus(status
, indirection
, &use_or_ap
,
748 /* We don't retry the alternate OR/Dir address for the same directory if
749 * the address we choose fails (#6772).
750 * Instead, we'll retry another directory on failure. */
752 directory_initiate_command_rend(&use_or_ap
, &use_dir_ap
,
753 status
->identity_digest
,
754 dir_purpose
, router_purpose
,
755 indirection
, resource
,
756 payload
, payload_len
, if_modified_since
,
760 /** Launch a new connection to the directory server <b>status</b> to
761 * upload or download a server or rendezvous
762 * descriptor. <b>dir_purpose</b> determines what
763 * kind of directory connection we're launching, and must be one of
764 * DIR_PURPOSE_{FETCH|UPLOAD}_{DIR|RENDDESC_V2}. <b>router_purpose</b>
765 * specifies the descriptor purposes we have in mind (currently only
766 * used for FETCH_DIR).
768 * When uploading, <b>payload</b> and <b>payload_len</b> determine the content
769 * of the HTTP post. Otherwise, <b>payload</b> should be NULL.
771 * When fetching a rendezvous descriptor, <b>resource</b> is the service ID we
774 MOCK_IMPL(void, directory_initiate_command_routerstatus
,
775 (const routerstatus_t
*status
,
777 uint8_t router_purpose
,
778 dir_indirection_t indirection
,
779 const char *resource
,
782 time_t if_modified_since
))
784 directory_initiate_command_routerstatus_rend(status
, dir_purpose
,
786 indirection
, resource
,
787 payload
, payload_len
,
788 if_modified_since
, NULL
);
791 /** Return true iff <b>conn</b> is the client side of a directory connection
792 * we launched to ourself in order to determine the reachability of our
795 directory_conn_is_self_reachability_test(dir_connection_t
*conn
)
797 if (conn
->requested_resource
&&
798 !strcmpstart(conn
->requested_resource
,"authority")) {
799 const routerinfo_t
*me
= router_get_my_routerinfo();
801 router_digest_is_me(conn
->identity_digest
) &&
802 tor_addr_eq_ipv4h(&conn
->base_
.addr
, me
->addr
) && /*XXXX prop 118*/
803 me
->dir_port
== conn
->base_
.port
)
809 /** Called when we are unable to complete the client's request to a directory
810 * server due to a network error: Mark the router as down and try again if
814 connection_dir_request_failed(dir_connection_t
*conn
)
816 if (directory_conn_is_self_reachability_test(conn
)) {
817 return; /* this was a test fetch. don't retry. */
819 if (!entry_list_is_constrained(get_options()))
820 router_set_status(conn
->identity_digest
, 0); /* don't try this one again */
821 if (conn
->base_
.purpose
== DIR_PURPOSE_FETCH_SERVERDESC
||
822 conn
->base_
.purpose
== DIR_PURPOSE_FETCH_EXTRAINFO
) {
823 log_info(LD_DIR
, "Giving up on serverdesc/extrainfo fetch from "
824 "directory server at '%s'; retrying",
825 conn
->base_
.address
);
826 if (conn
->router_purpose
== ROUTER_PURPOSE_BRIDGE
)
827 connection_dir_bridge_routerdesc_failed(conn
);
828 connection_dir_download_routerdesc_failed(conn
);
829 } else if (conn
->base_
.purpose
== DIR_PURPOSE_FETCH_CONSENSUS
) {
830 if (conn
->requested_resource
)
831 networkstatus_consensus_download_failed(0, conn
->requested_resource
);
832 } else if (conn
->base_
.purpose
== DIR_PURPOSE_FETCH_CERTIFICATE
) {
833 log_info(LD_DIR
, "Giving up on certificate fetch from directory server "
835 conn
->base_
.address
);
836 connection_dir_download_cert_failed(conn
, 0);
837 } else if (conn
->base_
.purpose
== DIR_PURPOSE_FETCH_DETACHED_SIGNATURES
) {
838 log_info(LD_DIR
, "Giving up downloading detached signatures from '%s'",
839 conn
->base_
.address
);
840 } else if (conn
->base_
.purpose
== DIR_PURPOSE_FETCH_STATUS_VOTE
) {
841 log_info(LD_DIR
, "Giving up downloading votes from '%s'",
842 conn
->base_
.address
);
843 } else if (conn
->base_
.purpose
== DIR_PURPOSE_FETCH_MICRODESC
) {
844 log_info(LD_DIR
, "Giving up on downloading microdescriptors from "
845 "directory server at '%s'; will retry", conn
->base_
.address
);
846 connection_dir_download_routerdesc_failed(conn
);
850 /** Helper: Attempt to fetch directly the descriptors of each bridge
851 * listed in <b>failed</b>.
854 connection_dir_retry_bridges(smartlist_t
*descs
)
856 char digest
[DIGEST_LEN
];
857 SMARTLIST_FOREACH(descs
, const char *, cp
,
859 if (base16_decode(digest
, DIGEST_LEN
, cp
, strlen(cp
))<0) {
860 log_warn(LD_BUG
, "Malformed fingerprint in list: %s",
864 retry_bridge_descriptor_fetch_directly(digest
);
868 /** Called when an attempt to download one or more router descriptors
869 * or extra-info documents on connection <b>conn</b> failed.
872 connection_dir_download_routerdesc_failed(dir_connection_t
*conn
)
874 /* No need to increment the failure count for routerdescs, since
875 * it's not their fault. */
877 /* No need to relaunch descriptor downloads here: we already do it
878 * every 10 or 60 seconds (FOO_DESCRIPTOR_RETRY_INTERVAL) in main.c. */
879 tor_assert(conn
->base_
.purpose
== DIR_PURPOSE_FETCH_SERVERDESC
||
880 conn
->base_
.purpose
== DIR_PURPOSE_FETCH_EXTRAINFO
||
881 conn
->base_
.purpose
== DIR_PURPOSE_FETCH_MICRODESC
);
886 /** Called when an attempt to download a bridge's routerdesc from
887 * one of the authorities failed due to a network error. If
888 * possible attempt to download descriptors from the bridge directly.
891 connection_dir_bridge_routerdesc_failed(dir_connection_t
*conn
)
893 smartlist_t
*which
= NULL
;
895 /* Requests for bridge descriptors are in the form 'fp/', so ignore
897 if (!conn
->requested_resource
|| strcmpstart(conn
->requested_resource
,"fp/"))
900 which
= smartlist_new();
901 dir_split_resource_into_fingerprints(conn
->requested_resource
905 tor_assert(conn
->base_
.purpose
!= DIR_PURPOSE_FETCH_EXTRAINFO
);
906 if (smartlist_len(which
)) {
907 connection_dir_retry_bridges(which
);
908 SMARTLIST_FOREACH(which
, char *, cp
, tor_free(cp
));
910 smartlist_free(which
);
913 /** Called when an attempt to fetch a certificate fails. */
915 connection_dir_download_cert_failed(dir_connection_t
*conn
, int status
)
917 const char *fp_pfx
= "fp/";
918 const char *fpsk_pfx
= "fp-sk/";
920 tor_assert(conn
->base_
.purpose
== DIR_PURPOSE_FETCH_CERTIFICATE
);
922 if (!conn
->requested_resource
)
924 failed
= smartlist_new();
926 * We have two cases download by fingerprint (resource starts
927 * with "fp/") or download by fingerprint/signing key pair
928 * (resource starts with "fp-sk/").
930 if (!strcmpstart(conn
->requested_resource
, fp_pfx
)) {
931 /* Download by fingerprint case */
932 dir_split_resource_into_fingerprints(conn
->requested_resource
+
934 failed
, NULL
, DSR_HEX
);
935 SMARTLIST_FOREACH_BEGIN(failed
, char *, cp
) {
936 /* Null signing key digest indicates download by fp only */
937 authority_cert_dl_failed(cp
, NULL
, status
);
939 } SMARTLIST_FOREACH_END(cp
);
940 } else if (!strcmpstart(conn
->requested_resource
, fpsk_pfx
)) {
941 /* Download by (fp,sk) pairs */
942 dir_split_resource_into_fingerprint_pairs(conn
->requested_resource
+
943 strlen(fpsk_pfx
), failed
);
944 SMARTLIST_FOREACH_BEGIN(failed
, fp_pair_t
*, cp
) {
945 authority_cert_dl_failed(cp
->first
, cp
->second
, status
);
947 } SMARTLIST_FOREACH_END(cp
);
950 "Don't know what to do with failure for cert fetch %s",
951 conn
->requested_resource
);
954 smartlist_free(failed
);
956 update_certificate_downloads(time(NULL
));
959 /** Evaluate the situation and decide if we should use an encrypted
960 * "begindir-style" connection for this directory request.
961 * 1) If or_port is 0, or it's a direct conn and or_port is firewalled
962 * or we're a dir mirror, no.
963 * 2) If we prefer to avoid begindir conns, and we're not fetching or
964 * publishing a bridge relay descriptor, no.
968 directory_command_should_use_begindir(const or_options_t
*options
,
969 const tor_addr_t
*addr
,
970 int or_port
, uint8_t router_purpose
,
971 dir_indirection_t indirection
)
973 (void) router_purpose
;
975 return 0; /* We don't know an ORPort -- no chance. */
976 if (indirection
== DIRIND_DIRECT_CONN
|| indirection
== DIRIND_ANON_DIRPORT
)
978 if (indirection
== DIRIND_ONEHOP
)
979 if (!fascist_firewall_allows_address_addr(addr
, or_port
,
980 FIREWALL_OR_CONNECTION
, 0, 0) ||
981 directory_fetches_from_authorities(options
))
982 return 0; /* We're firewalled or are acting like a relay -- also no. */
986 /** Helper for directory_initiate_command_rend: send the
987 * command to a server whose OR address/port is <b>or_addr</b>/<b>or_port</b>,
988 * whose directory address/port is <b>dir_addr</b>/<b>dir_port</b>, whose
989 * identity key digest is <b>digest</b>, with purposes <b>dir_purpose</b> and
990 * <b>router_purpose</b>, making an (in)direct connection as specified in
991 * <b>indirection</b>, with command <b>resource</b>, <b>payload</b> of
992 * <b>payload_len</b>, and asking for a result only <b>if_modified_since</b>.
995 directory_initiate_command(const tor_addr_t
*or_addr
, uint16_t or_port
,
996 const tor_addr_t
*dir_addr
, uint16_t dir_port
,
998 uint8_t dir_purpose
, uint8_t router_purpose
,
999 dir_indirection_t indirection
, const char *resource
,
1000 const char *payload
, size_t payload_len
,
1001 time_t if_modified_since
)
1003 tor_addr_port_t or_ap
, dir_ap
;
1005 /* Use the null tor_addr and 0 port if the address or port isn't valid. */
1006 if (tor_addr_port_is_valid(or_addr
, or_port
, 0)) {
1007 tor_addr_copy(&or_ap
.addr
, or_addr
);
1008 or_ap
.port
= or_port
;
1010 /* the family doesn't matter here, so make it IPv4 */
1011 tor_addr_make_null(&or_ap
.addr
, AF_INET
);
1012 or_ap
.port
= or_port
= 0;
1015 if (tor_addr_port_is_valid(dir_addr
, dir_port
, 0)) {
1016 tor_addr_copy(&dir_ap
.addr
, dir_addr
);
1017 dir_ap
.port
= dir_port
;
1019 /* the family doesn't matter here, so make it IPv4 */
1020 tor_addr_make_null(&dir_ap
.addr
, AF_INET
);
1021 dir_ap
.port
= dir_port
= 0;
1024 directory_initiate_command_rend(&or_ap
, &dir_ap
,
1025 digest
, dir_purpose
,
1026 router_purpose
, indirection
,
1027 resource
, payload
, payload_len
,
1028 if_modified_since
, NULL
);
1031 /** Return non-zero iff a directory connection with purpose
1032 * <b>dir_purpose</b> reveals sensitive information about a Tor
1033 * instance's client activities. (Such connections must be performed
1034 * through normal three-hop Tor circuits.) */
1036 is_sensitive_dir_purpose(uint8_t dir_purpose
)
1038 return ((dir_purpose
== DIR_PURPOSE_HAS_FETCHED_RENDDESC_V2
) ||
1039 (dir_purpose
== DIR_PURPOSE_UPLOAD_RENDDESC_V2
) ||
1040 (dir_purpose
== DIR_PURPOSE_FETCH_RENDDESC_V2
));
1043 /** Same as directory_initiate_command(), but accepts rendezvous data to
1044 * fetch a hidden service descriptor, and takes its address & port arguments
1045 * as tor_addr_port_t. */
1047 directory_initiate_command_rend(const tor_addr_port_t
*or_addr_port
,
1048 const tor_addr_port_t
*dir_addr_port
,
1050 uint8_t dir_purpose
, uint8_t router_purpose
,
1051 dir_indirection_t indirection
,
1052 const char *resource
,
1053 const char *payload
, size_t payload_len
,
1054 time_t if_modified_since
,
1055 const rend_data_t
*rend_query
)
1057 tor_assert(or_addr_port
);
1058 tor_assert(dir_addr_port
);
1059 tor_assert(or_addr_port
->port
|| dir_addr_port
->port
);
1062 dir_connection_t
*conn
;
1063 const or_options_t
*options
= get_options();
1064 int socket_error
= 0;
1065 /* Should the connection be to a relay's OR port (and inside that we will
1066 * send our directory request)? */
1067 const int use_begindir
= directory_command_should_use_begindir(options
,
1068 &or_addr_port
->addr
, or_addr_port
->port
,
1069 router_purpose
, indirection
);
1070 /* Will the connection go via a three-hop Tor circuit? Note that this
1071 * is separate from whether it will use_begindir. */
1072 const int anonymized_connection
= dirind_is_anon(indirection
);
1074 /* What is the address we want to make the directory request to? If
1075 * we're making a begindir request this is the ORPort of the relay
1076 * we're contacting; if not a begindir request, this is its DirPort.
1077 * Note that if anonymized_connection is true, we won't be initiating
1078 * a connection directly to this address. */
1080 tor_addr_copy(&addr
, &(use_begindir
? or_addr_port
: dir_addr_port
)->addr
);
1081 uint16_t port
= (use_begindir
? or_addr_port
: dir_addr_port
)->port
;
1083 log_debug(LD_DIR
, "anonymized %d, use_begindir %d.",
1084 anonymized_connection
, use_begindir
);
1086 log_debug(LD_DIR
, "Initiating %s", dir_conn_purpose_to_string(dir_purpose
));
1088 #ifndef NON_ANONYMOUS_MODE_ENABLED
1089 tor_assert(!(is_sensitive_dir_purpose(dir_purpose
) &&
1090 !anonymized_connection
));
1092 (void)is_sensitive_dir_purpose
;
1095 /* ensure that we don't make direct connections when a SOCKS server is
1097 if (!anonymized_connection
&& !use_begindir
&& !options
->HTTPProxy
&&
1098 (options
->Socks4Proxy
|| options
->Socks5Proxy
)) {
1099 log_warn(LD_DIR
, "Cannot connect to a directory server through a "
1104 /* Make sure that the destination addr and port we picked is viable. */
1105 if (!port
|| tor_addr_is_null(&addr
)) {
1106 static int logged_backtrace
= 0;
1108 "Cannot make an outgoing %sconnection without %sPort.",
1109 use_begindir
? "begindir " : "",
1110 use_begindir
? "an OR" : "a Dir");
1111 if (!logged_backtrace
) {
1112 log_backtrace(LOG_INFO
, LD_BUG
, "Address came from");
1113 logged_backtrace
= 1;
1118 /* ensure we don't make excess connections when we're already downloading
1119 * a consensus during bootstrap */
1120 if (connection_dir_avoid_extra_connection_for_purpose(dir_purpose
)) {
1124 conn
= dir_connection_new(tor_addr_family(&addr
));
1126 /* set up conn so it's got all the data we need to remember */
1127 tor_addr_copy(&conn
->base_
.addr
, &addr
);
1128 conn
->base_
.port
= port
;
1129 conn
->base_
.address
= tor_dup_addr(&addr
);
1130 memcpy(conn
->identity_digest
, digest
, DIGEST_LEN
);
1132 conn
->base_
.purpose
= dir_purpose
;
1133 conn
->router_purpose
= router_purpose
;
1135 /* give it an initial state */
1136 conn
->base_
.state
= DIR_CONN_STATE_CONNECTING
;
1138 /* decide whether we can learn our IP address from this conn */
1139 /* XXXX This is a bad name for this field now. */
1140 conn
->dirconn_direct
= !anonymized_connection
;
1142 /* copy rendezvous data, if any */
1144 conn
->rend_data
= rend_data_dup(rend_query
);
1146 if (!anonymized_connection
&& !use_begindir
) {
1147 /* then we want to connect to dirport directly */
1149 if (options
->HTTPProxy
) {
1150 tor_addr_copy(&addr
, &options
->HTTPProxyAddr
);
1151 port
= options
->HTTPProxyPort
;
1154 switch (connection_connect(TO_CONN(conn
), conn
->base_
.address
, &addr
,
1155 port
, &socket_error
)) {
1157 connection_mark_for_close(TO_CONN(conn
));
1160 /* start flushing conn */
1161 conn
->base_
.state
= DIR_CONN_STATE_CLIENT_SENDING
;
1164 /* Close this connection if there's another consensus connection
1165 * downloading (during bootstrap), or connecting (after bootstrap). */
1166 if (connection_dir_close_consensus_conn_if_extra(conn
)) {
1169 /* queue the command on the outbuf */
1170 directory_send_command(conn
, dir_purpose
, 1, resource
,
1171 payload
, payload_len
,
1173 connection_watch_events(TO_CONN(conn
), READ_EVENT
| WRITE_EVENT
);
1174 /* writable indicates finish, readable indicates broken link,
1175 error indicates broken link in windowsland. */
1178 /* We will use a Tor circuit (maybe 1-hop, maybe 3-hop, maybe with
1179 * begindir, maybe not with begindir) */
1181 entry_connection_t
*linked_conn
;
1183 /* Anonymized tunneled connections can never share a circuit.
1184 * One-hop directory connections can share circuits with each other
1185 * but nothing else. */
1186 int iso_flags
= anonymized_connection
? ISO_STREAM
: ISO_SESSIONGRP
;
1188 /* If it's an anonymized connection, remember the fact that we
1189 * wanted it for later: maybe we'll want it again soon. */
1190 if (anonymized_connection
&& use_begindir
)
1191 rep_hist_note_used_internal(time(NULL
), 0, 1);
1192 else if (anonymized_connection
&& !use_begindir
)
1193 rep_hist_note_used_port(time(NULL
), conn
->base_
.port
);
1195 /* make an AP connection
1196 * populate it and add it at the right state
1197 * hook up both sides
1200 connection_ap_make_link(TO_CONN(conn
),
1201 conn
->base_
.address
, conn
->base_
.port
,
1203 SESSION_GROUP_DIRCONN
, iso_flags
,
1204 use_begindir
, !anonymized_connection
);
1206 log_warn(LD_NET
,"Making tunnel to dirserver failed.");
1207 connection_mark_for_close(TO_CONN(conn
));
1211 if (connection_add(TO_CONN(conn
)) < 0) {
1212 log_warn(LD_NET
,"Unable to add connection for link to dirserver.");
1213 connection_mark_for_close(TO_CONN(conn
));
1216 /* Close this connection if there's another consensus connection
1217 * downloading (during bootstrap), or connecting (after bootstrap). */
1218 if (connection_dir_close_consensus_conn_if_extra(conn
)) {
1221 conn
->base_
.state
= DIR_CONN_STATE_CLIENT_SENDING
;
1222 /* queue the command on the outbuf */
1223 directory_send_command(conn
, dir_purpose
, 0, resource
,
1224 payload
, payload_len
,
1227 connection_watch_events(TO_CONN(conn
), READ_EVENT
|WRITE_EVENT
);
1228 IF_HAS_BUFFEREVENT(ENTRY_TO_CONN(linked_conn
), {
1229 connection_watch_events(ENTRY_TO_CONN(linked_conn
),
1230 READ_EVENT
|WRITE_EVENT
);
1231 }) ELSE_IF_NO_BUFFEREVENT
1232 connection_start_reading(ENTRY_TO_CONN(linked_conn
));
1236 /** Return true iff anything we say on <b>conn</b> is being encrypted before
1237 * we send it to the client/server. */
1239 connection_dir_is_encrypted(dir_connection_t
*conn
)
1241 /* Right now it's sufficient to see if conn is or has been linked, since
1242 * the only thing it could be linked to is an edge connection on a
1243 * circuit, and the only way it could have been unlinked is at the edge
1244 * connection getting closed.
1246 return TO_CONN(conn
)->linked
;
1249 /** Helper for sorting
1251 * sort strings alphabetically
1254 compare_strs_(const void **a
, const void **b
)
1256 const char *s1
= *a
, *s2
= *b
;
1257 return strcmp(s1
, s2
);
1260 #define CONDITIONAL_CONSENSUS_FPR_LEN 3
1261 #if (CONDITIONAL_CONSENSUS_FPR_LEN > DIGEST_LEN)
1262 #error "conditional consensus fingerprint length is larger than digest length"
1265 /** Return the URL we should use for a consensus download.
1267 * This url depends on whether or not the server we go to
1268 * is sufficiently new to support conditional consensus downloading,
1269 * i.e. GET .../consensus/<b>fpr</b>+<b>fpr</b>+<b>fpr</b>
1271 * If 'resource' is provided, it is the name of a consensus flavor to request.
1274 directory_get_consensus_url(const char *resource
)
1277 const char *hyphen
, *flavor
;
1278 if (resource
==NULL
|| strcmp(resource
, "ns")==0) {
1279 flavor
= ""; /* Request ns consensuses as "", so older servers will work*/
1287 char *authority_id_list
;
1288 smartlist_t
*authority_digests
= smartlist_new();
1290 SMARTLIST_FOREACH_BEGIN(router_get_trusted_dir_servers(),
1291 dir_server_t
*, ds
) {
1293 if (!(ds
->type
& V3_DIRINFO
))
1296 hex
= tor_malloc(2*CONDITIONAL_CONSENSUS_FPR_LEN
+1);
1297 base16_encode(hex
, 2*CONDITIONAL_CONSENSUS_FPR_LEN
+1,
1298 ds
->v3_identity_digest
, CONDITIONAL_CONSENSUS_FPR_LEN
);
1299 smartlist_add(authority_digests
, hex
);
1300 } SMARTLIST_FOREACH_END(ds
);
1301 smartlist_sort(authority_digests
, compare_strs_
);
1302 authority_id_list
= smartlist_join_strings(authority_digests
,
1305 tor_asprintf(&url
, "/tor/status-vote/current/consensus%s%s/%s.z",
1306 hyphen
, flavor
, authority_id_list
);
1308 SMARTLIST_FOREACH(authority_digests
, char *, cp
, tor_free(cp
));
1309 smartlist_free(authority_digests
);
1310 tor_free(authority_id_list
);
1316 * Copies the ipv6 from source to destination, subject to buffer size limit
1317 * size. If decorate is true, makes sure the copied address is decorated.
1320 copy_ipv6_address(char* destination
, const char* source
, size_t len
,
1322 tor_assert(destination
);
1325 if (decorate
&& source
[0] != '[') {
1326 tor_snprintf(destination
, len
, "[%s]", source
);
1328 strlcpy(destination
, source
, len
);
1332 /** Queue an appropriate HTTP command on conn-\>outbuf. The other args
1333 * are as in directory_initiate_command().
1336 directory_send_command(dir_connection_t
*conn
,
1337 int purpose
, int direct
, const char *resource
,
1338 const char *payload
, size_t payload_len
,
1339 time_t if_modified_since
)
1341 char proxystring
[256];
1342 char hoststring
[128];
1343 /* NEEDS to be the same size hoststring.
1344 Will be decorated with brackets around it if it is ipv6. */
1345 char decorated_address
[128];
1346 smartlist_t
*headers
= smartlist_new();
1349 const char *httpcommand
= NULL
;
1352 tor_assert(conn
->base_
.type
== CONN_TYPE_DIR
);
1354 tor_free(conn
->requested_resource
);
1356 conn
->requested_resource
= tor_strdup(resource
);
1358 /* decorate the ip address if it is ipv6 */
1359 if (strchr(conn
->base_
.address
, ':')) {
1360 copy_ipv6_address(decorated_address
, conn
->base_
.address
,
1361 sizeof(decorated_address
), 1);
1363 strlcpy(decorated_address
, conn
->base_
.address
, sizeof(decorated_address
));
1366 /* come up with a string for which Host: we want */
1367 if (conn
->base_
.port
== 80) {
1368 strlcpy(hoststring
, decorated_address
, sizeof(hoststring
));
1370 tor_snprintf(hoststring
, sizeof(hoststring
), "%s:%d",
1371 decorated_address
, conn
->base_
.port
);
1374 /* Format if-modified-since */
1375 if (if_modified_since
) {
1376 char b
[RFC1123_TIME_LEN
+1];
1377 format_rfc1123_time(b
, if_modified_since
);
1378 smartlist_add_asprintf(headers
, "If-Modified-Since: %s\r\n", b
);
1381 /* come up with some proxy lines, if we're using one. */
1382 if (direct
&& get_options()->HTTPProxy
) {
1383 char *base64_authenticator
=NULL
;
1384 const char *authenticator
= get_options()->HTTPProxyAuthenticator
;
1386 tor_snprintf(proxystring
, sizeof(proxystring
),"http://%s", hoststring
);
1387 if (authenticator
) {
1388 base64_authenticator
= alloc_http_authenticator(authenticator
);
1389 if (!base64_authenticator
)
1390 log_warn(LD_BUG
, "Encoding http authenticator failed");
1392 if (base64_authenticator
) {
1393 smartlist_add_asprintf(headers
,
1394 "Proxy-Authorization: Basic %s\r\n",
1395 base64_authenticator
);
1396 tor_free(base64_authenticator
);
1403 case DIR_PURPOSE_FETCH_CONSENSUS
:
1404 /* resource is optional. If present, it's a flavor name */
1405 tor_assert(!payload
);
1406 httpcommand
= "GET";
1407 url
= directory_get_consensus_url(resource
);
1408 log_info(LD_DIR
, "Downloading consensus from %s using %s",
1411 case DIR_PURPOSE_FETCH_CERTIFICATE
:
1412 tor_assert(resource
);
1413 tor_assert(!payload
);
1414 httpcommand
= "GET";
1415 tor_asprintf(&url
, "/tor/keys/%s", resource
);
1417 case DIR_PURPOSE_FETCH_STATUS_VOTE
:
1418 tor_assert(resource
);
1419 tor_assert(!payload
);
1420 httpcommand
= "GET";
1421 tor_asprintf(&url
, "/tor/status-vote/next/%s.z", resource
);
1423 case DIR_PURPOSE_FETCH_DETACHED_SIGNATURES
:
1424 tor_assert(!resource
);
1425 tor_assert(!payload
);
1426 httpcommand
= "GET";
1427 url
= tor_strdup("/tor/status-vote/next/consensus-signatures.z");
1429 case DIR_PURPOSE_FETCH_SERVERDESC
:
1430 tor_assert(resource
);
1431 httpcommand
= "GET";
1432 tor_asprintf(&url
, "/tor/server/%s", resource
);
1434 case DIR_PURPOSE_FETCH_EXTRAINFO
:
1435 tor_assert(resource
);
1436 httpcommand
= "GET";
1437 tor_asprintf(&url
, "/tor/extra/%s", resource
);
1439 case DIR_PURPOSE_FETCH_MICRODESC
:
1440 tor_assert(resource
);
1441 httpcommand
= "GET";
1442 tor_asprintf(&url
, "/tor/micro/%s", resource
);
1444 case DIR_PURPOSE_UPLOAD_DIR
: {
1445 const char *why
= router_get_descriptor_gen_reason();
1446 tor_assert(!resource
);
1447 tor_assert(payload
);
1448 httpcommand
= "POST";
1449 url
= tor_strdup("/tor/");
1451 smartlist_add_asprintf(headers
, "X-Desc-Gen-Reason: %s\r\n", why
);
1455 case DIR_PURPOSE_UPLOAD_VOTE
:
1456 tor_assert(!resource
);
1457 tor_assert(payload
);
1458 httpcommand
= "POST";
1459 url
= tor_strdup("/tor/post/vote");
1461 case DIR_PURPOSE_UPLOAD_SIGNATURES
:
1462 tor_assert(!resource
);
1463 tor_assert(payload
);
1464 httpcommand
= "POST";
1465 url
= tor_strdup("/tor/post/consensus-signature");
1467 case DIR_PURPOSE_FETCH_RENDDESC_V2
:
1468 tor_assert(resource
);
1469 tor_assert(strlen(resource
) <= REND_DESC_ID_V2_LEN_BASE32
);
1470 tor_assert(!payload
);
1471 httpcommand
= "GET";
1472 tor_asprintf(&url
, "/tor/rendezvous2/%s", resource
);
1474 case DIR_PURPOSE_UPLOAD_RENDDESC_V2
:
1475 tor_assert(!resource
);
1476 tor_assert(payload
);
1477 httpcommand
= "POST";
1478 url
= tor_strdup("/tor/rendezvous2/publish");
1485 /* warn in the non-tunneled case */
1486 if (direct
&& (strlen(proxystring
) + strlen(url
) >= 4096)) {
1488 "Squid does not like URLs longer than 4095 bytes, and this "
1489 "one is %d bytes long: %s%s",
1490 (int)(strlen(proxystring
) + strlen(url
)), proxystring
, url
);
1493 tor_snprintf(request
, sizeof(request
), "%s %s", httpcommand
, proxystring
);
1494 connection_write_to_buf(request
, strlen(request
), TO_CONN(conn
));
1495 connection_write_to_buf(url
, strlen(url
), TO_CONN(conn
));
1498 if (!strcmp(httpcommand
, "POST") || payload
) {
1499 smartlist_add_asprintf(headers
, "Content-Length: %lu\r\n",
1500 payload
? (unsigned long)payload_len
: 0);
1504 char *header
= smartlist_join_strings(headers
, "", 0, NULL
);
1505 tor_snprintf(request
, sizeof(request
), " HTTP/1.0\r\nHost: %s\r\n%s\r\n",
1506 hoststring
, header
);
1510 connection_write_to_buf(request
, strlen(request
), TO_CONN(conn
));
1513 /* then send the payload afterwards too */
1514 connection_write_to_buf(payload
, payload_len
, TO_CONN(conn
));
1517 SMARTLIST_FOREACH(headers
, char *, h
, tor_free(h
));
1518 smartlist_free(headers
);
1521 /** Parse an HTTP request string <b>headers</b> of the form
1523 * "\%s [http[s]://]\%s HTTP/1..."
1525 * If it's well-formed, strdup the second \%s into *<b>url</b>, and
1526 * nul-terminate it. If the url doesn't start with "/tor/", rewrite it
1527 * so it does. Return 0.
1528 * Otherwise, return -1.
1531 parse_http_url(const char *headers
, char **url
)
1533 char *s
, *start
, *tmp
;
1535 s
= (char *)eat_whitespace_no_nl(headers
);
1537 s
= (char *)find_whitespace(s
); /* get past GET/POST */
1539 s
= (char *)eat_whitespace_no_nl(s
);
1541 start
= s
; /* this is it, assuming it's valid */
1542 s
= (char *)find_whitespace(start
);
1545 /* tolerate the http[s] proxy style of putting the hostname in the url */
1546 if (s
-start
>= 4 && !strcmpstart(start
,"http")) {
1550 if (s
-tmp
>= 3 && !strcmpstart(tmp
,"://")) {
1551 tmp
= strchr(tmp
+3, '/');
1552 if (tmp
&& tmp
< s
) {
1553 log_debug(LD_DIR
,"Skipping over 'http[s]://hostname/' string");
1559 /* Check if the header is well formed (next sequence
1560 * should be HTTP/1.X\r\n). Assumes we're supporting 1.0? */
1564 char *e
= (char *)eat_whitespace_no_nl(s
);
1565 if (2 != tor_sscanf(e
, "HTTP/1.%u%c", &minor_ver
, &ch
)) {
1572 if (s
-start
< 5 || strcmpstart(start
,"/tor/")) { /* need to rewrite it */
1573 *url
= tor_malloc(s
- start
+ 5);
1574 strlcpy(*url
,"/tor", s
-start
+5);
1575 strlcat((*url
)+4, start
, s
-start
+1);
1577 *url
= tor_strndup(start
, s
-start
);
1582 /** Return a copy of the first HTTP header in <b>headers</b> whose key is
1583 * <b>which</b>. The key should be given with a terminating colon and space;
1584 * this function copies everything after, up to but not including the
1585 * following \\r\\n. */
1587 http_get_header(const char *headers
, const char *which
)
1589 const char *cp
= headers
;
1591 if (!strcasecmpstart(cp
, which
)) {
1593 cp
+= strlen(which
);
1594 if ((eos
= strchr(cp
,'\r')))
1595 return tor_strndup(cp
, eos
-cp
);
1597 return tor_strdup(cp
);
1599 cp
= strchr(cp
, '\n');
1606 /** If <b>headers</b> indicates that a proxy was involved, then rewrite
1607 * <b>conn</b>-\>address to describe our best guess of the address that
1608 * originated this HTTP request. */
1610 http_set_address_origin(const char *headers
, connection_t
*conn
)
1614 fwd
= http_get_header(headers
, "Forwarded-For: ");
1616 fwd
= http_get_header(headers
, "X-Forwarded-For: ");
1619 if (tor_addr_parse(&toraddr
,fwd
) == -1 ||
1620 tor_addr_is_internal(&toraddr
,0)) {
1621 log_debug(LD_DIR
, "Ignoring local/internal IP %s", escaped(fwd
));
1626 tor_free(conn
->address
);
1627 conn
->address
= tor_strdup(fwd
);
1632 /** Parse an HTTP response string <b>headers</b> of the form
1634 * "HTTP/1.\%d \%d\%s\r\n...".
1637 * If it's well-formed, assign the status code to *<b>code</b> and
1638 * return 0. Otherwise, return -1.
1640 * On success: If <b>date</b> is provided, set *date to the Date
1641 * header in the http headers, or 0 if no such header is found. If
1642 * <b>compression</b> is provided, set *<b>compression</b> to the
1643 * compression method given in the Content-Encoding header, or 0 if no
1644 * such header is found, or -1 if the value of the header is not
1645 * recognized. If <b>reason</b> is provided, strdup the reason string
1649 parse_http_response(const char *headers
, int *code
, time_t *date
,
1650 compress_method_t
*compression
, char **reason
)
1653 char datestr
[RFC1123_TIME_LEN
+1];
1654 smartlist_t
*parsed_headers
;
1655 tor_assert(headers
);
1658 while (TOR_ISSPACE(*headers
)) headers
++; /* tolerate leading whitespace */
1660 if (tor_sscanf(headers
, "HTTP/1.%u %u", &n1
, &n2
) < 2 ||
1661 (n1
!= 0 && n1
!= 1) ||
1662 (n2
< 100 || n2
>= 600)) {
1663 log_warn(LD_HTTP
,"Failed to parse header %s",escaped(headers
));
1668 parsed_headers
= smartlist_new();
1669 smartlist_split_string(parsed_headers
, headers
, "\n",
1670 SPLIT_SKIP_SPACE
|SPLIT_IGNORE_BLANK
, -1);
1672 smartlist_t
*status_line_elements
= smartlist_new();
1673 tor_assert(smartlist_len(parsed_headers
));
1674 smartlist_split_string(status_line_elements
,
1675 smartlist_get(parsed_headers
, 0),
1676 " ", SPLIT_SKIP_SPACE
|SPLIT_IGNORE_BLANK
, 3);
1677 tor_assert(smartlist_len(status_line_elements
) <= 3);
1678 if (smartlist_len(status_line_elements
) == 3) {
1679 *reason
= smartlist_get(status_line_elements
, 2);
1680 smartlist_set(status_line_elements
, 2, NULL
); /* Prevent free */
1682 SMARTLIST_FOREACH(status_line_elements
, char *, cp
, tor_free(cp
));
1683 smartlist_free(status_line_elements
);
1687 SMARTLIST_FOREACH(parsed_headers
, const char *, s
,
1688 if (!strcmpstart(s
, "Date: ")) {
1689 strlcpy(datestr
, s
+6, sizeof(datestr
));
1690 /* This will do nothing on failure, so we don't need to check
1691 the result. We shouldn't warn, since there are many other valid
1692 date formats besides the one we use. */
1693 parse_rfc1123_time(datestr
, date
);
1698 const char *enc
= NULL
;
1699 SMARTLIST_FOREACH(parsed_headers
, const char *, s
,
1700 if (!strcmpstart(s
, "Content-Encoding: ")) {
1703 if (!enc
|| !strcmp(enc
, "identity")) {
1704 *compression
= NO_METHOD
;
1705 } else if (!strcmp(enc
, "deflate") || !strcmp(enc
, "x-deflate")) {
1706 *compression
= ZLIB_METHOD
;
1707 } else if (!strcmp(enc
, "gzip") || !strcmp(enc
, "x-gzip")) {
1708 *compression
= GZIP_METHOD
;
1710 log_info(LD_HTTP
, "Unrecognized content encoding: %s. Trying to deal.",
1712 *compression
= UNKNOWN_METHOD
;
1715 SMARTLIST_FOREACH(parsed_headers
, char *, s
, tor_free(s
));
1716 smartlist_free(parsed_headers
);
1721 /** Return true iff <b>body</b> doesn't start with a plausible router or
1722 * network-status or microdescriptor opening. This is a sign of possible
1725 body_is_plausible(const char *body
, size_t len
, int purpose
)
1729 return 1; /* empty bodies don't need decompression */
1732 if (purpose
== DIR_PURPOSE_FETCH_MICRODESC
) {
1733 return (!strcmpstart(body
,"onion-key"));
1736 if (!strcmpstart(body
,"router") ||
1737 !strcmpstart(body
,"network-status"))
1739 for (i
=0;i
<32;++i
) {
1740 if (!TOR_ISPRINT(body
[i
]) && !TOR_ISSPACE(body
[i
]))
1747 /** Called when we've just fetched a bunch of router descriptors in
1748 * <b>body</b>. The list <b>which</b>, if present, holds digests for
1749 * descriptors we requested: descriptor digests if <b>descriptor_digests</b>
1750 * is true, or identity digests otherwise. Parse the descriptors, validate
1751 * them, and annotate them as having purpose <b>purpose</b> and as having been
1752 * downloaded from <b>source</b>.
1754 * Return the number of routers actually added. */
1756 load_downloaded_routers(const char *body
, smartlist_t
*which
,
1757 int descriptor_digests
,
1762 char time_buf
[ISO_TIME_LEN
+1];
1764 int general
= router_purpose
== ROUTER_PURPOSE_GENERAL
;
1765 format_iso_time(time_buf
, time(NULL
));
1768 if (tor_snprintf(buf
, sizeof(buf
),
1769 "@downloaded-at %s\n"
1771 "%s%s%s", time_buf
, escaped(source
),
1772 !general
? "@purpose " : "",
1773 !general
? router_purpose_to_string(router_purpose
) : "",
1774 !general
? "\n" : "")<0)
1777 added
= router_load_routers_from_string(body
, NULL
, SAVED_NOWHERE
, which
,
1778 descriptor_digests
, buf
);
1779 if (added
&& general
)
1780 control_event_bootstrap(BOOTSTRAP_STATUS_LOADING_DESCRIPTORS
,
1781 count_loading_descriptors_progress());
1785 /** We are a client, and we've finished reading the server's
1786 * response. Parse it and act appropriately.
1788 * If we're still happy with using this directory server in the future, return
1789 * 0. Otherwise return -1; and the caller should consider trying the request
1792 * The caller will take care of marking the connection for close.
1795 connection_dir_client_reached_eof(dir_connection_t
*conn
)
1799 char *reason
= NULL
;
1800 size_t body_len
= 0, orig_len
= 0;
1802 time_t date_header
= 0;
1804 compress_method_t compression
;
1807 int allow_partial
= (conn
->base_
.purpose
== DIR_PURPOSE_FETCH_SERVERDESC
||
1808 conn
->base_
.purpose
== DIR_PURPOSE_FETCH_EXTRAINFO
||
1809 conn
->base_
.purpose
== DIR_PURPOSE_FETCH_MICRODESC
);
1810 int was_compressed
= 0;
1811 time_t now
= time(NULL
);
1814 switch (connection_fetch_from_buf_http(TO_CONN(conn
),
1815 &headers
, MAX_HEADERS_SIZE
,
1816 &body
, &body_len
, MAX_DIR_DL_SIZE
,
1818 case -1: /* overflow */
1819 log_warn(LD_PROTOCOL
,
1820 "'fetch' response too large (server '%s:%d'). Closing.",
1821 conn
->base_
.address
, conn
->base_
.port
);
1825 "'fetch' response not all here, but we're at eof. Closing.");
1827 /* case 1, fall through */
1829 orig_len
= body_len
;
1831 if (parse_http_response(headers
, &status_code
, &date_header
,
1832 &compression
, &reason
) < 0) {
1833 log_warn(LD_HTTP
,"Unparseable headers (server '%s:%d'). Closing.",
1834 conn
->base_
.address
, conn
->base_
.port
);
1835 tor_free(body
); tor_free(headers
);
1838 if (!reason
) reason
= tor_strdup("[no reason given]");
1841 "Received response from directory server '%s:%d': %d %s "
1843 conn
->base_
.address
, conn
->base_
.port
, status_code
,
1845 conn
->base_
.purpose
);
1847 /* now check if it's got any hints for us about our IP address. */
1848 if (conn
->dirconn_direct
) {
1849 char *guess
= http_get_header(headers
, X_ADDRESS_HEADER
);
1851 router_new_address_suggestion(guess
, conn
);
1856 if (date_header
> 0) {
1857 /* The date header was written very soon after we sent our request,
1858 * so compute the skew as the difference between sending the request
1859 * and the date header. (We used to check now-date_header, but that's
1860 * inaccurate if we spend a lot of time downloading.)
1862 apparent_skew
= conn
->base_
.timestamp_lastwritten
- date_header
;
1863 if (labs(apparent_skew
)>ALLOW_DIRECTORY_TIME_SKEW
) {
1864 int trusted
= router_digest_is_trusted_dir(conn
->identity_digest
);
1865 clock_skew_warning(TO_CONN(conn
), apparent_skew
, trusted
, LD_HTTP
,
1866 "directory", "DIRSERV");
1867 skewed
= 1; /* don't check the recommended-versions line */
1869 log_debug(LD_HTTP
, "Time on received directory is within tolerance; "
1870 "we are %ld seconds skewed. (That's okay.)", apparent_skew
);
1873 (void) skewed
; /* skewed isn't used yet. */
1875 if (status_code
== 503) {
1878 const char *id_digest
= conn
->identity_digest
;
1879 log_info(LD_DIR
,"Received http status code %d (%s) from server "
1880 "'%s:%d'. I'll try again soon.",
1881 status_code
, escaped(reason
), conn
->base_
.address
,
1883 if ((rs
= router_get_mutable_consensus_status_by_id(id_digest
)))
1884 rs
->last_dir_503_at
= now
;
1885 if ((ds
= router_get_fallback_dirserver_by_digest(id_digest
)))
1886 ds
->fake_status
.last_dir_503_at
= now
;
1888 tor_free(body
); tor_free(headers
); tor_free(reason
);
1892 plausible
= body_is_plausible(body
, body_len
, conn
->base_
.purpose
);
1893 if (compression
!= NO_METHOD
|| !plausible
) {
1894 char *new_body
= NULL
;
1896 compress_method_t guessed
= detect_compression_method(body
, body_len
);
1897 if (compression
== UNKNOWN_METHOD
|| guessed
!= compression
) {
1898 /* Tell the user if we don't believe what we're told about compression.*/
1899 const char *description1
, *description2
;
1900 if (compression
== ZLIB_METHOD
)
1901 description1
= "as deflated";
1902 else if (compression
== GZIP_METHOD
)
1903 description1
= "as gzipped";
1904 else if (compression
== NO_METHOD
)
1905 description1
= "as uncompressed";
1907 description1
= "with an unknown Content-Encoding";
1908 if (guessed
== ZLIB_METHOD
)
1909 description2
= "deflated";
1910 else if (guessed
== GZIP_METHOD
)
1911 description2
= "gzipped";
1912 else if (!plausible
)
1913 description2
= "confusing binary junk";
1915 description2
= "uncompressed";
1917 log_info(LD_HTTP
, "HTTP body from server '%s:%d' was labeled %s, "
1918 "but it seems to be %s.%s",
1919 conn
->base_
.address
, conn
->base_
.port
, description1
,
1921 (compression
>0 && guessed
>0)?" Trying both.":"");
1923 /* Try declared compression first if we can. */
1924 if (compression
== GZIP_METHOD
|| compression
== ZLIB_METHOD
)
1925 tor_gzip_uncompress(&new_body
, &new_len
, body
, body_len
, compression
,
1926 !allow_partial
, LOG_PROTOCOL_WARN
);
1927 /* Okay, if that didn't work, and we think that it was compressed
1928 * differently, try that. */
1930 (guessed
== GZIP_METHOD
|| guessed
== ZLIB_METHOD
) &&
1931 compression
!= guessed
)
1932 tor_gzip_uncompress(&new_body
, &new_len
, body
, body_len
, guessed
,
1933 !allow_partial
, LOG_PROTOCOL_WARN
);
1934 /* If we're pretty sure that we have a compressed directory, and
1935 * we didn't manage to uncompress it, then warn and bail. */
1936 if (!plausible
&& !new_body
) {
1937 log_fn(LOG_PROTOCOL_WARN
, LD_HTTP
,
1938 "Unable to decompress HTTP body (server '%s:%d').",
1939 conn
->base_
.address
, conn
->base_
.port
);
1940 tor_free(body
); tor_free(headers
); tor_free(reason
);
1951 if (conn
->base_
.purpose
== DIR_PURPOSE_FETCH_CONSENSUS
) {
1953 const char *flavname
= conn
->requested_resource
;
1954 if (status_code
!= 200) {
1955 int severity
= (status_code
== 304) ? LOG_INFO
: LOG_WARN
;
1956 tor_log(severity
, LD_DIR
,
1957 "Received http status code %d (%s) from server "
1958 "'%s:%d' while fetching consensus directory.",
1959 status_code
, escaped(reason
), conn
->base_
.address
,
1961 tor_free(body
); tor_free(headers
); tor_free(reason
);
1962 networkstatus_consensus_download_failed(status_code
, flavname
);
1965 log_info(LD_DIR
,"Received consensus directory (size %d) from server "
1966 "'%s:%d'", (int)body_len
, conn
->base_
.address
, conn
->base_
.port
);
1967 if ((r
=networkstatus_set_current_consensus(body
, flavname
, 0))<0) {
1968 log_fn(r
<-1?LOG_WARN
:LOG_INFO
, LD_DIR
,
1969 "Unable to load %s consensus directory downloaded from "
1970 "server '%s:%d'. I'll try again soon.",
1971 flavname
, conn
->base_
.address
, conn
->base_
.port
);
1972 tor_free(body
); tor_free(headers
); tor_free(reason
);
1973 networkstatus_consensus_download_failed(0, flavname
);
1976 /* launches router downloads as needed */
1977 routers_update_all_from_networkstatus(now
, 3);
1978 update_microdescs_from_networkstatus(now
);
1979 update_microdesc_downloads(now
);
1980 directory_info_has_arrived(now
, 0, 0);
1981 log_info(LD_DIR
, "Successfully loaded consensus.");
1984 if (conn
->base_
.purpose
== DIR_PURPOSE_FETCH_CERTIFICATE
) {
1985 if (status_code
!= 200) {
1987 "Received http status code %d (%s) from server "
1988 "'%s:%d' while fetching \"/tor/keys/%s\".",
1989 status_code
, escaped(reason
), conn
->base_
.address
,
1990 conn
->base_
.port
, conn
->requested_resource
);
1991 connection_dir_download_cert_failed(conn
, status_code
);
1992 tor_free(body
); tor_free(headers
); tor_free(reason
);
1995 log_info(LD_DIR
,"Received authority certificates (size %d) from server "
1996 "'%s:%d'", (int)body_len
, conn
->base_
.address
, conn
->base_
.port
);
1999 * Tell trusted_dirs_load_certs_from_string() whether it was by fp
2003 if (!strcmpstart(conn
->requested_resource
, "fp/")) {
2004 src_code
= TRUSTED_DIRS_CERTS_SRC_DL_BY_ID_DIGEST
;
2005 } else if (!strcmpstart(conn
->requested_resource
, "fp-sk/")) {
2006 src_code
= TRUSTED_DIRS_CERTS_SRC_DL_BY_ID_SK_DIGEST
;
2009 if (src_code
!= -1) {
2010 if (trusted_dirs_load_certs_from_string(body
, src_code
, 1)<0) {
2011 log_warn(LD_DIR
, "Unable to parse fetched certificates");
2012 /* if we fetched more than one and only some failed, the successful
2013 * ones got flushed to disk so it's safe to call this on them */
2014 connection_dir_download_cert_failed(conn
, status_code
);
2016 directory_info_has_arrived(now
, 0, 0);
2017 log_info(LD_DIR
, "Successfully loaded certificates from fetch.");
2021 "Couldn't figure out what to do with fetched certificates for "
2022 "unknown resource %s",
2023 conn
->requested_resource
);
2024 connection_dir_download_cert_failed(conn
, status_code
);
2027 if (conn
->base_
.purpose
== DIR_PURPOSE_FETCH_STATUS_VOTE
) {
2030 log_info(LD_DIR
,"Got votes (size %d) from server %s:%d",
2031 (int)body_len
, conn
->base_
.address
, conn
->base_
.port
);
2032 if (status_code
!= 200) {
2034 "Received http status code %d (%s) from server "
2035 "'%s:%d' while fetching \"/tor/status-vote/next/%s.z\".",
2036 status_code
, escaped(reason
), conn
->base_
.address
,
2037 conn
->base_
.port
, conn
->requested_resource
);
2038 tor_free(body
); tor_free(headers
); tor_free(reason
);
2041 dirvote_add_vote(body
, &msg
, &st
);
2043 log_warn(LD_DIR
, "Error adding retrieved vote: %s", msg
);
2045 log_info(LD_DIR
, "Added vote(s) successfully [msg: %s]", msg
);
2048 if (conn
->base_
.purpose
== DIR_PURPOSE_FETCH_DETACHED_SIGNATURES
) {
2049 const char *msg
= NULL
;
2050 log_info(LD_DIR
,"Got detached signatures (size %d) from server %s:%d",
2051 (int)body_len
, conn
->base_
.address
, conn
->base_
.port
);
2052 if (status_code
!= 200) {
2054 "Received http status code %d (%s) from server '%s:%d' while fetching "
2055 "\"/tor/status-vote/next/consensus-signatures.z\".",
2056 status_code
, escaped(reason
), conn
->base_
.address
,
2058 tor_free(body
); tor_free(headers
); tor_free(reason
);
2061 if (dirvote_add_signatures(body
, conn
->base_
.address
, &msg
)<0) {
2062 log_warn(LD_DIR
, "Problem adding detached signatures from %s:%d: %s",
2063 conn
->base_
.address
, conn
->base_
.port
, msg
?msg
:"???");
2067 if (conn
->base_
.purpose
== DIR_PURPOSE_FETCH_SERVERDESC
||
2068 conn
->base_
.purpose
== DIR_PURPOSE_FETCH_EXTRAINFO
) {
2069 int was_ei
= conn
->base_
.purpose
== DIR_PURPOSE_FETCH_EXTRAINFO
;
2070 smartlist_t
*which
= NULL
;
2071 int n_asked_for
= 0;
2072 int descriptor_digests
= conn
->requested_resource
&&
2073 !strcmpstart(conn
->requested_resource
,"d/");
2074 log_info(LD_DIR
,"Received %s (size %d) from server '%s:%d'",
2075 was_ei
? "extra server info" : "server info",
2076 (int)body_len
, conn
->base_
.address
, conn
->base_
.port
);
2077 if (conn
->requested_resource
&&
2078 (!strcmpstart(conn
->requested_resource
,"d/") ||
2079 !strcmpstart(conn
->requested_resource
,"fp/"))) {
2080 which
= smartlist_new();
2081 dir_split_resource_into_fingerprints(conn
->requested_resource
+
2082 (descriptor_digests
? 2 : 3),
2084 n_asked_for
= smartlist_len(which
);
2086 if (status_code
!= 200) {
2087 int dir_okay
= status_code
== 404 ||
2088 (status_code
== 400 && !strcmp(reason
, "Servers unavailable."));
2089 /* 404 means that it didn't have them; no big deal.
2090 * Older (pre-0.1.1.8) servers said 400 Servers unavailable instead. */
2091 log_fn(dir_okay
? LOG_INFO
: LOG_WARN
, LD_DIR
,
2092 "Received http status code %d (%s) from server '%s:%d' "
2093 "while fetching \"/tor/server/%s\". I'll try again soon.",
2094 status_code
, escaped(reason
), conn
->base_
.address
,
2095 conn
->base_
.port
, conn
->requested_resource
);
2097 connection_dir_download_routerdesc_failed(conn
);
2099 dir_routerdesc_download_failed(which
, status_code
,
2100 conn
->router_purpose
,
2101 was_ei
, descriptor_digests
);
2102 SMARTLIST_FOREACH(which
, char *, cp
, tor_free(cp
));
2103 smartlist_free(which
);
2105 tor_free(body
); tor_free(headers
); tor_free(reason
);
2106 return dir_okay
? 0 : -1;
2108 /* Learn the routers, assuming we requested by fingerprint or "all"
2111 * We use "authority" to fetch our own descriptor for
2112 * testing, and to fetch bridge descriptors for bootstrapping. Ignore
2113 * the output of "authority" requests unless we are using bridges,
2114 * since otherwise they'll be the response from reachability tests,
2115 * and we don't really want to add that to our routerlist. */
2116 if (which
|| (conn
->requested_resource
&&
2117 (!strcmpstart(conn
->requested_resource
, "all") ||
2118 (!strcmpstart(conn
->requested_resource
, "authority") &&
2119 get_options()->UseBridges
)))) {
2120 /* as we learn from them, we remove them from 'which' */
2122 router_load_extrainfo_from_string(body
, NULL
, SAVED_NOWHERE
, which
,
2123 descriptor_digests
);
2125 //router_load_routers_from_string(body, NULL, SAVED_NOWHERE, which,
2126 // descriptor_digests, conn->router_purpose);
2127 if (load_downloaded_routers(body
, which
, descriptor_digests
,
2128 conn
->router_purpose
,
2129 conn
->base_
.address
))
2130 directory_info_has_arrived(now
, 0, 0);
2133 if (which
) { /* mark remaining ones as failed */
2134 log_info(LD_DIR
, "Received %d/%d %s requested from %s:%d",
2135 n_asked_for
-smartlist_len(which
), n_asked_for
,
2136 was_ei
? "extra-info documents" : "router descriptors",
2137 conn
->base_
.address
, (int)conn
->base_
.port
);
2138 if (smartlist_len(which
)) {
2139 dir_routerdesc_download_failed(which
, status_code
,
2140 conn
->router_purpose
,
2141 was_ei
, descriptor_digests
);
2143 SMARTLIST_FOREACH(which
, char *, cp
, tor_free(cp
));
2144 smartlist_free(which
);
2146 if (directory_conn_is_self_reachability_test(conn
))
2147 router_dirport_found_reachable();
2149 if (conn
->base_
.purpose
== DIR_PURPOSE_FETCH_MICRODESC
) {
2150 smartlist_t
*which
= NULL
;
2151 log_info(LD_DIR
,"Received answer to microdescriptor request (status %d, "
2152 "size %d) from server '%s:%d'",
2153 status_code
, (int)body_len
, conn
->base_
.address
,
2155 tor_assert(conn
->requested_resource
&&
2156 !strcmpstart(conn
->requested_resource
, "d/"));
2157 which
= smartlist_new();
2158 dir_split_resource_into_fingerprints(conn
->requested_resource
+2,
2160 DSR_DIGEST256
|DSR_BASE64
);
2161 if (status_code
!= 200) {
2162 log_info(LD_DIR
, "Received status code %d (%s) from server "
2163 "'%s:%d' while fetching \"/tor/micro/%s\". I'll try again "
2165 status_code
, escaped(reason
), conn
->base_
.address
,
2166 (int)conn
->base_
.port
, conn
->requested_resource
);
2167 dir_microdesc_download_failed(which
, status_code
);
2168 SMARTLIST_FOREACH(which
, char *, cp
, tor_free(cp
));
2169 smartlist_free(which
);
2170 tor_free(body
); tor_free(headers
); tor_free(reason
);
2174 mds
= microdescs_add_to_cache(get_microdesc_cache(),
2175 body
, body
+body_len
, SAVED_NOWHERE
, 0,
2177 if (smartlist_len(which
)) {
2178 /* Mark remaining ones as failed. */
2179 dir_microdesc_download_failed(which
, status_code
);
2181 if (mds
&& smartlist_len(mds
)) {
2182 control_event_bootstrap(BOOTSTRAP_STATUS_LOADING_DESCRIPTORS
,
2183 count_loading_descriptors_progress());
2184 directory_info_has_arrived(now
, 0, 1);
2186 SMARTLIST_FOREACH(which
, char *, cp
, tor_free(cp
));
2187 smartlist_free(which
);
2188 smartlist_free(mds
);
2192 if (conn
->base_
.purpose
== DIR_PURPOSE_UPLOAD_DIR
) {
2193 switch (status_code
) {
2196 router_get_trusteddirserver_by_digest(conn
->identity_digest
);
2197 char *rejected_hdr
= http_get_header(headers
,
2198 "X-Descriptor-Not-New: ");
2200 if (!strcmp(rejected_hdr
, "Yes")) {
2201 log_info(LD_GENERAL
,
2202 "Authority '%s' declined our descriptor (not new)",
2204 /* XXXX use this information; be sure to upload next one
2206 /* XXXX023 On further thought, the task above implies that we're
2207 * basing our regenerate-descriptor time on when we uploaded the
2208 * last descriptor, not on the published time of the last
2209 * descriptor. If those are different, that's a bad thing to
2212 tor_free(rejected_hdr
);
2214 log_info(LD_GENERAL
,"eof (status 200) after uploading server "
2215 "descriptor: finished.");
2216 control_event_server_status(
2217 LOG_NOTICE
, "ACCEPTED_SERVER_DESCRIPTOR DIRAUTH=%s:%d",
2218 conn
->base_
.address
, conn
->base_
.port
);
2220 ds
->has_accepted_serverdesc
= 1;
2221 if (directories_have_accepted_server_descriptor())
2222 control_event_server_status(LOG_NOTICE
, "GOOD_SERVER_DESCRIPTOR");
2226 log_warn(LD_GENERAL
,"http status 400 (%s) response from "
2227 "dirserver '%s:%d'. Please correct.",
2228 escaped(reason
), conn
->base_
.address
, conn
->base_
.port
);
2229 control_event_server_status(LOG_WARN
,
2230 "BAD_SERVER_DESCRIPTOR DIRAUTH=%s:%d REASON=\"%s\"",
2231 conn
->base_
.address
, conn
->base_
.port
, escaped(reason
));
2234 log_warn(LD_GENERAL
,
2235 "http status %d (%s) reason unexpected while uploading "
2236 "descriptor to server '%s:%d').",
2237 status_code
, escaped(reason
), conn
->base_
.address
,
2241 /* return 0 in all cases, since we don't want to mark any
2242 * dirservers down just because they don't like us. */
2245 if (conn
->base_
.purpose
== DIR_PURPOSE_UPLOAD_VOTE
) {
2246 switch (status_code
) {
2248 log_notice(LD_DIR
,"Uploaded a vote to dirserver %s:%d",
2249 conn
->base_
.address
, conn
->base_
.port
);
2253 log_warn(LD_DIR
,"http status 400 (%s) response after uploading "
2254 "vote to dirserver '%s:%d'. Please correct.",
2255 escaped(reason
), conn
->base_
.address
, conn
->base_
.port
);
2258 log_warn(LD_GENERAL
,
2259 "http status %d (%s) reason unexpected while uploading "
2260 "vote to server '%s:%d').",
2261 status_code
, escaped(reason
), conn
->base_
.address
,
2265 /* return 0 in all cases, since we don't want to mark any
2266 * dirservers down just because they don't like us. */
2269 if (conn
->base_
.purpose
== DIR_PURPOSE_UPLOAD_SIGNATURES
) {
2270 switch (status_code
) {
2272 log_notice(LD_DIR
,"Uploaded signature(s) to dirserver %s:%d",
2273 conn
->base_
.address
, conn
->base_
.port
);
2277 log_warn(LD_DIR
,"http status 400 (%s) response after uploading "
2278 "signatures to dirserver '%s:%d'. Please correct.",
2279 escaped(reason
), conn
->base_
.address
, conn
->base_
.port
);
2282 log_warn(LD_GENERAL
,
2283 "http status %d (%s) reason unexpected while uploading "
2284 "signatures to server '%s:%d').",
2285 status_code
, escaped(reason
), conn
->base_
.address
,
2289 /* return 0 in all cases, since we don't want to mark any
2290 * dirservers down just because they don't like us. */
2293 if (conn
->base_
.purpose
== DIR_PURPOSE_FETCH_RENDDESC_V2
) {
2294 #define SEND_HS_DESC_FAILED_EVENT(reason) ( \
2295 control_event_hs_descriptor_failed(conn->rend_data, \
2296 conn->identity_digest, \
2298 #define SEND_HS_DESC_FAILED_CONTENT() ( \
2299 control_event_hs_descriptor_content(conn->rend_data->onion_address, \
2300 conn->requested_resource, \
2301 conn->identity_digest, \
2303 tor_assert(conn
->rend_data
);
2304 log_info(LD_REND
,"Received rendezvous descriptor (size %d, status %d "
2306 (int)body_len
, status_code
, escaped(reason
));
2307 switch (status_code
) {
2310 rend_cache_entry_t
*entry
= NULL
;
2312 if (rend_cache_store_v2_desc_as_client(body
,
2313 conn
->requested_resource
, conn
->rend_data
, &entry
) < 0) {
2314 log_warn(LD_REND
,"Fetching v2 rendezvous descriptor failed. "
2315 "Retrying at another directory.");
2316 /* We'll retry when connection_about_to_close_connection()
2317 * cleans this dir conn up. */
2318 SEND_HS_DESC_FAILED_EVENT("BAD_DESC");
2319 SEND_HS_DESC_FAILED_CONTENT();
2321 char service_id
[REND_SERVICE_ID_LEN_BASE32
+ 1];
2322 /* Should never be NULL here if we found the descriptor. */
2324 rend_get_service_id(entry
->parsed
->pk
, service_id
);
2326 /* success. notify pending connections about this. */
2327 log_info(LD_REND
, "Successfully fetched v2 rendezvous "
2329 control_event_hs_descriptor_received(service_id
,
2331 conn
->identity_digest
);
2332 control_event_hs_descriptor_content(service_id
,
2333 conn
->requested_resource
,
2334 conn
->identity_digest
,
2336 conn
->base_
.purpose
= DIR_PURPOSE_HAS_FETCHED_RENDDESC_V2
;
2337 rend_client_desc_trynow(service_id
);
2338 memwipe(service_id
, 0, sizeof(service_id
));
2343 /* Not there. We'll retry when
2344 * connection_about_to_close_connection() cleans this conn up. */
2345 log_info(LD_REND
,"Fetching v2 rendezvous descriptor failed: "
2346 "Retrying at another directory.");
2347 SEND_HS_DESC_FAILED_EVENT("NOT_FOUND");
2348 SEND_HS_DESC_FAILED_CONTENT();
2351 log_warn(LD_REND
, "Fetching v2 rendezvous descriptor failed: "
2352 "http status 400 (%s). Dirserver didn't like our "
2353 "v2 rendezvous query? Retrying at another directory.",
2355 SEND_HS_DESC_FAILED_EVENT("QUERY_REJECTED");
2356 SEND_HS_DESC_FAILED_CONTENT();
2359 log_warn(LD_REND
, "Fetching v2 rendezvous descriptor failed: "
2360 "http status %d (%s) response unexpected while "
2361 "fetching v2 hidden service descriptor (server '%s:%d'). "
2362 "Retrying at another directory.",
2363 status_code
, escaped(reason
), conn
->base_
.address
,
2365 SEND_HS_DESC_FAILED_EVENT("UNEXPECTED");
2366 SEND_HS_DESC_FAILED_CONTENT();
2371 if (conn
->base_
.purpose
== DIR_PURPOSE_UPLOAD_RENDDESC_V2
) {
2372 #define SEND_HS_DESC_UPLOAD_FAILED_EVENT(reason) ( \
2373 control_event_hs_descriptor_upload_failed( \
2374 conn->identity_digest, \
2375 conn->rend_data->onion_address, \
2377 log_info(LD_REND
,"Uploaded rendezvous descriptor (status %d "
2379 status_code
, escaped(reason
));
2380 /* Without the rend data, we'll have a problem identifying what has been
2381 * uploaded for which service. */
2382 tor_assert(conn
->rend_data
);
2383 switch (status_code
) {
2386 "Uploading rendezvous descriptor: finished with status "
2387 "200 (%s)", escaped(reason
));
2388 control_event_hs_descriptor_uploaded(conn
->identity_digest
,
2389 conn
->rend_data
->onion_address
);
2390 rend_service_desc_has_uploaded(conn
->rend_data
);
2393 log_warn(LD_REND
,"http status 400 (%s) response from dirserver "
2394 "'%s:%d'. Malformed rendezvous descriptor?",
2395 escaped(reason
), conn
->base_
.address
, conn
->base_
.port
);
2396 SEND_HS_DESC_UPLOAD_FAILED_EVENT("UPLOAD_REJECTED");
2399 log_warn(LD_REND
,"http status %d (%s) response unexpected (server "
2401 status_code
, escaped(reason
), conn
->base_
.address
,
2403 SEND_HS_DESC_UPLOAD_FAILED_EVENT("UNEXPECTED");
2407 note_client_request(conn
->base_
.purpose
, was_compressed
, orig_len
);
2408 tor_free(body
); tor_free(headers
); tor_free(reason
);
2412 /** Called when a directory connection reaches EOF. */
2414 connection_dir_reached_eof(dir_connection_t
*conn
)
2417 if (conn
->base_
.state
!= DIR_CONN_STATE_CLIENT_READING
) {
2418 log_info(LD_HTTP
,"conn reached eof, not reading. [state=%d] Closing.",
2420 connection_close_immediate(TO_CONN(conn
)); /* error: give up on flushing */
2421 connection_mark_for_close(TO_CONN(conn
));
2425 retval
= connection_dir_client_reached_eof(conn
);
2426 if (retval
== 0) /* success */
2427 conn
->base_
.state
= DIR_CONN_STATE_CLIENT_FINISHED
;
2428 connection_mark_for_close(TO_CONN(conn
));
2432 /** If any directory object is arriving, and it's over 10MB large, we're
2433 * getting DoS'd. (As of 0.1.2.x, raw directories are about 1MB, and we never
2434 * ask for more than 96 router descriptors at a time.)
2436 #define MAX_DIRECTORY_OBJECT_SIZE (10*(1<<20))
2438 #define MAX_VOTE_DL_SIZE (MAX_DIRECTORY_OBJECT_SIZE * 5)
2440 /** Read handler for directory connections. (That's connections <em>to</em>
2441 * directory servers and connections <em>at</em> directory servers.)
2444 connection_dir_process_inbuf(dir_connection_t
*conn
)
2448 tor_assert(conn
->base_
.type
== CONN_TYPE_DIR
);
2450 /* Directory clients write, then read data until they receive EOF;
2451 * directory servers read data until they get an HTTP command, then
2452 * write their response (when it's finished flushing, they mark for
2456 /* If we're on the dirserver side, look for a command. */
2457 if (conn
->base_
.state
== DIR_CONN_STATE_SERVER_COMMAND_WAIT
) {
2458 if (directory_handle_command(conn
) < 0) {
2459 connection_mark_for_close(TO_CONN(conn
));
2466 (TO_CONN(conn
)->purpose
== DIR_PURPOSE_FETCH_STATUS_VOTE
) ?
2467 MAX_VOTE_DL_SIZE
: MAX_DIRECTORY_OBJECT_SIZE
;
2469 if (connection_get_inbuf_len(TO_CONN(conn
)) > max_size
) {
2471 "Too much data received from directory connection (%s): "
2472 "denial of service attempt, or you need to upgrade?",
2473 conn
->base_
.address
);
2474 connection_mark_for_close(TO_CONN(conn
));
2478 if (!conn
->base_
.inbuf_reached_eof
)
2479 log_debug(LD_HTTP
,"Got data, not eof. Leaving on inbuf.");
2483 /** Called when we're about to finally unlink and free a directory connection:
2484 * perform necessary accounting and cleanup */
2486 connection_dir_about_to_close(dir_connection_t
*dir_conn
)
2488 connection_t
*conn
= TO_CONN(dir_conn
);
2490 if (conn
->state
< DIR_CONN_STATE_CLIENT_FINISHED
) {
2491 /* It's a directory connection and connecting or fetching
2492 * failed: forget about this router, and maybe try again. */
2493 connection_dir_request_failed(dir_conn
);
2495 /* If we were trying to fetch a v2 rend desc and did not succeed,
2496 * retry as needed. (If a fetch is successful, the connection state
2497 * is changed to DIR_PURPOSE_HAS_FETCHED_RENDDESC_V2 to mark that
2498 * refetching is unnecessary.) */
2499 if (conn
->purpose
== DIR_PURPOSE_FETCH_RENDDESC_V2
&&
2500 dir_conn
->rend_data
&&
2501 strlen(dir_conn
->rend_data
->onion_address
) == REND_SERVICE_ID_LEN_BASE32
)
2502 rend_client_refetch_v2_renddesc(dir_conn
->rend_data
);
2505 /** Create an http response for the client <b>conn</b> out of
2506 * <b>status</b> and <b>reason_phrase</b>. Write it to <b>conn</b>.
2509 write_http_status_line(dir_connection_t
*conn
, int status
,
2510 const char *reason_phrase
)
2513 if (tor_snprintf(buf
, sizeof(buf
), "HTTP/1.0 %d %s\r\n\r\n",
2514 status
, reason_phrase
? reason_phrase
: "OK") < 0) {
2515 log_warn(LD_BUG
,"status line too long.");
2518 log_debug(LD_DIRSERV
,"Wrote status 'HTTP/1.0 %d %s'", status
, reason_phrase
);
2519 connection_write_to_buf(buf
, strlen(buf
), TO_CONN(conn
));
2522 /** Write the header for an HTTP/1.0 response onto <b>conn</b>-\>outbuf,
2523 * with <b>type</b> as the Content-Type.
2525 * If <b>length</b> is nonnegative, it is the Content-Length.
2526 * If <b>encoding</b> is provided, it is the Content-Encoding.
2527 * If <b>cache_lifetime</b> is greater than 0, the content may be cached for
2528 * up to cache_lifetime seconds. Otherwise, the content may not be cached. */
2530 write_http_response_header_impl(dir_connection_t
*conn
, ssize_t length
,
2531 const char *type
, const char *encoding
,
2532 const char *extra_headers
,
2533 long cache_lifetime
)
2535 char date
[RFC1123_TIME_LEN
+1];
2538 time_t now
= time(NULL
);
2542 format_rfc1123_time(date
, now
);
2544 tor_snprintf(cp
, sizeof(tmp
),
2545 "HTTP/1.0 200 OK\r\nDate: %s\r\n",
2549 tor_snprintf(cp
, sizeof(tmp
)-(cp
-tmp
), "Content-Type: %s\r\n", type
);
2552 if (!is_local_addr(&conn
->base_
.addr
)) {
2553 /* Don't report the source address for a nearby/private connection.
2554 * Otherwise we tend to mis-report in cases where incoming ports are
2555 * being forwarded to a Tor server running behind the firewall. */
2556 tor_snprintf(cp
, sizeof(tmp
)-(cp
-tmp
),
2557 X_ADDRESS_HEADER
"%s\r\n", conn
->base_
.address
);
2561 tor_snprintf(cp
, sizeof(tmp
)-(cp
-tmp
),
2562 "Content-Encoding: %s\r\n", encoding
);
2566 tor_snprintf(cp
, sizeof(tmp
)-(cp
-tmp
),
2567 "Content-Length: %ld\r\n", (long)length
);
2570 if (cache_lifetime
> 0) {
2571 char expbuf
[RFC1123_TIME_LEN
+1];
2572 format_rfc1123_time(expbuf
, (time_t)(now
+ cache_lifetime
));
2573 /* We could say 'Cache-control: max-age=%d' here if we start doing
2575 tor_snprintf(cp
, sizeof(tmp
)-(cp
-tmp
),
2576 "Expires: %s\r\n", expbuf
);
2578 } else if (cache_lifetime
== 0) {
2579 /* We could say 'Cache-control: no-cache' here if we start doing
2581 strlcpy(cp
, "Pragma: no-cache\r\n", sizeof(tmp
)-(cp
-tmp
));
2584 if (extra_headers
) {
2585 strlcpy(cp
, extra_headers
, sizeof(tmp
)-(cp
-tmp
));
2588 if (sizeof(tmp
)-(cp
-tmp
) > 3)
2589 memcpy(cp
, "\r\n", 3);
2592 connection_write_to_buf(tmp
, strlen(tmp
), TO_CONN(conn
));
2595 /** As write_http_response_header_impl, but sets encoding and content-typed
2596 * based on whether the response will be <b>compressed</b> or not. */
2598 write_http_response_header(dir_connection_t
*conn
, ssize_t length
,
2599 int compressed
, long cache_lifetime
)
2601 write_http_response_header_impl(conn
, length
,
2602 compressed
?"application/octet-stream":"text/plain",
2603 compressed
?"deflate":"identity",
2608 #if defined(INSTRUMENT_DOWNLOADS) || defined(RUNNING_DOXYGEN)
2610 typedef struct request_t
{
2611 uint64_t bytes
; /**< How many bytes have we transferred? */
2612 uint64_t count
; /**< How many requests have we made? */
2615 /** Map used to keep track of how much data we've up/downloaded in what kind
2616 * of request. Maps from request type to pointer to request_t. */
2617 static strmap_t
*request_map
= NULL
;
2619 /** Record that a client request of <b>purpose</b> was made, and that
2620 * <b>bytes</b> bytes of possibly <b>compressed</b> data were sent/received.
2621 * Used to keep track of how much we've up/downloaded in what kind of
2624 note_client_request(int purpose
, int compressed
, size_t bytes
)
2627 const char *kind
= NULL
;
2629 case DIR_PURPOSE_FETCH_CONSENSUS
: kind
= "dl/consensus"; break;
2630 case DIR_PURPOSE_FETCH_CERTIFICATE
: kind
= "dl/cert"; break;
2631 case DIR_PURPOSE_FETCH_STATUS_VOTE
: kind
= "dl/vote"; break;
2632 case DIR_PURPOSE_FETCH_DETACHED_SIGNATURES
: kind
= "dl/detached_sig";
2634 case DIR_PURPOSE_FETCH_SERVERDESC
: kind
= "dl/server"; break;
2635 case DIR_PURPOSE_FETCH_EXTRAINFO
: kind
= "dl/extra"; break;
2636 case DIR_PURPOSE_UPLOAD_DIR
: kind
= "dl/ul-dir"; break;
2637 case DIR_PURPOSE_UPLOAD_VOTE
: kind
= "dl/ul-vote"; break;
2638 case DIR_PURPOSE_UPLOAD_SIGNATURES
: kind
= "dl/ul-sig"; break;
2639 case DIR_PURPOSE_FETCH_RENDDESC_V2
: kind
= "dl/rend2"; break;
2640 case DIR_PURPOSE_UPLOAD_RENDDESC_V2
: kind
= "dl/ul-rend2"; break;
2643 tor_asprintf(&key
, "%s%s", kind
, compressed
?".z":"");
2645 tor_asprintf(&key
, "unknown purpose (%d)%s",
2646 purpose
, compressed
?".z":"");
2648 note_request(key
, bytes
);
2652 /** Helper: initialize the request map to instrument downloads. */
2654 ensure_request_map_initialized(void)
2657 request_map
= strmap_new();
2660 /** Called when we just transmitted or received <b>bytes</b> worth of data
2661 * because of a request of type <b>key</b> (an arbitrary identifier): adds
2662 * <b>bytes</b> to the total associated with key. */
2664 note_request(const char *key
, size_t bytes
)
2667 ensure_request_map_initialized();
2669 r
= strmap_get(request_map
, key
);
2671 r
= tor_malloc_zero(sizeof(request_t
));
2672 strmap_set(request_map
, key
, r
);
2678 /** Return a newly allocated string holding a summary of bytes used per
2681 directory_dump_request_log(void)
2685 strmap_iter_t
*iter
;
2687 ensure_request_map_initialized();
2689 lines
= smartlist_new();
2691 for (iter
= strmap_iter_init(request_map
);
2692 !strmap_iter_done(iter
);
2693 iter
= strmap_iter_next(request_map
, iter
)) {
2697 strmap_iter_get(iter
, &key
, &val
);
2699 smartlist_add_asprintf(lines
, "%s "U64_FORMAT
" "U64_FORMAT
"\n",
2700 key
, U64_PRINTF_ARG(r
->bytes
), U64_PRINTF_ARG(r
->count
));
2702 smartlist_sort_strings(lines
);
2703 result
= smartlist_join_strings(lines
, "", 0, NULL
);
2704 SMARTLIST_FOREACH(lines
, char *, cp
, tor_free(cp
));
2705 smartlist_free(lines
);
2710 note_client_request(int purpose
, int compressed
, size_t bytes
)
2718 note_request(const char *key
, size_t bytes
)
2725 directory_dump_request_log(void)
2727 return tor_strdup("Not supported.");
2731 /** Decide whether a client would accept the consensus we have.
2733 * Clients can say they only want a consensus if it's signed by more
2734 * than half the authorities in a list. They pass this list in
2735 * the url as "...consensus/<b>fpr</b>+<b>fpr</b>+<b>fpr</b>".
2737 * <b>fpr</b> may be an abbreviated fingerprint, i.e. only a left substring
2738 * of the full authority identity digest. (Only strings of even length,
2739 * i.e. encodings of full bytes, are handled correctly. In the case
2740 * of an odd number of hex digits the last one is silently ignored.)
2742 * Returns 1 if more than half of the requested authorities signed the
2743 * consensus, 0 otherwise.
2746 client_likes_consensus(networkstatus_t
*v
, const char *want_url
)
2748 smartlist_t
*want_authorities
= smartlist_new();
2752 dir_split_resource_into_fingerprints(want_url
, want_authorities
, NULL
, 0);
2753 need_at_least
= smartlist_len(want_authorities
)/2+1;
2754 SMARTLIST_FOREACH_BEGIN(want_authorities
, const char *, d
) {
2755 char want_digest
[DIGEST_LEN
];
2756 size_t want_len
= strlen(d
)/2;
2757 if (want_len
> DIGEST_LEN
)
2758 want_len
= DIGEST_LEN
;
2760 if (base16_decode(want_digest
, DIGEST_LEN
, d
, want_len
*2) < 0) {
2761 log_fn(LOG_PROTOCOL_WARN
, LD_DIR
,
2762 "Failed to decode requested authority digest %s.", escaped(d
));
2766 SMARTLIST_FOREACH_BEGIN(v
->voters
, networkstatus_voter_info_t
*, vi
) {
2767 if (smartlist_len(vi
->sigs
) &&
2768 tor_memeq(vi
->identity_digest
, want_digest
, want_len
)) {
2772 } SMARTLIST_FOREACH_END(vi
);
2774 /* early exit, if we already have enough */
2775 if (have
>= need_at_least
)
2777 } SMARTLIST_FOREACH_END(d
);
2779 SMARTLIST_FOREACH(want_authorities
, char *, d
, tor_free(d
));
2780 smartlist_free(want_authorities
);
2781 return (have
>= need_at_least
);
2784 /** Return the compression level we should use for sending a compressed
2785 * response of size <b>n_bytes</b>. */
2786 STATIC zlib_compression_level_t
2787 choose_compression_level(ssize_t n_bytes
)
2789 if (! have_been_under_memory_pressure()) {
2790 return HIGH_COMPRESSION
; /* we have plenty of RAM. */
2791 } else if (n_bytes
< 0) {
2792 return HIGH_COMPRESSION
; /* unknown; might be big. */
2793 } else if (n_bytes
< 1024) {
2794 return LOW_COMPRESSION
;
2795 } else if (n_bytes
< 2048) {
2796 return MEDIUM_COMPRESSION
;
2798 return HIGH_COMPRESSION
;
2802 /** Helper function: called when a dirserver gets a complete HTTP GET
2803 * request. Look for a request for a directory or for a rendezvous
2804 * service descriptor. On finding one, write a response into
2805 * conn-\>outbuf. If the request is unrecognized, send a 400.
2806 * Always return 0. */
2808 directory_handle_command_get(dir_connection_t
*conn
, const char *headers
,
2809 const char *req_body
, size_t req_body_len
)
2812 char *url
, *url_mem
, *header
;
2813 const or_options_t
*options
= get_options();
2814 time_t if_modified_since
= 0;
2818 /* We ignore the body of a GET request. */
2822 log_debug(LD_DIRSERV
,"Received GET command.");
2824 conn
->base_
.state
= DIR_CONN_STATE_SERVER_WRITING
;
2826 if (parse_http_url(headers
, &url
) < 0) {
2827 write_http_status_line(conn
, 400, "Bad request");
2830 if ((header
= http_get_header(headers
, "If-Modified-Since: "))) {
2832 if (parse_http_time(header
, &tm
) == 0) {
2833 if (tor_timegm(&tm
, &if_modified_since
)<0) {
2834 if_modified_since
= 0;
2836 log_debug(LD_DIRSERV
, "If-Modified-Since is '%s'.", escaped(header
));
2839 /* The correct behavior on a malformed If-Modified-Since header is to
2840 * act as if no If-Modified-Since header had been given. */
2843 log_debug(LD_DIRSERV
,"rewritten url as '%s'.", escaped(url
));
2846 url_len
= strlen(url
);
2847 compressed
= url_len
> 2 && !strcmp(url
+url_len
-2, ".z");
2849 url
[url_len
-2] = '\0';
2853 if (!strcmp(url
,"/tor/")) {
2854 const char *frontpage
= get_dirportfrontpage();
2857 dlen
= strlen(frontpage
);
2858 /* Let's return a disclaimer page (users shouldn't use V1 anymore,
2859 and caches don't fetch '/', so this is safe). */
2861 /* [We don't check for write_bucket_low here, since we want to serve
2862 * this page no matter what.] */
2863 note_request(url
, dlen
);
2864 write_http_response_header_impl(conn
, dlen
, "text/html", "identity",
2865 NULL
, DIRPORTFRONTPAGE_CACHE_LIFETIME
);
2866 connection_write_to_buf(frontpage
, dlen
, TO_CONN(conn
));
2869 /* if no disclaimer file, fall through and continue */
2872 if (!strcmpstart(url
, "/tor/status-vote/current/consensus")) {
2873 /* v3 network status fetch. */
2874 smartlist_t
*dir_fps
= smartlist_new();
2875 const char *request_type
= NULL
;
2876 long lifetime
= NETWORKSTATUS_CACHE_LIFETIME
;
2880 time_t now
= time(NULL
);
2881 const char *want_fps
= NULL
;
2882 char *flavor
= NULL
;
2884 #define CONSENSUS_URL_PREFIX "/tor/status-vote/current/consensus/"
2885 #define CONSENSUS_FLAVORED_PREFIX "/tor/status-vote/current/consensus-"
2886 /* figure out the flavor if any, and who we wanted to sign the thing */
2887 if (!strcmpstart(url
, CONSENSUS_FLAVORED_PREFIX
)) {
2889 f
= url
+ strlen(CONSENSUS_FLAVORED_PREFIX
);
2890 cp
= strchr(f
, '/');
2893 flavor
= tor_strndup(f
, cp
-f
);
2895 flavor
= tor_strdup(f
);
2897 flav
= networkstatus_parse_flavor_name(flavor
);
2901 if (!strcmpstart(url
, CONSENSUS_URL_PREFIX
))
2902 want_fps
= url
+strlen(CONSENSUS_URL_PREFIX
);
2905 v
= networkstatus_get_latest_consensus_by_flavor(flav
);
2907 if (v
&& want_fps
&&
2908 !client_likes_consensus(v
, want_fps
)) {
2909 write_http_status_line(conn
, 404, "Consensus not signed by sufficient "
2910 "number of requested authorities");
2911 smartlist_free(dir_fps
);
2912 geoip_note_ns_response(GEOIP_REJECT_NOT_ENOUGH_SIGS
);
2918 char *fp
= tor_malloc_zero(DIGEST_LEN
);
2920 strlcpy(fp
, flavor
, DIGEST_LEN
);
2922 smartlist_add(dir_fps
, fp
);
2924 request_type
= compressed
?"v3.z":"v3";
2925 lifetime
= (v
&& v
->fresh_until
> now
) ? v
->fresh_until
- now
: 0;
2928 if (!smartlist_len(dir_fps
)) { /* we failed to create/cache cp */
2929 write_http_status_line(conn
, 503, "Network status object unavailable");
2930 smartlist_free(dir_fps
);
2931 geoip_note_ns_response(GEOIP_REJECT_UNAVAILABLE
);
2935 if (!dirserv_remove_old_statuses(dir_fps
, if_modified_since
)) {
2936 write_http_status_line(conn
, 404, "Not found");
2937 SMARTLIST_FOREACH(dir_fps
, char *, cp
, tor_free(cp
));
2938 smartlist_free(dir_fps
);
2939 geoip_note_ns_response(GEOIP_REJECT_NOT_FOUND
);
2941 } else if (!smartlist_len(dir_fps
)) {
2942 write_http_status_line(conn
, 304, "Not modified");
2943 SMARTLIST_FOREACH(dir_fps
, char *, cp
, tor_free(cp
));
2944 smartlist_free(dir_fps
);
2945 geoip_note_ns_response(GEOIP_REJECT_NOT_MODIFIED
);
2949 dlen
= dirserv_estimate_data_size(dir_fps
, 0, compressed
);
2950 if (global_write_bucket_low(TO_CONN(conn
), dlen
, 2)) {
2951 log_debug(LD_DIRSERV
,
2952 "Client asked for network status lists, but we've been "
2953 "writing too many bytes lately. Sending 503 Dir busy.");
2954 write_http_status_line(conn
, 503, "Directory busy, try again later");
2955 SMARTLIST_FOREACH(dir_fps
, char *, fp
, tor_free(fp
));
2956 smartlist_free(dir_fps
);
2958 geoip_note_ns_response(GEOIP_REJECT_BUSY
);
2964 if (tor_addr_parse(&addr
, (TO_CONN(conn
))->address
) >= 0) {
2965 geoip_note_client_seen(GEOIP_CLIENT_NETWORKSTATUS
,
2968 geoip_note_ns_response(GEOIP_SUCCESS
);
2969 /* Note that a request for a network status has started, so that we
2970 * can measure the download time later on. */
2971 if (conn
->dirreq_id
)
2972 geoip_start_dirreq(conn
->dirreq_id
, dlen
, DIRREQ_TUNNELED
);
2974 geoip_start_dirreq(TO_CONN(conn
)->global_identifier
, dlen
,
2979 // note_request(request_type,dlen);
2980 (void) request_type
;
2981 write_http_response_header(conn
, -1, compressed
,
2982 smartlist_len(dir_fps
) == 1 ? lifetime
: 0);
2983 conn
->fingerprint_stack
= dir_fps
;
2985 conn
->zlib_state
= tor_zlib_new(0, ZLIB_METHOD
, HIGH_COMPRESSION
);
2987 /* Prime the connection with some data. */
2988 conn
->dir_spool_src
= DIR_SPOOL_NETWORKSTATUS
;
2989 connection_dirserv_flushed_some(conn
);
2993 if (!strcmpstart(url
,"/tor/status-vote/current/") ||
2994 !strcmpstart(url
,"/tor/status-vote/next/")) {
2995 /* XXXX If-modified-since is only implemented for the current
2996 * consensus: that's probably fine, since it's the only vote document
2997 * people fetch much. */
2999 ssize_t body_len
= 0;
3000 ssize_t estimated_len
= 0;
3001 smartlist_t
*items
= smartlist_new();
3002 smartlist_t
*dir_items
= smartlist_new();
3003 int lifetime
= 60; /* XXXX023 should actually use vote intervals. */
3004 url
+= strlen("/tor/status-vote/");
3005 current
= !strcmpstart(url
, "current/");
3006 url
= strchr(url
, '/');
3009 if (!strcmp(url
, "consensus")) {
3011 tor_assert(!current
); /* we handle current consensus specially above,
3012 * since it wants to be spooled. */
3013 if ((item
= dirvote_get_pending_consensus(FLAV_NS
)))
3014 smartlist_add(items
, (char*)item
);
3015 } else if (!current
&& !strcmp(url
, "consensus-signatures")) {
3016 /* XXXX the spec says that we should implement
3017 * current/consensus-signatures too. It doesn't seem to be needed,
3020 if ((item
=dirvote_get_pending_detached_signatures()))
3021 smartlist_add(items
, (char*)item
);
3022 } else if (!strcmp(url
, "authority")) {
3023 const cached_dir_t
*d
;
3024 int flags
= DGV_BY_ID
|
3025 (current
? DGV_INCLUDE_PREVIOUS
: DGV_INCLUDE_PENDING
);
3026 if ((d
=dirvote_get_vote(NULL
, flags
)))
3027 smartlist_add(dir_items
, (cached_dir_t
*)d
);
3029 const cached_dir_t
*d
;
3030 smartlist_t
*fps
= smartlist_new();
3032 if (!strcmpstart(url
, "d/")) {
3034 flags
= DGV_INCLUDE_PENDING
| DGV_INCLUDE_PREVIOUS
;
3037 (current
? DGV_INCLUDE_PREVIOUS
: DGV_INCLUDE_PENDING
);
3039 dir_split_resource_into_fingerprints(url
, fps
, NULL
,
3040 DSR_HEX
|DSR_SORT_UNIQ
);
3041 SMARTLIST_FOREACH(fps
, char *, fp
, {
3042 if ((d
= dirvote_get_vote(fp
, flags
)))
3043 smartlist_add(dir_items
, (cached_dir_t
*)d
);
3046 smartlist_free(fps
);
3048 if (!smartlist_len(dir_items
) && !smartlist_len(items
)) {
3049 write_http_status_line(conn
, 404, "Not found");
3052 SMARTLIST_FOREACH(dir_items
, cached_dir_t
*, d
,
3053 body_len
+= compressed
? d
->dir_z_len
: d
->dir_len
);
3054 estimated_len
+= body_len
;
3055 SMARTLIST_FOREACH(items
, const char *, item
, {
3056 size_t ln
= strlen(item
);
3058 estimated_len
+= ln
/2;
3060 body_len
+= ln
; estimated_len
+= ln
;
3064 if (global_write_bucket_low(TO_CONN(conn
), estimated_len
, 2)) {
3065 write_http_status_line(conn
, 503, "Directory busy, try again later");
3068 write_http_response_header(conn
, body_len
? body_len
: -1, compressed
,
3071 if (smartlist_len(items
)) {
3073 conn
->zlib_state
= tor_zlib_new(1, ZLIB_METHOD
,
3074 choose_compression_level(estimated_len
));
3075 SMARTLIST_FOREACH(items
, const char *, c
,
3076 connection_write_to_buf_zlib(c
, strlen(c
), conn
, 0));
3077 connection_write_to_buf_zlib("", 0, conn
, 1);
3079 SMARTLIST_FOREACH(items
, const char *, c
,
3080 connection_write_to_buf(c
, strlen(c
), TO_CONN(conn
)));
3083 SMARTLIST_FOREACH(dir_items
, cached_dir_t
*, d
,
3084 connection_write_to_buf(compressed
? d
->dir_z
: d
->dir
,
3085 compressed
? d
->dir_z_len
: d
->dir_len
,
3089 smartlist_free(items
);
3090 smartlist_free(dir_items
);
3094 if (!strcmpstart(url
, "/tor/micro/d/")) {
3095 smartlist_t
*fps
= smartlist_new();
3097 dir_split_resource_into_fingerprints(url
+strlen("/tor/micro/d/"),
3099 DSR_DIGEST256
|DSR_BASE64
|DSR_SORT_UNIQ
);
3101 if (!dirserv_have_any_microdesc(fps
)) {
3102 write_http_status_line(conn
, 404, "Not found");
3103 SMARTLIST_FOREACH(fps
, char *, fp
, tor_free(fp
));
3104 smartlist_free(fps
);
3107 dlen
= dirserv_estimate_microdesc_size(fps
, compressed
);
3108 if (global_write_bucket_low(TO_CONN(conn
), dlen
, 2)) {
3109 log_info(LD_DIRSERV
,
3110 "Client asked for server descriptors, but we've been "
3111 "writing too many bytes lately. Sending 503 Dir busy.");
3112 write_http_status_line(conn
, 503, "Directory busy, try again later");
3113 SMARTLIST_FOREACH(fps
, char *, fp
, tor_free(fp
));
3114 smartlist_free(fps
);
3118 write_http_response_header(conn
, -1, compressed
, MICRODESC_CACHE_LIFETIME
);
3119 conn
->dir_spool_src
= DIR_SPOOL_MICRODESC
;
3120 conn
->fingerprint_stack
= fps
;
3123 conn
->zlib_state
= tor_zlib_new(1, ZLIB_METHOD
,
3124 choose_compression_level(dlen
));
3126 connection_dirserv_flushed_some(conn
);
3130 if (!strcmpstart(url
,"/tor/server/") ||
3131 (!options
->BridgeAuthoritativeDir
&&
3132 !options
->BridgeRelay
&& !strcmpstart(url
,"/tor/extra/"))) {
3135 const char *request_type
= NULL
;
3136 int cache_lifetime
= 0;
3137 int is_extra
= !strcmpstart(url
,"/tor/extra/");
3138 url
+= is_extra
? strlen("/tor/extra/") : strlen("/tor/server/");
3139 conn
->fingerprint_stack
= smartlist_new();
3140 res
= dirserv_get_routerdesc_fingerprints(conn
->fingerprint_stack
, url
,
3142 !connection_dir_is_encrypted(conn
),
3145 if (!strcmpstart(url
, "fp/")) {
3146 request_type
= compressed
?"/tor/server/fp.z":"/tor/server/fp";
3147 if (smartlist_len(conn
->fingerprint_stack
) == 1)
3148 cache_lifetime
= ROUTERDESC_CACHE_LIFETIME
;
3149 } else if (!strcmpstart(url
, "authority")) {
3150 request_type
= compressed
?"/tor/server/authority.z":
3151 "/tor/server/authority";
3152 cache_lifetime
= ROUTERDESC_CACHE_LIFETIME
;
3153 } else if (!strcmpstart(url
, "all")) {
3154 request_type
= compressed
?"/tor/server/all.z":"/tor/server/all";
3155 cache_lifetime
= FULL_DIR_CACHE_LIFETIME
;
3156 } else if (!strcmpstart(url
, "d/")) {
3157 request_type
= compressed
?"/tor/server/d.z":"/tor/server/d";
3158 if (smartlist_len(conn
->fingerprint_stack
) == 1)
3159 cache_lifetime
= ROUTERDESC_BY_DIGEST_CACHE_LIFETIME
;
3161 request_type
= "/tor/server/?";
3163 (void) request_type
; /* usable for note_request. */
3164 if (!strcmpstart(url
, "d/"))
3165 conn
->dir_spool_src
=
3166 is_extra
? DIR_SPOOL_EXTRA_BY_DIGEST
: DIR_SPOOL_SERVER_BY_DIGEST
;
3168 conn
->dir_spool_src
=
3169 is_extra
? DIR_SPOOL_EXTRA_BY_FP
: DIR_SPOOL_SERVER_BY_FP
;
3171 if (!dirserv_have_any_serverdesc(conn
->fingerprint_stack
,
3172 conn
->dir_spool_src
)) {
3178 write_http_status_line(conn
, 404, msg
);
3180 dlen
= dirserv_estimate_data_size(conn
->fingerprint_stack
,
3182 if (global_write_bucket_low(TO_CONN(conn
), dlen
, 2)) {
3183 log_info(LD_DIRSERV
,
3184 "Client asked for server descriptors, but we've been "
3185 "writing too many bytes lately. Sending 503 Dir busy.");
3186 write_http_status_line(conn
, 503, "Directory busy, try again later");
3187 conn
->dir_spool_src
= DIR_SPOOL_NONE
;
3190 write_http_response_header(conn
, -1, compressed
, cache_lifetime
);
3192 conn
->zlib_state
= tor_zlib_new(1, ZLIB_METHOD
,
3193 choose_compression_level(dlen
));
3194 /* Prime the connection with some data. */
3195 connection_dirserv_flushed_some(conn
);
3200 if (!strcmpstart(url
,"/tor/keys/")) {
3201 smartlist_t
*certs
= smartlist_new();
3203 if (!strcmp(url
, "/tor/keys/all")) {
3204 authority_cert_get_all(certs
);
3205 } else if (!strcmp(url
, "/tor/keys/authority")) {
3206 authority_cert_t
*cert
= get_my_v3_authority_cert();
3208 smartlist_add(certs
, cert
);
3209 } else if (!strcmpstart(url
, "/tor/keys/fp/")) {
3210 smartlist_t
*fps
= smartlist_new();
3211 dir_split_resource_into_fingerprints(url
+strlen("/tor/keys/fp/"),
3213 DSR_HEX
|DSR_SORT_UNIQ
);
3214 SMARTLIST_FOREACH(fps
, char *, d
, {
3215 authority_cert_t
*c
= authority_cert_get_newest_by_id(d
);
3216 if (c
) smartlist_add(certs
, c
);
3219 smartlist_free(fps
);
3220 } else if (!strcmpstart(url
, "/tor/keys/sk/")) {
3221 smartlist_t
*fps
= smartlist_new();
3222 dir_split_resource_into_fingerprints(url
+strlen("/tor/keys/sk/"),
3224 DSR_HEX
|DSR_SORT_UNIQ
);
3225 SMARTLIST_FOREACH(fps
, char *, d
, {
3226 authority_cert_t
*c
= authority_cert_get_by_sk_digest(d
);
3227 if (c
) smartlist_add(certs
, c
);
3230 smartlist_free(fps
);
3231 } else if (!strcmpstart(url
, "/tor/keys/fp-sk/")) {
3232 smartlist_t
*fp_sks
= smartlist_new();
3233 dir_split_resource_into_fingerprint_pairs(url
+strlen("/tor/keys/fp-sk/"),
3235 SMARTLIST_FOREACH(fp_sks
, fp_pair_t
*, pair
, {
3236 authority_cert_t
*c
= authority_cert_get_by_digests(pair
->first
,
3238 if (c
) smartlist_add(certs
, c
);
3241 smartlist_free(fp_sks
);
3243 write_http_status_line(conn
, 400, "Bad request");
3246 if (!smartlist_len(certs
)) {
3247 write_http_status_line(conn
, 404, "Not found");
3250 SMARTLIST_FOREACH(certs
, authority_cert_t
*, c
,
3251 if (c
->cache_info
.published_on
< if_modified_since
)
3252 SMARTLIST_DEL_CURRENT(certs
, c
));
3253 if (!smartlist_len(certs
)) {
3254 write_http_status_line(conn
, 304, "Not modified");
3258 SMARTLIST_FOREACH(certs
, authority_cert_t
*, c
,
3259 len
+= c
->cache_info
.signed_descriptor_len
);
3261 if (global_write_bucket_low(TO_CONN(conn
), compressed
?len
/2:len
, 2)) {
3262 write_http_status_line(conn
, 503, "Directory busy, try again later");
3266 write_http_response_header(conn
, compressed
?-1:len
, compressed
, 60*60);
3268 conn
->zlib_state
= tor_zlib_new(1, ZLIB_METHOD
,
3269 choose_compression_level(len
));
3270 SMARTLIST_FOREACH(certs
, authority_cert_t
*, c
,
3271 connection_write_to_buf_zlib(c
->cache_info
.signed_descriptor_body
,
3272 c
->cache_info
.signed_descriptor_len
,
3274 connection_write_to_buf_zlib("", 0, conn
, 1);
3276 SMARTLIST_FOREACH(certs
, authority_cert_t
*, c
,
3277 connection_write_to_buf(c
->cache_info
.signed_descriptor_body
,
3278 c
->cache_info
.signed_descriptor_len
,
3282 smartlist_free(certs
);
3286 if (connection_dir_is_encrypted(conn
) &&
3287 !strcmpstart(url
,"/tor/rendezvous2/")) {
3288 /* Handle v2 rendezvous descriptor fetch request. */
3290 const char *query
= url
+ strlen("/tor/rendezvous2/");
3291 if (rend_valid_descriptor_id(query
)) {
3292 log_info(LD_REND
, "Got a v2 rendezvous descriptor request for ID '%s'",
3293 safe_str(escaped(query
)));
3294 switch (rend_cache_lookup_v2_desc_as_dir(query
, &descp
)) {
3296 write_http_response_header(conn
, strlen(descp
), 0, 0);
3297 connection_write_to_buf(descp
, strlen(descp
), TO_CONN(conn
));
3299 case 0: /* well-formed but not present */
3300 write_http_status_line(conn
, 404, "Not found");
3302 case -1: /* not well-formed */
3303 write_http_status_line(conn
, 400, "Bad request");
3306 } else { /* not well-formed */
3307 write_http_status_line(conn
, 400, "Bad request");
3312 if (options
->BridgeAuthoritativeDir
&&
3313 options
->BridgePassword_AuthDigest_
&&
3314 connection_dir_is_encrypted(conn
) &&
3315 !strcmp(url
,"/tor/networkstatus-bridges")) {
3317 char digest
[DIGEST256_LEN
];
3319 header
= http_get_header(headers
, "Authorization: Basic ");
3321 crypto_digest256(digest
, header
, strlen(header
), DIGEST_SHA256
);
3323 /* now make sure the password is there and right */
3326 options
->BridgePassword_AuthDigest_
, DIGEST256_LEN
)) {
3327 write_http_status_line(conn
, 404, "Not found");
3333 /* all happy now. send an answer. */
3334 status
= networkstatus_getinfo_by_purpose("bridge", time(NULL
));
3335 dlen
= strlen(status
);
3336 write_http_response_header(conn
, dlen
, 0, 0);
3337 connection_write_to_buf(status
, dlen
, TO_CONN(conn
));
3342 if (!strcmpstart(url
,"/tor/bytes.txt")) {
3343 char *bytes
= directory_dump_request_log();
3344 size_t len
= strlen(bytes
);
3345 write_http_response_header(conn
, len
, 0, 0);
3346 connection_write_to_buf(bytes
, len
, TO_CONN(conn
));
3351 if (!strcmp(url
,"/tor/robots.txt")) { /* /robots.txt will have been
3352 rewritten to /tor/robots.txt */
3353 char robots
[] = "User-agent: *\r\nDisallow: /\r\n";
3354 size_t len
= strlen(robots
);
3355 write_http_response_header(conn
, len
, 0, ROBOTS_CACHE_LIFETIME
);
3356 connection_write_to_buf(robots
, len
, TO_CONN(conn
));
3360 #if defined(EXPORTMALLINFO) && defined(HAVE_MALLOC_H) && defined(HAVE_MALLINFO)
3361 #define ADD_MALLINFO_LINE(x) do { \
3362 smartlist_add_asprintf(lines, "%s %d\n", #x, mi.x); \
3365 if (!strcmp(url
,"/tor/mallinfo.txt") &&
3366 (tor_addr_eq_ipv4h(&conn
->base_
.addr
, 0x7f000001ul
))) {
3372 memset(&mi
, 0, sizeof(mi
));
3374 lines
= smartlist_new();
3376 ADD_MALLINFO_LINE(arena
)
3377 ADD_MALLINFO_LINE(ordblks
)
3378 ADD_MALLINFO_LINE(smblks
)
3379 ADD_MALLINFO_LINE(hblks
)
3380 ADD_MALLINFO_LINE(hblkhd
)
3381 ADD_MALLINFO_LINE(usmblks
)
3382 ADD_MALLINFO_LINE(fsmblks
)
3383 ADD_MALLINFO_LINE(uordblks
)
3384 ADD_MALLINFO_LINE(fordblks
)
3385 ADD_MALLINFO_LINE(keepcost
)
3387 result
= smartlist_join_strings(lines
, "", 0, NULL
);
3388 SMARTLIST_FOREACH(lines
, char *, cp
, tor_free(cp
));
3389 smartlist_free(lines
);
3391 len
= strlen(result
);
3392 write_http_response_header(conn
, len
, 0, 0);
3393 connection_write_to_buf(result
, len
, TO_CONN(conn
));
3399 /* we didn't recognize the url */
3400 write_http_status_line(conn
, 404, "Not found");
3407 /** Helper function: called when a dirserver gets a complete HTTP POST
3408 * request. Look for an uploaded server descriptor or rendezvous
3409 * service descriptor. On finding one, process it and write a
3410 * response into conn-\>outbuf. If the request is unrecognized, send a
3411 * 400. Always return 0. */
3413 directory_handle_command_post(dir_connection_t
*conn
, const char *headers
,
3414 const char *body
, size_t body_len
)
3417 const or_options_t
*options
= get_options();
3419 log_debug(LD_DIRSERV
,"Received POST command.");
3421 conn
->base_
.state
= DIR_CONN_STATE_SERVER_WRITING
;
3423 if (!public_server_mode(options
)) {
3424 log_info(LD_DIR
, "Rejected dir post request from %s "
3425 "since we're not a public relay.", conn
->base_
.address
);
3426 write_http_status_line(conn
, 503, "Not acting as a public relay");
3430 if (parse_http_url(headers
, &url
) < 0) {
3431 write_http_status_line(conn
, 400, "Bad request");
3434 log_debug(LD_DIRSERV
,"rewritten url as '%s'.", escaped(url
));
3436 /* Handle v2 rendezvous service publish request. */
3437 if (connection_dir_is_encrypted(conn
) &&
3438 !strcmpstart(url
,"/tor/rendezvous2/publish")) {
3439 if (rend_cache_store_v2_desc_as_dir(body
) < 0) {
3440 log_warn(LD_REND
, "Rejected v2 rend descriptor (length %d) from %s.",
3441 (int)body_len
, conn
->base_
.address
);
3442 write_http_status_line(conn
, 400,
3443 "Invalid v2 service descriptor rejected");
3445 write_http_status_line(conn
, 200, "Service descriptor (v2) stored");
3446 log_info(LD_REND
, "Handled v2 rendezvous descriptor post: accepted");
3451 if (!authdir_mode(options
)) {
3452 /* we just provide cached directories; we don't want to
3453 * receive anything. */
3454 write_http_status_line(conn
, 400, "Nonauthoritative directory does not "
3455 "accept posted server descriptors");
3459 if (authdir_mode_handles_descs(options
, -1) &&
3460 !strcmp(url
,"/tor/")) { /* server descriptor post */
3461 const char *msg
= "[None]";
3462 uint8_t purpose
= authdir_mode_bridge(options
) ?
3463 ROUTER_PURPOSE_BRIDGE
: ROUTER_PURPOSE_GENERAL
;
3464 was_router_added_t r
= dirserv_add_multiple_descriptors(body
, purpose
,
3465 conn
->base_
.address
, &msg
);
3468 if (r
== ROUTER_ADDED_NOTIFY_GENERATOR
) {
3469 /* Accepted with a message. */
3470 log_info(LD_DIRSERV
,
3471 "Problematic router descriptor or extra-info from %s "
3473 conn
->base_
.address
, msg
);
3474 write_http_status_line(conn
, 400, msg
);
3475 } else if (r
== ROUTER_ADDED_SUCCESSFULLY
) {
3476 write_http_status_line(conn
, 200, msg
);
3477 } else if (WRA_WAS_OUTDATED(r
)) {
3478 write_http_response_header_impl(conn
, -1, NULL
, NULL
,
3479 "X-Descriptor-Not-New: Yes\r\n", -1);
3481 log_info(LD_DIRSERV
,
3482 "Rejected router descriptor or extra-info from %s "
3484 conn
->base_
.address
, msg
);
3485 write_http_status_line(conn
, 400, msg
);
3490 if (authdir_mode_v3(options
) &&
3491 !strcmp(url
,"/tor/post/vote")) { /* v3 networkstatus vote */
3492 const char *msg
= "OK";
3494 if (dirvote_add_vote(body
, &msg
, &status
)) {
3495 write_http_status_line(conn
, status
, "Vote stored");
3498 log_warn(LD_DIRSERV
, "Rejected vote from %s (\"%s\").",
3499 conn
->base_
.address
, msg
);
3500 write_http_status_line(conn
, status
, msg
);
3505 if (authdir_mode_v3(options
) &&
3506 !strcmp(url
,"/tor/post/consensus-signature")) { /* sigs on consensus. */
3507 const char *msg
= NULL
;
3508 if (dirvote_add_signatures(body
, conn
->base_
.address
, &msg
)>=0) {
3509 write_http_status_line(conn
, 200, msg
?msg
:"Signatures stored");
3511 log_warn(LD_DIR
, "Unable to store signatures posted by %s: %s",
3512 conn
->base_
.address
, msg
?msg
:"???");
3513 write_http_status_line(conn
, 400, msg
?msg
:"Unable to store signatures");
3518 /* we didn't recognize the url */
3519 write_http_status_line(conn
, 404, "Not found");
3526 /** Called when a dirserver receives data on a directory connection;
3527 * looks for an HTTP request. If the request is complete, remove it
3528 * from the inbuf, try to process it; otherwise, leave it on the
3529 * buffer. Return a 0 on success, or -1 on error.
3532 directory_handle_command(dir_connection_t
*conn
)
3534 char *headers
=NULL
, *body
=NULL
;
3539 tor_assert(conn
->base_
.type
== CONN_TYPE_DIR
);
3541 switch (connection_fetch_from_buf_http(TO_CONN(conn
),
3542 &headers
, MAX_HEADERS_SIZE
,
3543 &body
, &body_len
, MAX_DIR_UL_SIZE
, 0)) {
3544 case -1: /* overflow */
3545 log_warn(LD_DIRSERV
,
3546 "Request too large from address '%s' to DirPort. Closing.",
3547 safe_str(conn
->base_
.address
));
3550 log_debug(LD_DIRSERV
,"command not all here yet.");
3552 /* case 1, fall through */
3555 http_set_address_origin(headers
, TO_CONN(conn
));
3556 // we should escape headers here as well,
3557 // but we can't call escaped() twice, as it uses the same buffer
3558 //log_debug(LD_DIRSERV,"headers %s, body %s.", headers, escaped(body));
3560 if (!strncasecmp(headers
,"GET",3))
3561 r
= directory_handle_command_get(conn
, headers
, body
, body_len
);
3562 else if (!strncasecmp(headers
,"POST",4))
3563 r
= directory_handle_command_post(conn
, headers
, body
, body_len
);
3565 log_fn(LOG_PROTOCOL_WARN
, LD_PROTOCOL
,
3566 "Got headers %s with unknown command. Closing.",
3571 tor_free(headers
); tor_free(body
);
3575 /** Write handler for directory connections; called when all data has
3576 * been flushed. Close the connection or wait for a response as
3580 connection_dir_finished_flushing(dir_connection_t
*conn
)
3583 tor_assert(conn
->base_
.type
== CONN_TYPE_DIR
);
3585 /* Note that we have finished writing the directory response. For direct
3586 * connections this means we're done; for tunneled connections it's only
3587 * an intermediate step. */
3588 if (conn
->dirreq_id
)
3589 geoip_change_dirreq_state(conn
->dirreq_id
, DIRREQ_TUNNELED
,
3590 DIRREQ_FLUSHING_DIR_CONN_FINISHED
);
3592 geoip_change_dirreq_state(TO_CONN(conn
)->global_identifier
,
3594 DIRREQ_FLUSHING_DIR_CONN_FINISHED
);
3595 switch (conn
->base_
.state
) {
3596 case DIR_CONN_STATE_CONNECTING
:
3597 case DIR_CONN_STATE_CLIENT_SENDING
:
3598 log_debug(LD_DIR
,"client finished sending command.");
3599 conn
->base_
.state
= DIR_CONN_STATE_CLIENT_READING
;
3601 case DIR_CONN_STATE_SERVER_WRITING
:
3602 if (conn
->dir_spool_src
!= DIR_SPOOL_NONE
) {
3603 #ifdef USE_BUFFEREVENTS
3604 /* This can happen with paired bufferevents, since a paired connection
3605 * can flush immediately when you write to it, making the subsequent
3606 * check in connection_handle_write_cb() decide that the connection
3608 log_debug(LD_DIRSERV
, "Emptied a dirserv buffer, but still spooling.");
3610 log_warn(LD_BUG
, "Emptied a dirserv buffer, but it's still spooling!");
3611 connection_mark_for_close(TO_CONN(conn
));
3614 log_debug(LD_DIRSERV
, "Finished writing server response. Closing.");
3615 connection_mark_for_close(TO_CONN(conn
));
3619 log_warn(LD_BUG
,"called in unexpected state %d.",
3621 tor_fragile_assert();
3627 /* A helper function for connection_dir_close_consensus_conn_if_extra()
3628 * and connection_dir_close_extra_consensus_conns() that returns 0 if
3629 * we can't have, or don't want to close, excess consensus connections. */
3631 connection_dir_would_close_consensus_conn_helper(void)
3633 const or_options_t
*options
= get_options();
3635 /* we're only interested in closing excess connections if we could
3636 * have created any in the first place */
3637 if (!networkstatus_consensus_can_use_multiple_directories(options
)) {
3641 /* We want to close excess connections downloading a consensus.
3642 * If there aren't any excess, we don't have anything to close. */
3643 if (!networkstatus_consensus_has_excess_connections()) {
3647 /* If we have excess connections, but none of them are downloading a
3648 * consensus, and we are still bootstrapping (that is, we have no usable
3649 * consensus), we don't want to close any until one starts downloading. */
3650 if (!networkstatus_consensus_is_downloading_usable_flavor()
3651 && networkstatus_consensus_is_boostrapping(time(NULL
))) {
3655 /* If we have just stopped bootstrapping (that is, just parsed a consensus),
3656 * we might still have some excess connections hanging around. So we still
3657 * have to check if we want to close any, even if we've stopped
3662 /* Check if we would close excess consensus connections. If we would, any
3663 * new consensus connection would become excess immediately, so return 1.
3664 * Otherwise, return 0. */
3666 connection_dir_avoid_extra_connection_for_purpose(unsigned int purpose
)
3668 const or_options_t
*options
= get_options();
3670 /* We're not interested in connections that aren't fetching a consensus. */
3671 if (purpose
!= DIR_PURPOSE_FETCH_CONSENSUS
) {
3675 /* we're only interested in avoiding excess connections if we could
3676 * have created any in the first place */
3677 if (!networkstatus_consensus_can_use_multiple_directories(options
)) {
3681 /* If there are connections downloading a consensus, and we are still
3682 * bootstrapping (that is, we have no usable consensus), we can be sure that
3683 * any further connections would be excess. */
3684 if (networkstatus_consensus_is_downloading_usable_flavor()
3685 && networkstatus_consensus_is_boostrapping(time(NULL
))) {
3692 /* Check if we have more than one consensus download connection attempt, and
3694 * - if we don't have a consensus, and we're downloading a consensus, and conn
3695 * is not downloading a consensus yet;
3696 * - if we do have a consensus, and there's more than one consensus connection.
3698 * Post-bootstrap consensus connection attempts are initiated one at a time.
3699 * So this function won't close any consensus connection attempts that
3700 * are initiated after bootstrap.
3703 connection_dir_close_consensus_conn_if_extra(dir_connection_t
*conn
)
3706 tor_assert(conn
->base_
.type
== CONN_TYPE_DIR
);
3708 /* We're not interested in connections that aren't fetching a consensus. */
3709 if (conn
->base_
.purpose
!= DIR_PURPOSE_FETCH_CONSENSUS
) {
3713 /* The connection has already been closed */
3714 if (conn
->base_
.marked_for_close
) {
3718 /* Only close this connection if there's another consensus connection
3719 * downloading (during bootstrap), or connecting (after bootstrap).
3720 * Post-bootstrap consensus connection attempts won't be closed, because
3721 * they only occur one at a time. */
3722 if (!connection_dir_would_close_consensus_conn_helper()) {
3726 const int we_are_bootstrapping
= networkstatus_consensus_is_boostrapping(
3729 /* We don't want to check other connections to see if they are downloading,
3730 * as this is prone to race-conditions. So leave it for
3731 * connection_dir_consider_close_extra_consensus_conns() to clean up.
3733 * But if conn has just started connecting, or we have a consensus already,
3734 * we can be sure it's not needed any more. */
3735 if (!we_are_bootstrapping
3736 || conn
->base_
.state
== DIR_CONN_STATE_CONNECTING
) {
3737 connection_close_immediate(&conn
->base_
);
3738 connection_mark_for_close(&conn
->base_
);
3745 /* Clean up excess consensus download connection attempts.
3746 * During bootstrap, or when the bootstrap consensus has just been downloaded,
3747 * if we have more than one active consensus connection:
3748 * - if we don't have a consensus, and we're downloading a consensus, keep an
3749 * earlier connection, or a connection to a fallback directory, and close
3750 * all other connections;
3751 * - if we have just downloaded the bootstrap consensus, and have other
3752 * consensus connections left over, close all of them.
3754 * Post-bootstrap consensus connection attempts are initiated one at a time.
3755 * So this function won't close any consensus connection attempts that
3756 * are initiated after bootstrap.
3759 connection_dir_close_extra_consensus_conns(void)
3761 /* Only cleanup connections if there is more than one consensus connection,
3762 * and at least one of those connections is already downloading
3763 * (during bootstrap), or connecting (just after the bootstrap consensus is
3765 * Post-bootstrap consensus connection attempts won't be cleaned up, because
3766 * they only occur one at a time. */
3767 if (!connection_dir_would_close_consensus_conn_helper()) {
3771 int we_are_bootstrapping
= networkstatus_consensus_is_boostrapping(
3774 const char *usable_resource
= networkstatus_get_flavor_name(
3775 usable_consensus_flavor());
3776 smartlist_t
*consens_usable_conns
=
3777 connection_dir_list_by_purpose_and_resource(
3778 DIR_PURPOSE_FETCH_CONSENSUS
,
3781 /* If we want to keep a connection that's downloading, find a connection to
3783 * - connections opened earlier (they are likely to have progressed further)
3784 * - connections to fallbacks (to reduce the load on authorities) */
3785 dir_connection_t
*kept_download_conn
= NULL
;
3786 int kept_is_authority
= 0;
3787 if (we_are_bootstrapping
) {
3788 SMARTLIST_FOREACH_BEGIN(consens_usable_conns
,
3789 dir_connection_t
*, d
) {
3791 int d_is_authority
= router_digest_is_trusted_dir(d
->identity_digest
);
3792 /* keep the first connection that is past the connecting state, but
3793 * prefer fallbacks. */
3794 if (d
->base_
.state
!= DIR_CONN_STATE_CONNECTING
) {
3795 if (!kept_download_conn
|| (kept_is_authority
&& !d_is_authority
)) {
3796 kept_download_conn
= d
;
3797 kept_is_authority
= d_is_authority
;
3798 /* we've found the earliest fallback, and want to keep it regardless
3799 * of any other connections */
3800 if (!kept_is_authority
)
3804 } SMARTLIST_FOREACH_END(d
);
3807 SMARTLIST_FOREACH_BEGIN(consens_usable_conns
,
3808 dir_connection_t
*, d
) {
3810 /* don't close this connection if it's the one we want to keep */
3811 if (kept_download_conn
&& d
== kept_download_conn
)
3813 /* mark all other connections for close */
3814 if (!d
->base_
.marked_for_close
) {
3815 connection_close_immediate(&d
->base_
);
3816 connection_mark_for_close(&d
->base_
);
3818 } SMARTLIST_FOREACH_END(d
);
3820 smartlist_free(consens_usable_conns
);
3821 consens_usable_conns
= NULL
;
3823 /* make sure we've closed all excess connections */
3824 const int final_connecting_conn_count
=
3825 connection_dir_count_by_purpose_resource_and_state(
3826 DIR_PURPOSE_FETCH_CONSENSUS
,
3828 DIR_CONN_STATE_CONNECTING
);
3829 if (final_connecting_conn_count
> 0) {
3830 log_warn(LD_BUG
, "Expected 0 consensus connections connecting after "
3831 "cleanup, got %d.", final_connecting_conn_count
);
3833 const int expected_final_conn_count
= (we_are_bootstrapping
? 1 : 0);
3834 const int final_conn_count
=
3835 connection_dir_count_by_purpose_and_resource(
3836 DIR_PURPOSE_FETCH_CONSENSUS
,
3838 if (final_conn_count
> expected_final_conn_count
) {
3839 log_warn(LD_BUG
, "Expected %d consensus connections after cleanup, got "
3840 "%d.", expected_final_conn_count
, final_connecting_conn_count
);
3844 /** Connected handler for directory connections: begin sending data to the
3845 * server, and return 0, or, if the connection is an excess bootstrap
3846 * connection, close all excess bootstrap connections.
3847 * Only used when connections don't immediately connect. */
3849 connection_dir_finished_connecting(dir_connection_t
*conn
)
3852 tor_assert(conn
->base_
.type
== CONN_TYPE_DIR
);
3853 tor_assert(conn
->base_
.state
== DIR_CONN_STATE_CONNECTING
);
3855 log_debug(LD_HTTP
,"Dir connection to router %s:%u established.",
3856 conn
->base_
.address
,conn
->base_
.port
);
3858 /* Close this connection if there's another consensus connection
3859 * downloading (during bootstrap), or connecting (after bootstrap). */
3860 if (connection_dir_close_consensus_conn_if_extra(conn
)) {
3864 /* start flushing conn */
3865 conn
->base_
.state
= DIR_CONN_STATE_CLIENT_SENDING
;
3869 /** Decide which download schedule we want to use based on descriptor type
3870 * in <b>dls</b> and <b>options</b>.
3871 * Then return a list of int pointers defining download delays in seconds.
3872 * Helper function for download_status_increment_failure(),
3873 * download_status_reset(), and download_status_increment_attempt(). */
3874 STATIC
const smartlist_t
*
3875 find_dl_schedule(download_status_t
*dls
, const or_options_t
*options
)
3877 const int dir_server
= dir_server_mode(options
);
3878 const int multi_d
= networkstatus_consensus_can_use_multiple_directories(
3880 const int we_are_bootstrapping
= networkstatus_consensus_is_boostrapping(
3882 const int use_fallbacks
= networkstatus_consensus_can_use_extra_fallbacks(
3884 switch (dls
->schedule
) {
3885 case DL_SCHED_GENERIC
:
3887 return options
->TestingServerDownloadSchedule
;
3889 return options
->TestingClientDownloadSchedule
;
3891 case DL_SCHED_CONSENSUS
:
3893 return options
->TestingServerConsensusDownloadSchedule
;
3895 if (we_are_bootstrapping
) {
3896 if (!use_fallbacks
) {
3897 /* A bootstrapping client without extra fallback directories */
3899 options
->TestingClientBootstrapConsensusAuthorityOnlyDownloadSchedule
;
3900 } else if (dls
->want_authority
) {
3901 /* A bootstrapping client with extra fallback directories, but
3902 * connecting to an authority */
3904 options
->TestingClientBootstrapConsensusAuthorityDownloadSchedule
;
3906 /* A bootstrapping client connecting to extra fallback directories
3909 options
->TestingClientBootstrapConsensusFallbackDownloadSchedule
;
3912 return options
->TestingClientConsensusDownloadSchedule
;
3915 case DL_SCHED_BRIDGE
:
3916 return options
->TestingBridgeDownloadSchedule
;
3921 /* Impossible, but gcc will fail with -Werror without a `return`. */
3925 /* Find the current delay for dls based on schedule.
3926 * Set dls->next_attempt_at based on now, and return the delay.
3927 * Helper for download_status_increment_failure and
3928 * download_status_increment_attempt. */
3930 download_status_schedule_get_delay(download_status_t
*dls
,
3931 const smartlist_t
*schedule
,
3935 tor_assert(schedule
);
3937 int delay
= INT_MAX
;
3938 uint8_t dls_schedule_position
= (dls
->increment_on
3939 == DL_SCHED_INCREMENT_ATTEMPT
3940 ? dls
->n_download_attempts
3941 : dls
->n_download_failures
);
3943 if (dls_schedule_position
< smartlist_len(schedule
))
3944 delay
= *(int *)smartlist_get(schedule
, dls_schedule_position
);
3945 else if (dls_schedule_position
== IMPOSSIBLE_TO_DOWNLOAD
)
3948 delay
= *(int *)smartlist_get(schedule
, smartlist_len(schedule
) - 1);
3950 /* A negative delay makes no sense. Knowing that delay is
3951 * non-negative allows us to safely do the wrapping check below. */
3952 tor_assert(delay
>= 0);
3954 /* Avoid now+delay overflowing INT_MAX, by comparing with a subtraction
3955 * that won't overflow (since delay is non-negative). */
3956 if (delay
< INT_MAX
&& now
<= INT_MAX
- delay
) {
3957 dls
->next_attempt_at
= now
+delay
;
3959 dls
->next_attempt_at
= TIME_MAX
;
3965 /* Log a debug message about item, which increments on increment_action, has
3966 * incremented dls_n_download_increments times. The message varies based on
3967 * was_schedule_incremented (if not, not_incremented_response is logged), and
3968 * the values of increment, dls_next_attempt_at, and now.
3969 * Helper for download_status_increment_failure and
3970 * download_status_increment_attempt. */
3972 download_status_log_helper(const char *item
, int was_schedule_incremented
,
3973 const char *increment_action
,
3974 const char *not_incremented_response
,
3975 uint8_t dls_n_download_increments
, int increment
,
3976 time_t dls_next_attempt_at
, time_t now
)
3979 if (!was_schedule_incremented
)
3980 log_debug(LD_DIR
, "%s %s %d time(s); I'll try again %s.",
3981 item
, increment_action
, (int)dls_n_download_increments
,
3982 not_incremented_response
);
3983 else if (increment
== 0)
3984 log_debug(LD_DIR
, "%s %s %d time(s); I'll try again immediately.",
3985 item
, increment_action
, (int)dls_n_download_increments
);
3986 else if (dls_next_attempt_at
< TIME_MAX
)
3987 log_debug(LD_DIR
, "%s %s %d time(s); I'll try again in %d seconds.",
3988 item
, increment_action
, (int)dls_n_download_increments
,
3989 (int)(dls_next_attempt_at
-now
));
3991 log_debug(LD_DIR
, "%s %s %d time(s); Giving up for a while.",
3992 item
, increment_action
, (int)dls_n_download_increments
);
3996 /** Determine when a failed download attempt should be retried.
3997 * Called when an attempt to download <b>dls</b> has failed with HTTP status
3998 * <b>status_code</b>. Increment the failure count (if the code indicates a
3999 * real failure, or if we're a server) and set <b>dls</b>-\>next_attempt_at to
4000 * an appropriate time in the future and return it.
4001 * If <b>dls->increment_on</b> is DL_SCHED_INCREMENT_ATTEMPT, increment the
4002 * failure count, and return a time in the far future for the next attempt (to
4003 * avoid an immediate retry). */
4005 download_status_increment_failure(download_status_t
*dls
, int status_code
,
4006 const char *item
, int server
, time_t now
)
4011 /* only count the failure if it's permanent, or we're a server */
4012 if (status_code
!= 503 || server
) {
4013 if (dls
->n_download_failures
< IMPOSSIBLE_TO_DOWNLOAD
-1)
4014 ++dls
->n_download_failures
;
4017 if (dls
->increment_on
== DL_SCHED_INCREMENT_FAILURE
) {
4018 /* We don't find out that a failure-based schedule has attempted a
4019 * connection until that connection fails.
4020 * We'll never find out about successful connections, but this doesn't
4021 * matter, because schedules are reset after a successful download.
4023 if (dls
->n_download_attempts
< IMPOSSIBLE_TO_DOWNLOAD
-1)
4024 ++dls
->n_download_attempts
;
4026 /* only return a failure retry time if this schedule increments on failures
4028 const smartlist_t
*schedule
= find_dl_schedule(dls
, get_options());
4029 increment
= download_status_schedule_get_delay(dls
, schedule
, now
);
4032 download_status_log_helper(item
, !dls
->increment_on
, "failed",
4033 "concurrently", dls
->n_download_failures
,
4034 increment
, dls
->next_attempt_at
, now
);
4036 if (dls
->increment_on
== DL_SCHED_INCREMENT_ATTEMPT
) {
4037 /* stop this schedule retrying on failure, it will launch concurrent
4038 * connections instead */
4041 return dls
->next_attempt_at
;
4045 /** Determine when the next download attempt should be made when using an
4046 * attempt-based (potentially concurrent) download schedule.
4047 * Called when an attempt to download <b>dls</b> is being initiated.
4048 * Increment the attempt count and set <b>dls</b>-\>next_attempt_at to an
4049 * appropriate time in the future and return it.
4050 * If <b>dls->increment_on</b> is DL_SCHED_INCREMENT_FAILURE, don't increment
4051 * the attempts, and return a time in the far future (to avoid launching a
4052 * concurrent attempt). */
4054 download_status_increment_attempt(download_status_t
*dls
, const char *item
,
4060 if (dls
->increment_on
== DL_SCHED_INCREMENT_FAILURE
) {
4061 /* this schedule should retry on failure, and not launch any concurrent
4063 log_info(LD_BUG
, "Tried to launch an attempt-based connection on a "
4064 "failure-based schedule.");
4068 if (dls
->n_download_attempts
< IMPOSSIBLE_TO_DOWNLOAD
-1)
4069 ++dls
->n_download_attempts
;
4071 const smartlist_t
*schedule
= find_dl_schedule(dls
, get_options());
4072 delay
= download_status_schedule_get_delay(dls
, schedule
, now
);
4074 download_status_log_helper(item
, dls
->increment_on
, "attempted",
4075 "on failure", dls
->n_download_attempts
,
4076 delay
, dls
->next_attempt_at
, now
);
4078 return dls
->next_attempt_at
;
4081 /** Reset <b>dls</b> so that it will be considered downloadable
4082 * immediately, and/or to show that we don't need it anymore.
4084 * Must be called to initialise a download schedule, otherwise the zeroth item
4085 * in the schedule will never be used.
4087 * (We find the zeroth element of the download schedule, and set
4088 * next_attempt_at to be the appropriate offset from 'now'. In most
4089 * cases this means setting it to 'now', so the item will be immediately
4090 * downloadable; in the case of bridge descriptors, the zeroth element
4091 * is an hour from now.) */
4093 download_status_reset(download_status_t
*dls
)
4095 if (dls
->n_download_failures
== IMPOSSIBLE_TO_DOWNLOAD
4096 || dls
->n_download_attempts
== IMPOSSIBLE_TO_DOWNLOAD
)
4097 return; /* Don't reset this. */
4099 const smartlist_t
*schedule
= find_dl_schedule(dls
, get_options());
4101 dls
->n_download_failures
= 0;
4102 dls
->n_download_attempts
= 0;
4103 dls
->next_attempt_at
= time(NULL
) + *(int *)smartlist_get(schedule
, 0);
4104 /* Don't reset dls->want_authority or dls->increment_on */
4107 /** Return the number of failures on <b>dls</b> since the last success (if
4110 download_status_get_n_failures(const download_status_t
*dls
)
4112 return dls
->n_download_failures
;
4115 /** Return the number of attempts to download <b>dls</b> since the last success
4116 * (if any). This can differ from download_status_get_n_failures() due to
4117 * outstanding concurrent attempts. */
4119 download_status_get_n_attempts(const download_status_t
*dls
)
4121 return dls
->n_download_attempts
;
4124 /** Return the next time to attempt to download <b>dls</b>. */
4126 download_status_get_next_attempt_at(const download_status_t
*dls
)
4128 return dls
->next_attempt_at
;
4131 /** Called when one or more routerdesc (or extrainfo, if <b>was_extrainfo</b>)
4132 * fetches have failed (with uppercase fingerprints listed in <b>failed</b>,
4133 * either as descriptor digests or as identity digests based on
4134 * <b>was_descriptor_digests</b>).
4137 dir_routerdesc_download_failed(smartlist_t
*failed
, int status_code
,
4139 int was_extrainfo
, int was_descriptor_digests
)
4141 char digest
[DIGEST_LEN
];
4142 time_t now
= time(NULL
);
4143 int server
= directory_fetches_from_authorities(get_options());
4144 if (!was_descriptor_digests
) {
4145 if (router_purpose
== ROUTER_PURPOSE_BRIDGE
) {
4146 tor_assert(!was_extrainfo
);
4147 connection_dir_retry_bridges(failed
);
4149 return; /* FFFF should implement for other-than-router-purpose someday */
4151 SMARTLIST_FOREACH_BEGIN(failed
, const char *, cp
) {
4152 download_status_t
*dls
= NULL
;
4153 if (base16_decode(digest
, DIGEST_LEN
, cp
, strlen(cp
)) < 0) {
4154 log_warn(LD_BUG
, "Malformed fingerprint in list: %s", escaped(cp
));
4157 if (was_extrainfo
) {
4158 signed_descriptor_t
*sd
=
4159 router_get_by_extrainfo_digest(digest
);
4161 dls
= &sd
->ei_dl_status
;
4163 dls
= router_get_dl_status_by_descriptor_digest(digest
);
4165 if (!dls
|| dls
->n_download_failures
>=
4166 get_options()->TestingDescriptorMaxDownloadTries
)
4168 download_status_increment_failure(dls
, status_code
, cp
, server
, now
);
4169 } SMARTLIST_FOREACH_END(cp
);
4171 /* No need to relaunch descriptor downloads here: we already do it
4172 * every 10 or 60 seconds (FOO_DESCRIPTOR_RETRY_INTERVAL) in main.c. */
4175 /** Called when a connection to download microdescriptors has failed in whole
4176 * or in part. <b>failed</b> is a list of every microdesc digest we didn't
4177 * get. <b>status_code</b> is the http status code we received. Reschedule the
4178 * microdesc downloads as appropriate. */
4180 dir_microdesc_download_failed(smartlist_t
*failed
,
4183 networkstatus_t
*consensus
4184 = networkstatus_get_latest_consensus_by_flavor(FLAV_MICRODESC
);
4186 download_status_t
*dls
;
4187 time_t now
= time(NULL
);
4188 int server
= directory_fetches_from_authorities(get_options());
4192 SMARTLIST_FOREACH_BEGIN(failed
, const char *, d
) {
4193 rs
= router_get_mutable_consensus_status_by_descriptor_digest(consensus
,d
);
4196 dls
= &rs
->dl_status
;
4197 if (dls
->n_download_failures
>=
4198 get_options()->TestingMicrodescMaxDownloadTries
)
4201 char buf
[BASE64_DIGEST256_LEN
+1];
4202 digest256_to_base64(buf
, d
);
4203 download_status_increment_failure(dls
, status_code
, buf
,
4206 } SMARTLIST_FOREACH_END(d
);
4209 /** Helper. Compare two fp_pair_t objects, and return negative, 0, or
4210 * positive as appropriate. */
4212 compare_pairs_(const void **a
, const void **b
)
4214 const fp_pair_t
*fp1
= *a
, *fp2
= *b
;
4216 if ((r
= fast_memcmp(fp1
->first
, fp2
->first
, DIGEST_LEN
)))
4219 return fast_memcmp(fp1
->second
, fp2
->second
, DIGEST_LEN
);
4222 /** Divide a string <b>res</b> of the form FP1-FP2+FP3-FP4...[.z], where each
4223 * FP is a hex-encoded fingerprint, into a sequence of distinct sorted
4224 * fp_pair_t. Skip malformed pairs. On success, return 0 and add those
4225 * fp_pair_t into <b>pairs_out</b>. On failure, return -1. */
4227 dir_split_resource_into_fingerprint_pairs(const char *res
,
4228 smartlist_t
*pairs_out
)
4230 smartlist_t
*pairs_tmp
= smartlist_new();
4231 smartlist_t
*pairs_result
= smartlist_new();
4233 smartlist_split_string(pairs_tmp
, res
, "+", 0, 0);
4234 if (smartlist_len(pairs_tmp
)) {
4235 char *last
= smartlist_get(pairs_tmp
,smartlist_len(pairs_tmp
)-1);
4236 size_t last_len
= strlen(last
);
4237 if (last_len
> 2 && !strcmp(last
+last_len
-2, ".z")) {
4238 last
[last_len
-2] = '\0';
4241 SMARTLIST_FOREACH_BEGIN(pairs_tmp
, char *, cp
) {
4242 if (strlen(cp
) != HEX_DIGEST_LEN
*2+1) {
4244 "Skipping digest pair %s with non-standard length.", escaped(cp
));
4245 } else if (cp
[HEX_DIGEST_LEN
] != '-') {
4247 "Skipping digest pair %s with missing dash.", escaped(cp
));
4250 if (base16_decode(pair
.first
, DIGEST_LEN
, cp
, HEX_DIGEST_LEN
)<0 ||
4251 base16_decode(pair
.second
,
4252 DIGEST_LEN
, cp
+HEX_DIGEST_LEN
+1, HEX_DIGEST_LEN
)<0) {
4253 log_info(LD_DIR
, "Skipping non-decodable digest pair %s", escaped(cp
));
4255 smartlist_add(pairs_result
, tor_memdup(&pair
, sizeof(pair
)));
4259 } SMARTLIST_FOREACH_END(cp
);
4260 smartlist_free(pairs_tmp
);
4263 smartlist_sort(pairs_result
, compare_pairs_
);
4264 smartlist_uniq(pairs_result
, compare_pairs_
, tor_free_
);
4266 smartlist_add_all(pairs_out
, pairs_result
);
4267 smartlist_free(pairs_result
);
4271 /** Given a directory <b>resource</b> request, containing zero
4272 * or more strings separated by plus signs, followed optionally by ".z", store
4273 * the strings, in order, into <b>fp_out</b>. If <b>compressed_out</b> is
4274 * non-NULL, set it to 1 if the resource ends in ".z", else set it to 0.
4276 * If (flags & DSR_HEX), then delete all elements that aren't hex digests, and
4277 * decode the rest. If (flags & DSR_BASE64), then use "-" rather than "+" as
4278 * a separator, delete all the elements that aren't base64-encoded digests,
4279 * and decode the rest. If (flags & DSR_DIGEST256), these digests should be
4280 * 256 bits long; else they should be 160.
4282 * If (flags & DSR_SORT_UNIQ), then sort the list and remove all duplicates.
4285 dir_split_resource_into_fingerprints(const char *resource
,
4286 smartlist_t
*fp_out
, int *compressed_out
,
4289 const int decode_hex
= flags
& DSR_HEX
;
4290 const int decode_base64
= flags
& DSR_BASE64
;
4291 const int digests_are_256
= flags
& DSR_DIGEST256
;
4292 const int sort_uniq
= flags
& DSR_SORT_UNIQ
;
4294 const int digest_len
= digests_are_256
? DIGEST256_LEN
: DIGEST_LEN
;
4295 const int hex_digest_len
= digests_are_256
?
4296 HEX_DIGEST256_LEN
: HEX_DIGEST_LEN
;
4297 const int base64_digest_len
= digests_are_256
?
4298 BASE64_DIGEST256_LEN
: BASE64_DIGEST_LEN
;
4299 smartlist_t
*fp_tmp
= smartlist_new();
4301 tor_assert(!(decode_hex
&& decode_base64
));
4304 smartlist_split_string(fp_tmp
, resource
, decode_base64
?"-":"+", 0, 0);
4306 *compressed_out
= 0;
4307 if (smartlist_len(fp_tmp
)) {
4308 char *last
= smartlist_get(fp_tmp
,smartlist_len(fp_tmp
)-1);
4309 size_t last_len
= strlen(last
);
4310 if (last_len
> 2 && !strcmp(last
+last_len
-2, ".z")) {
4311 last
[last_len
-2] = '\0';
4313 *compressed_out
= 1;
4316 if (decode_hex
|| decode_base64
) {
4317 const size_t encoded_len
= decode_hex
? hex_digest_len
: base64_digest_len
;
4319 char *cp
, *d
= NULL
;
4320 for (i
= 0; i
< smartlist_len(fp_tmp
); ++i
) {
4321 cp
= smartlist_get(fp_tmp
, i
);
4322 if (strlen(cp
) != encoded_len
) {
4324 "Skipping digest %s with non-standard length.", escaped(cp
));
4325 smartlist_del_keeporder(fp_tmp
, i
--);
4328 d
= tor_malloc_zero(digest_len
);
4330 (base16_decode(d
, digest_len
, cp
, hex_digest_len
)<0) :
4331 (base64_decode(d
, digest_len
, cp
, base64_digest_len
)<0)) {
4332 log_info(LD_DIR
, "Skipping non-decodable digest %s", escaped(cp
));
4333 smartlist_del_keeporder(fp_tmp
, i
--);
4336 smartlist_set(fp_tmp
, i
, d
);
4344 if (decode_hex
|| decode_base64
) {
4345 if (digests_are_256
) {
4346 smartlist_sort_digests256(fp_tmp
);
4347 smartlist_uniq_digests256(fp_tmp
);
4349 smartlist_sort_digests(fp_tmp
);
4350 smartlist_uniq_digests(fp_tmp
);
4353 smartlist_sort_strings(fp_tmp
);
4354 smartlist_uniq_strings(fp_tmp
);
4357 smartlist_add_all(fp_out
, fp_tmp
);
4358 smartlist_free(fp_tmp
);