From f403af22076b0092807822ddc9890acdbd62f578 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Thu, 27 Sep 2018 09:35:47 -0400 Subject: [PATCH] Split geoip from geoip-related stats. This commit just moves the code to two separate files. The geoip code still has a few needless dependencies on core/* and features/*. --- src/app/config/config.c | 1 + src/app/main/main.c | 3 + src/core/include.am | 4 + src/core/mainloop/mainloop.c | 2 +- src/core/or/channel.c | 23 +- src/core/or/dos.c | 10 +- src/core/or/relay.c | 2 +- src/core/or/status.c | 2 +- src/feature/control/control.c | 3 +- src/feature/control/getinfo_geoip.c | 45 + src/feature/control/getinfo_geoip.h | 14 + src/feature/dircache/dircache.c | 2 +- src/feature/dircommon/directory.c | 2 +- src/feature/relay/router.c | 1 + src/feature/stats/geoip.c | 2427 ++++++-------------------- src/feature/stats/geoip.h | 199 +-- src/feature/stats/{geoip.c => geoip_stats.c} | 506 +----- src/feature/stats/{geoip.h => geoip_stats.h} | 30 +- src/test/test_dir_handle_get.c | 1 + src/test/test_dos.c | 3 +- src/test/test_geoip.c | 1 + 21 files changed, 654 insertions(+), 2627 deletions(-) create mode 100644 src/feature/control/getinfo_geoip.c create mode 100644 src/feature/control/getinfo_geoip.h rewrite src/feature/stats/geoip.c (74%) rewrite src/feature/stats/geoip.h (84%) copy src/feature/stats/{geoip.c => geoip_stats.c} (74%) copy src/feature/stats/{geoip.h => geoip_stats.h} (83%) diff --git a/src/app/config/config.c b/src/app/config/config.c index 2818cdaad8..0909ced506 100644 --- a/src/app/config/config.c +++ b/src/app/config/config.c @@ -103,6 +103,7 @@ #include "feature/rend/rendclient.h" #include "feature/rend/rendservice.h" #include "feature/stats/geoip.h" +#include "feature/stats/geoip_stats.h" #include "feature/stats/predict_ports.h" #include "feature/stats/rephist.h" #include "lib/compress/compress.h" diff --git a/src/app/main/main.c b/src/app/main/main.c index c756e55b9f..2266a5ff7f 100644 --- a/src/app/main/main.c +++ b/src/app/main/main.c @@ -27,6 +27,7 @@ #include "core/or/command.h" #include "core/or/connection_edge.h" #include "core/or/connection_or.h" +#include "core/or/dos.h" #include "core/or/policies.h" #include "core/or/protover.h" #include "core/or/relay.h" @@ -61,6 +62,7 @@ #include "feature/rend/rendclient.h" #include "feature/rend/rendservice.h" #include "feature/stats/geoip.h" +#include "feature/stats/geoip_stats.h" #include "feature/stats/predict_ports.h" #include "feature/stats/rephist.h" #include "lib/compress/compress.h" @@ -755,6 +757,7 @@ tor_free_all(int postfork) evdns_shutdown(1); } geoip_free_all(); + geoip_stats_free_all(); dirvote_free_all(); routerlist_free_all(); networkstatus_free_all(); diff --git a/src/core/include.am b/src/core/include.am index 68bbf94743..1a26a65c27 100644 --- a/src/core/include.am +++ b/src/core/include.am @@ -60,6 +60,7 @@ LIBTOR_APP_A_SOURCES = \ src/feature/client/transports.c \ src/feature/control/control.c \ src/feature/control/fmt_serverstatus.c \ + src/feature/control/getinfo_geoip.c \ src/feature/dirauth/keypin.c \ src/feature/dircache/conscache.c \ src/feature/dircache/consdiffmgr.c \ @@ -117,6 +118,7 @@ LIBTOR_APP_A_SOURCES = \ src/feature/rend/rendmid.c \ src/feature/rend/rendservice.c \ src/feature/stats/geoip.c \ + src/feature/stats/geoip_stats.c \ src/feature/stats/rephist.c \ src/feature/stats/predict_ports.c @@ -254,6 +256,7 @@ noinst_HEADERS += \ src/feature/control/control.h \ src/feature/control/control_connection_st.h \ src/feature/control/fmt_serverstatus.h \ + src/feature/control/getinfo_geoip.h \ src/feature/dirauth/authmode.h \ src/feature/dirauth/bwauth.h \ src/feature/dirauth/dircollate.h \ @@ -348,6 +351,7 @@ noinst_HEADERS += \ src/feature/rend/rendmid.h \ src/feature/rend/rendservice.h \ src/feature/stats/geoip.h \ + src/feature/stats/geoip_stats.h \ src/feature/stats/rephist.h \ src/feature/stats/predict_ports.h diff --git a/src/core/mainloop/mainloop.c b/src/core/mainloop/mainloop.c index ebfdf1c943..4eaeef46ce 100644 --- a/src/core/mainloop/mainloop.c +++ b/src/core/mainloop/mainloop.c @@ -92,7 +92,7 @@ #include "feature/relay/selftest.h" #include "feature/rend/rendcache.h" #include "feature/rend/rendservice.h" -#include "feature/stats/geoip.h" +#include "feature/stats/geoip_stats.h" #include "feature/stats/predict_ports.h" #include "feature/stats/rephist.h" #include "lib/container/buffers.h" diff --git a/src/core/or/channel.c b/src/core/or/channel.c index e9011520ae..fe3785d52e 100644 --- a/src/core/or/channel.c +++ b/src/core/or/channel.c @@ -58,28 +58,29 @@ #define CHANNEL_PRIVATE_ #include "core/or/or.h" +#include "app/config/config.h" +#include "core/mainloop/mainloop.h" #include "core/or/channel.h" -#include "core/or/channeltls.h" #include "core/or/channelpadding.h" +#include "core/or/channeltls.h" #include "core/or/circuitbuild.h" #include "core/or/circuitlist.h" +#include "core/or/circuitmux.h" #include "core/or/circuitstats.h" -#include "app/config/config.h" #include "core/or/connection_or.h" /* For var_cell_free() */ -#include "core/or/circuitmux.h" -#include "feature/client/entrynodes.h" -#include "feature/stats/geoip.h" -#include "core/mainloop/mainloop.h" -#include "feature/nodelist/nodelist.h" +#include "core/or/dos.h" #include "core/or/relay.h" -#include "feature/stats/rephist.h" -#include "feature/relay/router.h" -#include "feature/nodelist/routerlist.h" #include "core/or/scheduler.h" -#include "lib/time/compat_time.h" +#include "feature/client/entrynodes.h" #include "feature/nodelist/networkstatus.h" +#include "feature/nodelist/nodelist.h" +#include "feature/nodelist/routerlist.h" +#include "feature/relay/router.h" #include "feature/rend/rendservice.h" +#include "feature/stats/geoip_stats.h" +#include "feature/stats/rephist.h" #include "lib/evloop/timers.h" +#include "lib/time/compat_time.h" #include "core/or/cell_queue_st.h" diff --git a/src/core/or/dos.c b/src/core/or/dos.c index c59e2f6361..4303b8fd59 100644 --- a/src/core/or/dos.c +++ b/src/core/or/dos.c @@ -9,17 +9,17 @@ #define DOS_PRIVATE #include "core/or/or.h" -#include "core/or/channel.h" #include "app/config/config.h" #include "core/mainloop/connection.h" -#include "core/or/connection_or.h" -#include "lib/crypt_ops/crypto_rand.h" -#include "feature/stats/geoip.h" #include "core/mainloop/mainloop.h" +#include "core/or/channel.h" +#include "core/or/connection_or.h" +#include "core/or/relay.h" #include "feature/nodelist/networkstatus.h" #include "feature/nodelist/nodelist.h" -#include "core/or/relay.h" #include "feature/relay/routermode.h" +#include "feature/stats/geoip_stats.h" +#include "lib/crypt_ops/crypto_rand.h" #include "core/or/dos.h" diff --git a/src/core/or/relay.c b/src/core/or/relay.c index 88bcdf0703..9e45a35246 100644 --- a/src/core/or/relay.c +++ b/src/core/or/relay.c @@ -65,7 +65,7 @@ #include "lib/crypt_ops/crypto_util.h" #include "feature/dircommon/directory.h" #include "feature/relay/dns.h" -#include "feature/stats/geoip.h" +#include "feature/stats/geoip_stats.h" #include "feature/hs/hs_cache.h" #include "core/mainloop/mainloop.h" #include "feature/nodelist/networkstatus.h" diff --git a/src/core/or/status.c b/src/core/or/status.c index d8a9da0a55..cdb6df1230 100644 --- a/src/core/or/status.c +++ b/src/core/or/status.c @@ -30,13 +30,13 @@ #include "feature/hs/hs_stats.h" #include "feature/hs/hs_service.h" #include "core/or/dos.h" +#include "feature/stats/geoip_stats.h" #include "app/config/or_state_st.h" #include "feature/nodelist/routerinfo_st.h" #include "lib/tls/tortls.h" static void log_accounting(const time_t now, const or_options_t *options); -#include "feature/stats/geoip.h" /** Return the total number of circuits. */ STATIC int diff --git a/src/feature/control/control.c b/src/feature/control/control.c index b8ddb3c13e..d02ecbac80 100644 --- a/src/feature/control/control.c +++ b/src/feature/control/control.c @@ -60,6 +60,7 @@ #include "feature/client/entrynodes.h" #include "feature/control/control.h" #include "feature/control/fmt_serverstatus.h" +#include "feature/control/getinfo_geoip.h" #include "feature/dircache/dirserv.h" #include "feature/dirclient/dirclient.h" #include "feature/dirclient/dlstatus.h" @@ -83,7 +84,7 @@ #include "feature/rend/rendclient.h" #include "feature/rend/rendcommon.h" #include "feature/rend/rendservice.h" -#include "feature/stats/geoip.h" +#include "feature/stats/geoip_stats.h" #include "feature/stats/predict_ports.h" #include "lib/container/buffers.h" #include "lib/crypt_ops/crypto_rand.h" diff --git a/src/feature/control/getinfo_geoip.c b/src/feature/control/getinfo_geoip.c new file mode 100644 index 0000000000..35eb3f6aed --- /dev/null +++ b/src/feature/control/getinfo_geoip.c @@ -0,0 +1,45 @@ + +#include "core/or/or.h" +#include "core/mainloop/connection.h" +#include "feature/control/control.h" +#include "feature/control/getinfo_geoip.h" +#include "feature/stats/geoip.h" + +/** Helper used to implement GETINFO ip-to-country/... controller command. */ +int +getinfo_helper_geoip(control_connection_t *control_conn, + const char *question, char **answer, + const char **errmsg) +{ + (void)control_conn; + if (!strcmpstart(question, "ip-to-country/")) { + int c; + sa_family_t family; + tor_addr_t addr; + question += strlen("ip-to-country/"); + + if (!strcmp(question, "ipv4-available") || + !strcmp(question, "ipv6-available")) { + family = !strcmp(question, "ipv4-available") ? AF_INET : AF_INET6; + const int available = geoip_is_loaded(family); + tor_asprintf(answer, "%d", !! available); + return 0; + } + + family = tor_addr_parse(&addr, question); + if (family != AF_INET && family != AF_INET6) { + *errmsg = "Invalid address family"; + return -1; + } + if (!geoip_is_loaded(family)) { + *errmsg = "GeoIP data not loaded"; + return -1; + } + if (family == AF_INET) + c = geoip_get_country_by_ipv4(tor_addr_to_ipv4h(&addr)); + else /* AF_INET6 */ + c = geoip_get_country_by_ipv6(tor_addr_to_in6(&addr)); + *answer = tor_strdup(geoip_get_country_name(c)); + } + return 0; +} diff --git a/src/feature/control/getinfo_geoip.h b/src/feature/control/getinfo_geoip.h new file mode 100644 index 0000000000..ff77cefecd --- /dev/null +++ b/src/feature/control/getinfo_geoip.h @@ -0,0 +1,14 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#ifndef TOR_GETINFO_GEOIP_H +#define TOR_GETINFO_GEOIP_H + +int getinfo_helper_geoip(control_connection_t *control_conn, + const char *question, char **answer, + const char **errmsg); + +#endif diff --git a/src/feature/dircache/dircache.c b/src/feature/dircache/dircache.c index 76bde0b787..872a88018f 100644 --- a/src/feature/dircache/dircache.c +++ b/src/feature/dircache/dircache.c @@ -25,7 +25,7 @@ #include "feature/nodelist/routerlist.h" #include "feature/relay/routermode.h" #include "feature/rend/rendcache.h" -#include "feature/stats/geoip.h" +#include "feature/stats/geoip_stats.h" #include "feature/stats/rephist.h" #include "lib/compress/compress.h" diff --git a/src/feature/dircommon/directory.c b/src/feature/dircommon/directory.c index 8116d72c62..dff68fcf0e 100644 --- a/src/feature/dircommon/directory.c +++ b/src/feature/dircommon/directory.c @@ -12,7 +12,7 @@ #include "feature/dirclient/dirclient.h" #include "feature/dircommon/directory.h" #include "feature/dircommon/fp_pair.h" -#include "feature/stats/geoip.h" +#include "feature/stats/geoip_stats.h" #include "lib/compress/compress.h" #include "feature/dircommon/dir_connection_st.h" diff --git a/src/feature/relay/router.c b/src/feature/relay/router.c index 8629ad9561..4946348451 100644 --- a/src/feature/relay/router.c +++ b/src/feature/relay/router.c @@ -37,6 +37,7 @@ #include "feature/relay/routermode.h" #include "feature/relay/selftest.h" #include "feature/stats/geoip.h" +#include "feature/stats/geoip_stats.h" #include "feature/stats/rephist.h" #include "lib/crypt_ops/crypto_ed25519.h" #include "lib/crypt_ops/crypto_format.h" diff --git a/src/feature/stats/geoip.c b/src/feature/stats/geoip.c dissimilarity index 74% index 7bae448488..885f26bc3c 100644 --- a/src/feature/stats/geoip.c +++ b/src/feature/stats/geoip.c @@ -1,1920 +1,507 @@ -/* Copyright (c) 2007-2018, The Tor Project, Inc. */ -/* See LICENSE for licensing information */ - -/** - * \file geoip.c - * \brief Functions related to maintaining an IP-to-country database; - * to summarizing client connections by country to entry guards, bridges, - * and directory servers; and for statistics on answering network status - * requests. - * - * There are two main kinds of functions in this module: geoip functions, - * which map groups of IPv4 and IPv6 addresses to country codes, and - * statistical functions, which collect statistics about different kinds of - * per-country usage. - * - * The geoip lookup tables are implemented as sorted lists of disjoint address - * ranges, each mapping to a singleton geoip_country_t. These country objects - * are also indexed by their names in a hashtable. - * - * The tables are populated from disk at startup by the geoip_load_file() - * function. For more information on the file format they read, see that - * function. See the scripts and the README file in src/config for more - * information about how those files are generated. - * - * Tor uses GeoIP information in order to implement user requests (such as - * ExcludeNodes {cc}), and to keep track of how much usage relays are getting - * for each country. - */ - -#define GEOIP_PRIVATE -#include "core/or/or.h" -#include "ht.h" -#include "lib/container/buffers.h" -#include "app/config/config.h" -#include "feature/control/control.h" -#include "feature/client/dnsserv.h" -#include "core/or/dos.h" -#include "feature/stats/geoip.h" -#include "feature/nodelist/routerlist.h" - -#include "lib/container/order.h" -#include "lib/time/tvdiff.h" - -static void init_geoip_countries(void); - -/** An entry from the GeoIP IPv4 file: maps an IPv4 range to a country. */ -typedef struct geoip_ipv4_entry_t { - uint32_t ip_low; /**< The lowest IP in the range, in host order */ - uint32_t ip_high; /**< The highest IP in the range, in host order */ - intptr_t country; /**< An index into geoip_countries */ -} geoip_ipv4_entry_t; - -/** An entry from the GeoIP IPv6 file: maps an IPv6 range to a country. */ -typedef struct geoip_ipv6_entry_t { - struct in6_addr ip_low; /**< The lowest IP in the range, in host order */ - struct in6_addr ip_high; /**< The highest IP in the range, in host order */ - intptr_t country; /**< An index into geoip_countries */ -} geoip_ipv6_entry_t; - -/** A per-country record for GeoIP request history. */ -typedef struct geoip_country_t { - char countrycode[3]; -} geoip_country_t; - -/** A list of geoip_country_t */ -static smartlist_t *geoip_countries = NULL; -/** A map from lowercased country codes to their position in geoip_countries. - * The index is encoded in the pointer, and 1 is added so that NULL can mean - * not found. */ -static strmap_t *country_idxplus1_by_lc_code = NULL; -/** Lists of all known geoip_ipv4_entry_t and geoip_ipv6_entry_t, sorted - * by their respective ip_low. */ -static smartlist_t *geoip_ipv4_entries = NULL, *geoip_ipv6_entries = NULL; - -/** SHA1 digest of the GeoIP files to include in extra-info descriptors. */ -static char geoip_digest[DIGEST_LEN]; -static char geoip6_digest[DIGEST_LEN]; - -/** Number of entries in n_v3_ns_requests */ -size_t n_v3_ns_requests_len = 0; -/** Array, indexed by country index, of number of v3 networkstatus requests - * received from that country */ -static uint32_t *n_v3_ns_requests; - -/* Total size in bytes of the geoip client history cache. Used by the OOM - * handler. */ -static size_t geoip_client_history_cache_size; - -/* Increment the geoip client history cache size counter with the given bytes. - * This prevents an overflow and set it to its maximum in that case. */ -static inline void -geoip_increment_client_history_cache_size(size_t bytes) -{ - /* This is shockingly high, lets log it so it can be reported. */ - IF_BUG_ONCE(geoip_client_history_cache_size > (SIZE_MAX - bytes)) { - geoip_client_history_cache_size = SIZE_MAX; - return; - } - geoip_client_history_cache_size += bytes; -} - -/* Decrement the geoip client history cache size counter with the given bytes. - * This prevents an underflow and set it to 0 in that case. */ -static inline void -geoip_decrement_client_history_cache_size(size_t bytes) -{ - /* Going below 0 means that we either allocated an entry without - * incrementing the counter or we have different sizes when allocating and - * freeing. It shouldn't happened so log it. */ - IF_BUG_ONCE(geoip_client_history_cache_size < bytes) { - geoip_client_history_cache_size = 0; - return; - } - geoip_client_history_cache_size -= bytes; -} - -/** Add 1 to the count of v3 ns requests received from country. */ -static void -increment_v3_ns_request(country_t country) -{ - if (country < 0) - return; - - if ((size_t)country >= n_v3_ns_requests_len) { - /* We need to reallocate the array. */ - size_t new_len; - if (n_v3_ns_requests_len == 0) - new_len = 256; - else - new_len = n_v3_ns_requests_len * 2; - if (new_len <= (size_t)country) - new_len = ((size_t)country)+1; - n_v3_ns_requests = tor_reallocarray(n_v3_ns_requests, new_len, - sizeof(uint32_t)); - memset(n_v3_ns_requests + n_v3_ns_requests_len, 0, - sizeof(uint32_t)*(new_len - n_v3_ns_requests_len)); - n_v3_ns_requests_len = new_len; - } - - n_v3_ns_requests[country] += 1; -} - -/** Return the index of the country's entry in the GeoIP - * country list if it is a valid 2-letter country code, otherwise - * return -1. */ -MOCK_IMPL(country_t, -geoip_get_country,(const char *country)) -{ - void *idxplus1_; - intptr_t idx; - - idxplus1_ = strmap_get_lc(country_idxplus1_by_lc_code, country); - if (!idxplus1_) - return -1; - - idx = ((uintptr_t)idxplus1_)-1; - return (country_t)idx; -} - -/** Add an entry to a GeoIP table, mapping all IP addresses between low - * and high, inclusive, to the 2-letter country code country. */ -static void -geoip_add_entry(const tor_addr_t *low, const tor_addr_t *high, - const char *country) -{ - intptr_t idx; - void *idxplus1_; - - IF_BUG_ONCE(tor_addr_family(low) != tor_addr_family(high)) - return; - IF_BUG_ONCE(tor_addr_compare(high, low, CMP_EXACT) < 0) - return; - - idxplus1_ = strmap_get_lc(country_idxplus1_by_lc_code, country); - - if (!idxplus1_) { - geoip_country_t *c = tor_malloc_zero(sizeof(geoip_country_t)); - strlcpy(c->countrycode, country, sizeof(c->countrycode)); - tor_strlower(c->countrycode); - smartlist_add(geoip_countries, c); - idx = smartlist_len(geoip_countries) - 1; - strmap_set_lc(country_idxplus1_by_lc_code, country, (void*)(idx+1)); - } else { - idx = ((uintptr_t)idxplus1_)-1; - } - { - geoip_country_t *c = smartlist_get(geoip_countries, (int)idx); - tor_assert(!strcasecmp(c->countrycode, country)); - } - - if (tor_addr_family(low) == AF_INET) { - geoip_ipv4_entry_t *ent = tor_malloc_zero(sizeof(geoip_ipv4_entry_t)); - ent->ip_low = tor_addr_to_ipv4h(low); - ent->ip_high = tor_addr_to_ipv4h(high); - ent->country = idx; - smartlist_add(geoip_ipv4_entries, ent); - } else if (tor_addr_family(low) == AF_INET6) { - geoip_ipv6_entry_t *ent = tor_malloc_zero(sizeof(geoip_ipv6_entry_t)); - ent->ip_low = *tor_addr_to_in6_assert(low); - ent->ip_high = *tor_addr_to_in6_assert(high); - ent->country = idx; - smartlist_add(geoip_ipv6_entries, ent); - } -} - -/** Add an entry to the GeoIP table indicated by family, - * parsing it from line. The format is as for geoip_load_file(). */ -STATIC int -geoip_parse_entry(const char *line, sa_family_t family) -{ - tor_addr_t low_addr, high_addr; - char c[3]; - char *country = NULL; - - if (!geoip_countries) - init_geoip_countries(); - if (family == AF_INET) { - if (!geoip_ipv4_entries) - geoip_ipv4_entries = smartlist_new(); - } else if (family == AF_INET6) { - if (!geoip_ipv6_entries) - geoip_ipv6_entries = smartlist_new(); - } else { - log_warn(LD_GENERAL, "Unsupported family: %d", family); - return -1; - } - - while (TOR_ISSPACE(*line)) - ++line; - if (*line == '#') - return 0; - - char buf[512]; - if (family == AF_INET) { - unsigned int low, high; - if (tor_sscanf(line,"%u,%u,%2s", &low, &high, c) == 3 || - tor_sscanf(line,"\"%u\",\"%u\",\"%2s\",", &low, &high, c) == 3) { - tor_addr_from_ipv4h(&low_addr, low); - tor_addr_from_ipv4h(&high_addr, high); - } else - goto fail; - country = c; - } else { /* AF_INET6 */ - char *low_str, *high_str; - struct in6_addr low, high; - char *strtok_state; - strlcpy(buf, line, sizeof(buf)); - low_str = tor_strtok_r(buf, ",", &strtok_state); - if (!low_str) - goto fail; - high_str = tor_strtok_r(NULL, ",", &strtok_state); - if (!high_str) - goto fail; - country = tor_strtok_r(NULL, "\n", &strtok_state); - if (!country) - goto fail; - if (strlen(country) != 2) - goto fail; - if (tor_inet_pton(AF_INET6, low_str, &low) <= 0) - goto fail; - tor_addr_from_in6(&low_addr, &low); - if (tor_inet_pton(AF_INET6, high_str, &high) <= 0) - goto fail; - tor_addr_from_in6(&high_addr, &high); - } - geoip_add_entry(&low_addr, &high_addr, country); - return 0; - - fail: - log_warn(LD_GENERAL, "Unable to parse line from GEOIP %s file: %s", - family == AF_INET ? "IPv4" : "IPv6", escaped(line)); - return -1; -} - -/** Sorting helper: return -1, 1, or 0 based on comparison of two - * geoip_ipv4_entry_t */ -static int -geoip_ipv4_compare_entries_(const void **_a, const void **_b) -{ - const geoip_ipv4_entry_t *a = *_a, *b = *_b; - if (a->ip_low < b->ip_low) - return -1; - else if (a->ip_low > b->ip_low) - return 1; - else - return 0; -} - -/** bsearch helper: return -1, 1, or 0 based on comparison of an IP (a pointer - * to a uint32_t in host order) to a geoip_ipv4_entry_t */ -static int -geoip_ipv4_compare_key_to_entry_(const void *_key, const void **_member) -{ - /* No alignment issue here, since _key really is a pointer to uint32_t */ - const uint32_t addr = *(uint32_t *)_key; - const geoip_ipv4_entry_t *entry = *_member; - if (addr < entry->ip_low) - return -1; - else if (addr > entry->ip_high) - return 1; - else - return 0; -} - -/** Sorting helper: return -1, 1, or 0 based on comparison of two - * geoip_ipv6_entry_t */ -static int -geoip_ipv6_compare_entries_(const void **_a, const void **_b) -{ - const geoip_ipv6_entry_t *a = *_a, *b = *_b; - return fast_memcmp(a->ip_low.s6_addr, b->ip_low.s6_addr, - sizeof(struct in6_addr)); -} - -/** bsearch helper: return -1, 1, or 0 based on comparison of an IPv6 - * (a pointer to a in6_addr) to a geoip_ipv6_entry_t */ -static int -geoip_ipv6_compare_key_to_entry_(const void *_key, const void **_member) -{ - const struct in6_addr *addr = (struct in6_addr *)_key; - const geoip_ipv6_entry_t *entry = *_member; - - if (fast_memcmp(addr->s6_addr, entry->ip_low.s6_addr, - sizeof(struct in6_addr)) < 0) - return -1; - else if (fast_memcmp(addr->s6_addr, entry->ip_high.s6_addr, - sizeof(struct in6_addr)) > 0) - return 1; - else - return 0; -} - -/** Return 1 if we should collect geoip stats on bridge users, and - * include them in our extrainfo descriptor. Else return 0. */ -int -should_record_bridge_info(const or_options_t *options) -{ - return options->BridgeRelay && options->BridgeRecordUsageByCountry; -} - -/** Set up a new list of geoip countries with no countries (yet) set in it, - * except for the unknown country. - */ -static void -init_geoip_countries(void) -{ - geoip_country_t *geoip_unresolved; - geoip_countries = smartlist_new(); - /* Add a geoip_country_t for requests that could not be resolved to a - * country as first element (index 0) to geoip_countries. */ - geoip_unresolved = tor_malloc_zero(sizeof(geoip_country_t)); - strlcpy(geoip_unresolved->countrycode, "??", - sizeof(geoip_unresolved->countrycode)); - smartlist_add(geoip_countries, geoip_unresolved); - country_idxplus1_by_lc_code = strmap_new(); - strmap_set_lc(country_idxplus1_by_lc_code, "??", (void*)(1)); -} - -/** Clear appropriate GeoIP database, based on family, and - * reload it from the file filename. Return 0 on success, -1 on - * failure. - * - * Recognized line formats for IPv4 are: - * INTIPLOW,INTIPHIGH,CC - * and - * "INTIPLOW","INTIPHIGH","CC","CC3","COUNTRY NAME" - * where INTIPLOW and INTIPHIGH are IPv4 addresses encoded as 4-byte unsigned - * integers, and CC is a country code. - * - * Recognized line format for IPv6 is: - * IPV6LOW,IPV6HIGH,CC - * where IPV6LOW and IPV6HIGH are IPv6 addresses and CC is a country code. - * - * It also recognizes, and skips over, blank lines and lines that start - * with '#' (comments). - */ -int -geoip_load_file(sa_family_t family, const char *filename) -{ - FILE *f; - const char *msg = ""; - const or_options_t *options = get_options(); - int severity = options_need_geoip_info(options, &msg) ? LOG_WARN : LOG_INFO; - crypto_digest_t *geoip_digest_env = NULL; - - tor_assert(family == AF_INET || family == AF_INET6); - - if (!(f = tor_fopen_cloexec(filename, "r"))) { - log_fn(severity, LD_GENERAL, "Failed to open GEOIP file %s. %s", - filename, msg); - return -1; - } - if (!geoip_countries) - init_geoip_countries(); - - if (family == AF_INET) { - if (geoip_ipv4_entries) { - SMARTLIST_FOREACH(geoip_ipv4_entries, geoip_ipv4_entry_t *, e, - tor_free(e)); - smartlist_free(geoip_ipv4_entries); - } - geoip_ipv4_entries = smartlist_new(); - } else { /* AF_INET6 */ - if (geoip_ipv6_entries) { - SMARTLIST_FOREACH(geoip_ipv6_entries, geoip_ipv6_entry_t *, e, - tor_free(e)); - smartlist_free(geoip_ipv6_entries); - } - geoip_ipv6_entries = smartlist_new(); - } - geoip_digest_env = crypto_digest_new(); - - log_notice(LD_GENERAL, "Parsing GEOIP %s file %s.", - (family == AF_INET) ? "IPv4" : "IPv6", filename); - while (!feof(f)) { - char buf[512]; - if (fgets(buf, (int)sizeof(buf), f) == NULL) - break; - crypto_digest_add_bytes(geoip_digest_env, buf, strlen(buf)); - /* FFFF track full country name. */ - geoip_parse_entry(buf, family); - } - /*XXXX abort and return -1 if no entries/illformed?*/ - fclose(f); - - /* Sort list and remember file digests so that we can include it in - * our extra-info descriptors. */ - if (family == AF_INET) { - smartlist_sort(geoip_ipv4_entries, geoip_ipv4_compare_entries_); - /* Okay, now we need to maybe change our mind about what is in - * which country. We do this for IPv4 only since that's what we - * store in node->country. */ - refresh_all_country_info(); - crypto_digest_get_digest(geoip_digest_env, geoip_digest, DIGEST_LEN); - } else { - /* AF_INET6 */ - smartlist_sort(geoip_ipv6_entries, geoip_ipv6_compare_entries_); - crypto_digest_get_digest(geoip_digest_env, geoip6_digest, DIGEST_LEN); - } - crypto_digest_free(geoip_digest_env); - - return 0; -} - -/** Given an IP address in host order, return a number representing the - * country to which that address belongs, -1 for "No geoip information - * available", or 0 for the 'unknown country'. The return value will always - * be less than geoip_get_n_countries(). To decode it, call - * geoip_get_country_name(). - */ -STATIC int -geoip_get_country_by_ipv4(uint32_t ipaddr) -{ - geoip_ipv4_entry_t *ent; - if (!geoip_ipv4_entries) - return -1; - ent = smartlist_bsearch(geoip_ipv4_entries, &ipaddr, - geoip_ipv4_compare_key_to_entry_); - return ent ? (int)ent->country : 0; -} - -/** Given an IPv6 address, return a number representing the country to - * which that address belongs, -1 for "No geoip information available", or - * 0 for the 'unknown country'. The return value will always be less than - * geoip_get_n_countries(). To decode it, call geoip_get_country_name(). - */ -STATIC int -geoip_get_country_by_ipv6(const struct in6_addr *addr) -{ - geoip_ipv6_entry_t *ent; - - if (!geoip_ipv6_entries) - return -1; - ent = smartlist_bsearch(geoip_ipv6_entries, addr, - geoip_ipv6_compare_key_to_entry_); - return ent ? (int)ent->country : 0; -} - -/** Given an IP address, return a number representing the country to which - * that address belongs, -1 for "No geoip information available", or 0 for - * the 'unknown country'. The return value will always be less than - * geoip_get_n_countries(). To decode it, call geoip_get_country_name(). - */ -MOCK_IMPL(int, -geoip_get_country_by_addr,(const tor_addr_t *addr)) -{ - if (tor_addr_family(addr) == AF_INET) { - return geoip_get_country_by_ipv4(tor_addr_to_ipv4h(addr)); - } else if (tor_addr_family(addr) == AF_INET6) { - return geoip_get_country_by_ipv6(tor_addr_to_in6(addr)); - } else { - return -1; - } -} - -/** Return the number of countries recognized by the GeoIP country list. */ -MOCK_IMPL(int, -geoip_get_n_countries,(void)) -{ - if (!geoip_countries) - init_geoip_countries(); - return (int) smartlist_len(geoip_countries); -} - -/** Return the two-letter country code associated with the number num, - * or "??" for an unknown value. */ -const char * -geoip_get_country_name(country_t num) -{ - if (geoip_countries && num >= 0 && num < smartlist_len(geoip_countries)) { - geoip_country_t *c = smartlist_get(geoip_countries, num); - return c->countrycode; - } else - return "??"; -} - -/** Return true iff we have loaded a GeoIP database.*/ -MOCK_IMPL(int, -geoip_is_loaded,(sa_family_t family)) -{ - tor_assert(family == AF_INET || family == AF_INET6); - if (geoip_countries == NULL) - return 0; - if (family == AF_INET) - return geoip_ipv4_entries != NULL; - else /* AF_INET6 */ - return geoip_ipv6_entries != NULL; -} - -/** Return the hex-encoded SHA1 digest of the loaded GeoIP file. The - * result does not need to be deallocated, but will be overwritten by the - * next call of hex_str(). */ -const char * -geoip_db_digest(sa_family_t family) -{ - tor_assert(family == AF_INET || family == AF_INET6); - if (family == AF_INET) - return hex_str(geoip_digest, DIGEST_LEN); - else /* AF_INET6 */ - return hex_str(geoip6_digest, DIGEST_LEN); -} - -/** Largest allowable value for last_seen_in_minutes. (It's a 30-bit field, - * so it can hold up to (1u<<30)-1, or 0x3fffffffu. - */ -#define MAX_LAST_SEEN_IN_MINUTES 0X3FFFFFFFu - -/** Map from client IP address to last time seen. */ -static HT_HEAD(clientmap, clientmap_entry_t) client_history = - HT_INITIALIZER(); - -/** Hashtable helper: compute a hash of a clientmap_entry_t. */ -static inline unsigned -clientmap_entry_hash(const clientmap_entry_t *a) -{ - unsigned h = (unsigned) tor_addr_hash(&a->addr); - - if (a->transport_name) - h += (unsigned) siphash24g(a->transport_name, strlen(a->transport_name)); - - return h; -} -/** Hashtable helper: compare two clientmap_entry_t values for equality. */ -static inline int -clientmap_entries_eq(const clientmap_entry_t *a, const clientmap_entry_t *b) -{ - if (strcmp_opt(a->transport_name, b->transport_name)) - return 0; - - return !tor_addr_compare(&a->addr, &b->addr, CMP_EXACT) && - a->action == b->action; -} - -HT_PROTOTYPE(clientmap, clientmap_entry_t, node, clientmap_entry_hash, - clientmap_entries_eq) -HT_GENERATE2(clientmap, clientmap_entry_t, node, clientmap_entry_hash, - clientmap_entries_eq, 0.6, tor_reallocarray_, tor_free_) - -#define clientmap_entry_free(ent) \ - FREE_AND_NULL(clientmap_entry_t, clientmap_entry_free_, ent) - -/** Return the size of a client map entry. */ -static inline size_t -clientmap_entry_size(const clientmap_entry_t *ent) -{ - tor_assert(ent); - return (sizeof(clientmap_entry_t) + - (ent->transport_name ? strlen(ent->transport_name) : 0)); -} - -/** Free all storage held by ent. */ -static void -clientmap_entry_free_(clientmap_entry_t *ent) -{ - if (!ent) - return; - - /* This entry is about to be freed so pass it to the DoS subsystem to see if - * any actions can be taken about it. */ - dos_geoip_entry_about_to_free(ent); - geoip_decrement_client_history_cache_size(clientmap_entry_size(ent)); - - tor_free(ent->transport_name); - tor_free(ent); -} - -/* Return a newly allocated clientmap entry with the given action and address - * that are mandatory. The transport_name can be optional. This can't fail. */ -static clientmap_entry_t * -clientmap_entry_new(geoip_client_action_t action, const tor_addr_t *addr, - const char *transport_name) -{ - clientmap_entry_t *entry; - - tor_assert(action == GEOIP_CLIENT_CONNECT || - action == GEOIP_CLIENT_NETWORKSTATUS); - tor_assert(addr); - - entry = tor_malloc_zero(sizeof(clientmap_entry_t)); - entry->action = action; - tor_addr_copy(&entry->addr, addr); - if (transport_name) { - entry->transport_name = tor_strdup(transport_name); - } - - /* Allocated and initialized, note down its size for the OOM handler. */ - geoip_increment_client_history_cache_size(clientmap_entry_size(entry)); - - return entry; -} - -/** Clear history of connecting clients used by entry and bridge stats. */ -static void -client_history_clear(void) -{ - clientmap_entry_t **ent, **next, *this; - for (ent = HT_START(clientmap, &client_history); ent != NULL; - ent = next) { - if ((*ent)->action == GEOIP_CLIENT_CONNECT) { - this = *ent; - next = HT_NEXT_RMV(clientmap, &client_history, ent); - clientmap_entry_free(this); - } else { - next = HT_NEXT(clientmap, &client_history, ent); - } - } -} - -/** Note that we've seen a client connect from the IP addr - * at time now. Ignored by all but bridges and directories if - * configured accordingly. */ -void -geoip_note_client_seen(geoip_client_action_t action, - const tor_addr_t *addr, - const char *transport_name, - time_t now) -{ - const or_options_t *options = get_options(); - clientmap_entry_t *ent; - - if (action == GEOIP_CLIENT_CONNECT) { - /* Only remember statistics if the DoS mitigation subsystem is enabled. If - * not, only if as entry guard or as bridge. */ - if (!dos_enabled()) { - if (!options->EntryStatistics && !should_record_bridge_info(options)) { - return; - } - } - } else { - /* Only gather directory-request statistics if configured, and - * forcibly disable them on bridge authorities. */ - if (!options->DirReqStatistics || options->BridgeAuthoritativeDir) - return; - } - - log_debug(LD_GENERAL, "Seen client from '%s' with transport '%s'.", - safe_str_client(fmt_addr((addr))), - transport_name ? transport_name : ""); - - ent = geoip_lookup_client(addr, transport_name, action); - if (! ent) { - ent = clientmap_entry_new(action, addr, transport_name); - HT_INSERT(clientmap, &client_history, ent); - } - if (now / 60 <= (int)MAX_LAST_SEEN_IN_MINUTES && now >= 0) - ent->last_seen_in_minutes = (unsigned)(now/60); - else - ent->last_seen_in_minutes = 0; - - if (action == GEOIP_CLIENT_NETWORKSTATUS) { - int country_idx = geoip_get_country_by_addr(addr); - if (country_idx < 0) - country_idx = 0; /** unresolved requests are stored at index 0. */ - increment_v3_ns_request(country_idx); - } -} - -/** HT_FOREACH helper: remove a clientmap_entry_t from the hashtable if it's - * older than a certain time. */ -static int -remove_old_client_helper_(struct clientmap_entry_t *ent, void *_cutoff) -{ - time_t cutoff = *(time_t*)_cutoff / 60; - if (ent->last_seen_in_minutes < cutoff) { - clientmap_entry_free(ent); - return 1; - } else { - return 0; - } -} - -/** Forget about all clients that haven't connected since cutoff. */ -void -geoip_remove_old_clients(time_t cutoff) -{ - clientmap_HT_FOREACH_FN(&client_history, - remove_old_client_helper_, - &cutoff); -} - -/* Return a client entry object matching the given address, transport name and - * geoip action from the clientmap. NULL if not found. The transport_name can - * be NULL. */ -clientmap_entry_t * -geoip_lookup_client(const tor_addr_t *addr, const char *transport_name, - geoip_client_action_t action) -{ - clientmap_entry_t lookup; - - tor_assert(addr); - - /* We always look for a client connection with no transport. */ - tor_addr_copy(&lookup.addr, addr); - lookup.action = action; - lookup.transport_name = (char *) transport_name; - - return HT_FIND(clientmap, &client_history, &lookup); -} - -/* Cleanup client entries older than the cutoff. Used for the OOM. Return the - * number of bytes freed. If 0 is returned, nothing was freed. */ -static size_t -oom_clean_client_entries(time_t cutoff) -{ - size_t bytes = 0; - clientmap_entry_t **ent, **ent_next; - - for (ent = HT_START(clientmap, &client_history); ent; ent = ent_next) { - clientmap_entry_t *entry = *ent; - if (entry->last_seen_in_minutes < (cutoff / 60)) { - ent_next = HT_NEXT_RMV(clientmap, &client_history, ent); - bytes += clientmap_entry_size(entry); - clientmap_entry_free(entry); - } else { - ent_next = HT_NEXT(clientmap, &client_history, ent); - } - } - return bytes; -} - -/* Below this minimum lifetime, the OOM won't cleanup any entries. */ -#define GEOIP_CLIENT_CACHE_OOM_MIN_CUTOFF (4 * 60 * 60) -/* The OOM moves the cutoff by that much every run. */ -#define GEOIP_CLIENT_CACHE_OOM_STEP (15 * 50) - -/* Cleanup the geoip client history cache called from the OOM handler. Return - * the amount of bytes removed. This can return a value below or above - * min_remove_bytes but will stop as oon as the min_remove_bytes has been - * reached. */ -size_t -geoip_client_cache_handle_oom(time_t now, size_t min_remove_bytes) -{ - time_t k; - size_t bytes_removed = 0; - - /* Our OOM handler called with 0 bytes to remove is a code flow error. */ - tor_assert(min_remove_bytes != 0); - - /* Set k to the initial cutoff of an entry. We then going to move it by step - * to try to remove as much as we can. */ - k = WRITE_STATS_INTERVAL; - - do { - time_t cutoff; - - /* If k has reached the minimum lifetime, we have to stop else we might - * remove every single entries which would be pretty bad for the DoS - * mitigation subsystem if by just filling the geoip cache, it was enough - * to trigger the OOM and clean every single entries. */ - if (k <= GEOIP_CLIENT_CACHE_OOM_MIN_CUTOFF) { - break; - } - - cutoff = now - k; - bytes_removed += oom_clean_client_entries(cutoff); - k -= GEOIP_CLIENT_CACHE_OOM_STEP; - } while (bytes_removed < min_remove_bytes); - - return bytes_removed; -} - -/* Return the total size in bytes of the client history cache. */ -size_t -geoip_client_cache_total_allocation(void) -{ - return geoip_client_history_cache_size; -} - -/** How many responses are we giving to clients requesting v3 network - * statuses? */ -static uint32_t ns_v3_responses[GEOIP_NS_RESPONSE_NUM]; - -/** Note that we've rejected a client's request for a v3 network status - * for reason reason at time now. */ -void -geoip_note_ns_response(geoip_ns_response_t response) -{ - static int arrays_initialized = 0; - if (!get_options()->DirReqStatistics) - return; - if (!arrays_initialized) { - memset(ns_v3_responses, 0, sizeof(ns_v3_responses)); - arrays_initialized = 1; - } - tor_assert(response < GEOIP_NS_RESPONSE_NUM); - ns_v3_responses[response]++; -} - -/** Do not mention any country from which fewer than this number of IPs have - * connected. This conceivably avoids reporting information that could - * deanonymize users, though analysis is lacking. */ -#define MIN_IPS_TO_NOTE_COUNTRY 1 -/** Do not report any geoip data at all if we have fewer than this number of - * IPs to report about. */ -#define MIN_IPS_TO_NOTE_ANYTHING 1 -/** When reporting geoip data about countries, round up to the nearest - * multiple of this value. */ -#define IP_GRANULARITY 8 - -/** Helper type: used to sort per-country totals by value. */ -typedef struct c_hist_t { - char country[3]; /**< Two-letter country code. */ - unsigned total; /**< Total IP addresses seen in this country. */ -} c_hist_t; - -/** Sorting helper: return -1, 1, or 0 based on comparison of two - * geoip_ipv4_entry_t. Sort in descending order of total, and then by country - * code. */ -static int -c_hist_compare_(const void **_a, const void **_b) -{ - const c_hist_t *a = *_a, *b = *_b; - if (a->total > b->total) - return -1; - else if (a->total < b->total) - return 1; - else - return strcmp(a->country, b->country); -} - -/** When there are incomplete directory requests at the end of a 24-hour - * period, consider those requests running for longer than this timeout as - * failed, the others as still running. */ -#define DIRREQ_TIMEOUT (10*60) - -/** Entry in a map from either chan->global_identifier for direct requests - * or a unique circuit identifier for tunneled requests to request time, - * response size, and completion time of a network status request. Used to - * measure download times of requests to derive average client - * bandwidths. */ -typedef struct dirreq_map_entry_t { - HT_ENTRY(dirreq_map_entry_t) node; - /** Unique identifier for this network status request; this is either the - * chan->global_identifier of the dir channel (direct request) or a new - * locally unique identifier of a circuit (tunneled request). This ID is - * only unique among other direct or tunneled requests, respectively. */ - uint64_t dirreq_id; - unsigned int state:3; /**< State of this directory request. */ - unsigned int type:1; /**< Is this a direct or a tunneled request? */ - unsigned int completed:1; /**< Is this request complete? */ - /** When did we receive the request and started sending the response? */ - struct timeval request_time; - size_t response_size; /**< What is the size of the response in bytes? */ - struct timeval completion_time; /**< When did the request succeed? */ -} dirreq_map_entry_t; - -/** Map of all directory requests asking for v2 or v3 network statuses in - * the current geoip-stats interval. Values are - * of type *dirreq_map_entry_t. */ -static HT_HEAD(dirreqmap, dirreq_map_entry_t) dirreq_map = - HT_INITIALIZER(); - -static int -dirreq_map_ent_eq(const dirreq_map_entry_t *a, - const dirreq_map_entry_t *b) -{ - return a->dirreq_id == b->dirreq_id && a->type == b->type; -} - -/* DOCDOC dirreq_map_ent_hash */ -static unsigned -dirreq_map_ent_hash(const dirreq_map_entry_t *entry) -{ - unsigned u = (unsigned) entry->dirreq_id; - u += entry->type << 20; - return u; -} - -HT_PROTOTYPE(dirreqmap, dirreq_map_entry_t, node, dirreq_map_ent_hash, - dirreq_map_ent_eq) -HT_GENERATE2(dirreqmap, dirreq_map_entry_t, node, dirreq_map_ent_hash, - dirreq_map_ent_eq, 0.6, tor_reallocarray_, tor_free_) - -/** Helper: Put entry into map of directory requests using - * type and dirreq_id as key parts. If there is - * already an entry for that key, print out a BUG warning and return. */ -static void -dirreq_map_put_(dirreq_map_entry_t *entry, dirreq_type_t type, - uint64_t dirreq_id) -{ - dirreq_map_entry_t *old_ent; - tor_assert(entry->type == type); - tor_assert(entry->dirreq_id == dirreq_id); - - /* XXXX we could switch this to HT_INSERT some time, since it seems that - * this bug doesn't happen. But since this function doesn't seem to be - * critical-path, it's sane to leave it alone. */ - old_ent = HT_REPLACE(dirreqmap, &dirreq_map, entry); - if (old_ent && old_ent != entry) { - log_warn(LD_BUG, "Error when putting directory request into local " - "map. There was already an entry for the same identifier."); - return; - } -} - -/** Helper: Look up and return an entry in the map of directory requests - * using type and dirreq_id as key parts. If there - * is no such entry, return NULL. */ -static dirreq_map_entry_t * -dirreq_map_get_(dirreq_type_t type, uint64_t dirreq_id) -{ - dirreq_map_entry_t lookup; - lookup.type = type; - lookup.dirreq_id = dirreq_id; - return HT_FIND(dirreqmap, &dirreq_map, &lookup); -} - -/** Note that an either direct or tunneled (see type) directory - * request for a v3 network status with unique ID dirreq_id of size - * response_size has started. */ -void -geoip_start_dirreq(uint64_t dirreq_id, size_t response_size, - dirreq_type_t type) -{ - dirreq_map_entry_t *ent; - if (!get_options()->DirReqStatistics) - return; - ent = tor_malloc_zero(sizeof(dirreq_map_entry_t)); - ent->dirreq_id = dirreq_id; - tor_gettimeofday(&ent->request_time); - ent->response_size = response_size; - ent->type = type; - dirreq_map_put_(ent, type, dirreq_id); -} - -/** Change the state of the either direct or tunneled (see type) - * directory request with dirreq_id to new_state and - * possibly mark it as completed. If no entry can be found for the given - * key parts (e.g., if this is a directory request that we are not - * measuring, or one that was started in the previous measurement period), - * or if the state cannot be advanced to new_state, do nothing. */ -void -geoip_change_dirreq_state(uint64_t dirreq_id, dirreq_type_t type, - dirreq_state_t new_state) -{ - dirreq_map_entry_t *ent; - if (!get_options()->DirReqStatistics) - return; - ent = dirreq_map_get_(type, dirreq_id); - if (!ent) - return; - if (new_state == DIRREQ_IS_FOR_NETWORK_STATUS) - return; - if (new_state - 1 != ent->state) - return; - ent->state = new_state; - if ((type == DIRREQ_DIRECT && - new_state == DIRREQ_FLUSHING_DIR_CONN_FINISHED) || - (type == DIRREQ_TUNNELED && - new_state == DIRREQ_CHANNEL_BUFFER_FLUSHED)) { - tor_gettimeofday(&ent->completion_time); - ent->completed = 1; - } -} - -/** Return the bridge-ip-transports string that should be inserted in - * our extra-info descriptor. Return NULL if the bridge-ip-transports - * line should be empty. */ -char * -geoip_get_transport_history(void) -{ - unsigned granularity = IP_GRANULARITY; - /** String hash table (name of transport) -> (number of users). */ - strmap_t *transport_counts = strmap_new(); - - /** Smartlist that contains copies of the names of the transports - that have been used. */ - smartlist_t *transports_used = smartlist_new(); - - /* Special string to signify that no transport was used for this - connection. Pluggable transport names can't have symbols in their - names, so this string will never collide with a real transport. */ - static const char* no_transport_str = ""; - - clientmap_entry_t **ent; - smartlist_t *string_chunks = smartlist_new(); - char *the_string = NULL; - - /* If we haven't seen any clients yet, return NULL. */ - if (HT_EMPTY(&client_history)) - goto done; - - /** We do the following steps to form the transport history string: - * a) Foreach client that uses a pluggable transport, we increase the - * times that transport was used by one. If the client did not use - * a transport, we increase the number of times someone connected - * without obfuscation. - * b) Foreach transport we observed, we write its transport history - * string and push it to string_chunks. So, for example, if we've - * seen 665 obfs2 clients, we write "obfs2=665". - * c) We concatenate string_chunks to form the final string. - */ - - log_debug(LD_GENERAL,"Starting iteration for transport history. %d clients.", - HT_SIZE(&client_history)); - - /* Loop through all clients. */ - HT_FOREACH(ent, clientmap, &client_history) { - uintptr_t val; - void *ptr; - const char *transport_name = (*ent)->transport_name; - if (!transport_name) - transport_name = no_transport_str; - - /* Increase the count for this transport name. */ - ptr = strmap_get(transport_counts, transport_name); - val = (uintptr_t)ptr; - val++; - ptr = (void*)val; - strmap_set(transport_counts, transport_name, ptr); - - /* If it's the first time we see this transport, note it. */ - if (val == 1) - smartlist_add_strdup(transports_used, transport_name); - - log_debug(LD_GENERAL, "Client from '%s' with transport '%s'. " - "I've now seen %d clients.", - safe_str_client(fmt_addr(&(*ent)->addr)), - transport_name ? transport_name : "", - (int)val); - } - - /* Sort the transport names (helps with unit testing). */ - smartlist_sort_strings(transports_used); - - /* Loop through all seen transports. */ - SMARTLIST_FOREACH_BEGIN(transports_used, const char *, transport_name) { - void *transport_count_ptr = strmap_get(transport_counts, transport_name); - uintptr_t transport_count = (uintptr_t) transport_count_ptr; - - log_debug(LD_GENERAL, "We got %"PRIu64" clients with transport '%s'.", - ((uint64_t)transport_count), transport_name); - - smartlist_add_asprintf(string_chunks, "%s=%"PRIu64, - transport_name, - (round_uint64_to_next_multiple_of( - (uint64_t)transport_count, - granularity))); - } SMARTLIST_FOREACH_END(transport_name); - - the_string = smartlist_join_strings(string_chunks, ",", 0, NULL); - - log_debug(LD_GENERAL, "Final bridge-ip-transports string: '%s'", the_string); - - done: - strmap_free(transport_counts, NULL); - SMARTLIST_FOREACH(transports_used, char *, s, tor_free(s)); - smartlist_free(transports_used); - SMARTLIST_FOREACH(string_chunks, char *, s, tor_free(s)); - smartlist_free(string_chunks); - - return the_string; -} - -/** Return a newly allocated comma-separated string containing statistics - * on network status downloads. The string contains the number of completed - * requests, timeouts, and still running requests as well as the download - * times by deciles and quartiles. Return NULL if we have not observed - * requests for long enough. */ -static char * -geoip_get_dirreq_history(dirreq_type_t type) -{ - char *result = NULL; - buf_t *buf = NULL; - smartlist_t *dirreq_completed = NULL; - uint32_t complete = 0, timeouts = 0, running = 0; - dirreq_map_entry_t **ptr, **next; - struct timeval now; - - tor_gettimeofday(&now); - dirreq_completed = smartlist_new(); - for (ptr = HT_START(dirreqmap, &dirreq_map); ptr; ptr = next) { - dirreq_map_entry_t *ent = *ptr; - if (ent->type != type) { - next = HT_NEXT(dirreqmap, &dirreq_map, ptr); - continue; - } else { - if (ent->completed) { - smartlist_add(dirreq_completed, ent); - complete++; - next = HT_NEXT_RMV(dirreqmap, &dirreq_map, ptr); - } else { - if (tv_mdiff(&ent->request_time, &now) / 1000 > DIRREQ_TIMEOUT) - timeouts++; - else - running++; - next = HT_NEXT_RMV(dirreqmap, &dirreq_map, ptr); - tor_free(ent); - } - } - } -#define DIR_REQ_GRANULARITY 4 - complete = round_uint32_to_next_multiple_of(complete, - DIR_REQ_GRANULARITY); - timeouts = round_uint32_to_next_multiple_of(timeouts, - DIR_REQ_GRANULARITY); - running = round_uint32_to_next_multiple_of(running, - DIR_REQ_GRANULARITY); - buf = buf_new_with_capacity(1024); - buf_add_printf(buf, "complete=%u,timeout=%u," - "running=%u", complete, timeouts, running); - -#define MIN_DIR_REQ_RESPONSES 16 - if (complete >= MIN_DIR_REQ_RESPONSES) { - uint32_t *dltimes; - /* We may have rounded 'completed' up. Here we want to use the - * real value. */ - complete = smartlist_len(dirreq_completed); - dltimes = tor_calloc(complete, sizeof(uint32_t)); - SMARTLIST_FOREACH_BEGIN(dirreq_completed, dirreq_map_entry_t *, ent) { - uint32_t bytes_per_second; - uint32_t time_diff_ = (uint32_t) tv_mdiff(&ent->request_time, - &ent->completion_time); - if (time_diff_ == 0) - time_diff_ = 1; /* Avoid DIV/0; "instant" answers are impossible - * by law of nature or something, but a millisecond - * is a bit greater than "instantly" */ - bytes_per_second = (uint32_t)(1000 * ent->response_size / time_diff_); - dltimes[ent_sl_idx] = bytes_per_second; - } SMARTLIST_FOREACH_END(ent); - median_uint32(dltimes, complete); /* sorts as a side effect. */ - buf_add_printf(buf, - ",min=%u,d1=%u,d2=%u,q1=%u,d3=%u,d4=%u,md=%u," - "d6=%u,d7=%u,q3=%u,d8=%u,d9=%u,max=%u", - dltimes[0], - dltimes[1*complete/10-1], - dltimes[2*complete/10-1], - dltimes[1*complete/4-1], - dltimes[3*complete/10-1], - dltimes[4*complete/10-1], - dltimes[5*complete/10-1], - dltimes[6*complete/10-1], - dltimes[7*complete/10-1], - dltimes[3*complete/4-1], - dltimes[8*complete/10-1], - dltimes[9*complete/10-1], - dltimes[complete-1]); - tor_free(dltimes); - } - - result = buf_extract(buf, NULL); - - SMARTLIST_FOREACH(dirreq_completed, dirreq_map_entry_t *, ent, - tor_free(ent)); - smartlist_free(dirreq_completed); - buf_free(buf); - return result; -} - -/** Store a newly allocated comma-separated string in - * *country_str containing entries for all the countries from - * which we've seen enough clients connect as a bridge, directory - * server, or entry guard. The entry format is cc=num where num is the - * number of IPs we've seen connecting from that country, and cc is a - * lowercased country code. *country_str is set to NULL if - * we're not ready to export per country data yet. - * - * Store a newly allocated comma-separated string in ipver_str - * containing entries for clients connecting over IPv4 and IPv6. The - * format is family=num where num is the nubmer of IPs we've seen - * connecting over that protocol family, and family is 'v4' or 'v6'. - * - * Return 0 on success and -1 if we're missing geoip data. */ -int -geoip_get_client_history(geoip_client_action_t action, - char **country_str, char **ipver_str) -{ - unsigned granularity = IP_GRANULARITY; - smartlist_t *entries = NULL; - int n_countries = geoip_get_n_countries(); - int i; - clientmap_entry_t **cm_ent; - unsigned *counts = NULL; - unsigned total = 0; - unsigned ipv4_count = 0, ipv6_count = 0; - - if (!geoip_is_loaded(AF_INET) && !geoip_is_loaded(AF_INET6)) - return -1; - - counts = tor_calloc(n_countries, sizeof(unsigned)); - HT_FOREACH(cm_ent, clientmap, &client_history) { - int country; - if ((*cm_ent)->action != (int)action) - continue; - country = geoip_get_country_by_addr(&(*cm_ent)->addr); - if (country < 0) - country = 0; /** unresolved requests are stored at index 0. */ - tor_assert(0 <= country && country < n_countries); - ++counts[country]; - ++total; - switch (tor_addr_family(&(*cm_ent)->addr)) { - case AF_INET: - ipv4_count++; - break; - case AF_INET6: - ipv6_count++; - break; - } - } - if (ipver_str) { - smartlist_t *chunks = smartlist_new(); - smartlist_add_asprintf(chunks, "v4=%u", - round_to_next_multiple_of(ipv4_count, granularity)); - smartlist_add_asprintf(chunks, "v6=%u", - round_to_next_multiple_of(ipv6_count, granularity)); - *ipver_str = smartlist_join_strings(chunks, ",", 0, NULL); - SMARTLIST_FOREACH(chunks, char *, c, tor_free(c)); - smartlist_free(chunks); - } - - /* Don't record per country data if we haven't seen enough IPs. */ - if (total < MIN_IPS_TO_NOTE_ANYTHING) { - tor_free(counts); - if (country_str) - *country_str = NULL; - return 0; - } - - /* Make a list of c_hist_t */ - entries = smartlist_new(); - for (i = 0; i < n_countries; ++i) { - unsigned c = counts[i]; - const char *countrycode; - c_hist_t *ent; - /* Only report a country if it has a minimum number of IPs. */ - if (c >= MIN_IPS_TO_NOTE_COUNTRY) { - c = round_to_next_multiple_of(c, granularity); - countrycode = geoip_get_country_name(i); - ent = tor_malloc(sizeof(c_hist_t)); - strlcpy(ent->country, countrycode, sizeof(ent->country)); - ent->total = c; - smartlist_add(entries, ent); - } - } - /* Sort entries. Note that we must do this _AFTER_ rounding, or else - * the sort order could leak info. */ - smartlist_sort(entries, c_hist_compare_); - - if (country_str) { - smartlist_t *chunks = smartlist_new(); - SMARTLIST_FOREACH(entries, c_hist_t *, ch, { - smartlist_add_asprintf(chunks, "%s=%u", ch->country, ch->total); - }); - *country_str = smartlist_join_strings(chunks, ",", 0, NULL); - SMARTLIST_FOREACH(chunks, char *, c, tor_free(c)); - smartlist_free(chunks); - } - - SMARTLIST_FOREACH(entries, c_hist_t *, c, tor_free(c)); - smartlist_free(entries); - tor_free(counts); - - return 0; -} - -/** Return a newly allocated string holding the per-country request history - * for v3 network statuses in a format suitable for an extra-info document, - * or NULL on failure. */ -char * -geoip_get_request_history(void) -{ - smartlist_t *entries, *strings; - char *result; - unsigned granularity = IP_GRANULARITY; - - if (!geoip_countries) - return NULL; - - entries = smartlist_new(); - SMARTLIST_FOREACH_BEGIN(geoip_countries, geoip_country_t *, c) { - uint32_t tot = 0; - c_hist_t *ent; - if ((size_t)c_sl_idx < n_v3_ns_requests_len) - tot = n_v3_ns_requests[c_sl_idx]; - else - tot = 0; - if (!tot) - continue; - ent = tor_malloc_zero(sizeof(c_hist_t)); - strlcpy(ent->country, c->countrycode, sizeof(ent->country)); - ent->total = round_to_next_multiple_of(tot, granularity); - smartlist_add(entries, ent); - } SMARTLIST_FOREACH_END(c); - smartlist_sort(entries, c_hist_compare_); - - strings = smartlist_new(); - SMARTLIST_FOREACH(entries, c_hist_t *, ent, { - smartlist_add_asprintf(strings, "%s=%u", ent->country, ent->total); - }); - result = smartlist_join_strings(strings, ",", 0, NULL); - SMARTLIST_FOREACH(strings, char *, cp, tor_free(cp)); - SMARTLIST_FOREACH(entries, c_hist_t *, ent, tor_free(ent)); - smartlist_free(strings); - smartlist_free(entries); - return result; -} - -/** Start time of directory request stats or 0 if we're not collecting - * directory request statistics. */ -static time_t start_of_dirreq_stats_interval; - -/** Initialize directory request stats. */ -void -geoip_dirreq_stats_init(time_t now) -{ - start_of_dirreq_stats_interval = now; -} - -/** Reset counters for dirreq stats. */ -void -geoip_reset_dirreq_stats(time_t now) -{ - memset(n_v3_ns_requests, 0, - n_v3_ns_requests_len * sizeof(uint32_t)); - { - clientmap_entry_t **ent, **next, *this; - for (ent = HT_START(clientmap, &client_history); ent != NULL; - ent = next) { - if ((*ent)->action == GEOIP_CLIENT_NETWORKSTATUS) { - this = *ent; - next = HT_NEXT_RMV(clientmap, &client_history, ent); - clientmap_entry_free(this); - } else { - next = HT_NEXT(clientmap, &client_history, ent); - } - } - } - memset(ns_v3_responses, 0, sizeof(ns_v3_responses)); - { - dirreq_map_entry_t **ent, **next, *this; - for (ent = HT_START(dirreqmap, &dirreq_map); ent != NULL; ent = next) { - this = *ent; - next = HT_NEXT_RMV(dirreqmap, &dirreq_map, ent); - tor_free(this); - } - } - start_of_dirreq_stats_interval = now; -} - -/** Stop collecting directory request stats in a way that we can re-start - * doing so in geoip_dirreq_stats_init(). */ -void -geoip_dirreq_stats_term(void) -{ - geoip_reset_dirreq_stats(0); -} - -/** Return a newly allocated string containing the dirreq statistics - * until now, or NULL if we're not collecting dirreq stats. Caller - * must ensure start_of_dirreq_stats_interval is in the past. */ -char * -geoip_format_dirreq_stats(time_t now) -{ - char t[ISO_TIME_LEN+1]; - int i; - char *v3_ips_string = NULL, *v3_reqs_string = NULL, - *v3_direct_dl_string = NULL, *v3_tunneled_dl_string = NULL; - char *result = NULL; - - if (!start_of_dirreq_stats_interval) - return NULL; /* Not initialized. */ - - tor_assert(now >= start_of_dirreq_stats_interval); - - format_iso_time(t, now); - geoip_get_client_history(GEOIP_CLIENT_NETWORKSTATUS, &v3_ips_string, NULL); - v3_reqs_string = geoip_get_request_history(); - -#define RESPONSE_GRANULARITY 8 - for (i = 0; i < GEOIP_NS_RESPONSE_NUM; i++) { - ns_v3_responses[i] = round_uint32_to_next_multiple_of( - ns_v3_responses[i], RESPONSE_GRANULARITY); - } -#undef RESPONSE_GRANULARITY - - v3_direct_dl_string = geoip_get_dirreq_history(DIRREQ_DIRECT); - v3_tunneled_dl_string = geoip_get_dirreq_history(DIRREQ_TUNNELED); - - /* Put everything together into a single string. */ - tor_asprintf(&result, "dirreq-stats-end %s (%d s)\n" - "dirreq-v3-ips %s\n" - "dirreq-v3-reqs %s\n" - "dirreq-v3-resp ok=%u,not-enough-sigs=%u,unavailable=%u," - "not-found=%u,not-modified=%u,busy=%u\n" - "dirreq-v3-direct-dl %s\n" - "dirreq-v3-tunneled-dl %s\n", - t, - (unsigned) (now - start_of_dirreq_stats_interval), - v3_ips_string ? v3_ips_string : "", - v3_reqs_string ? v3_reqs_string : "", - ns_v3_responses[GEOIP_SUCCESS], - ns_v3_responses[GEOIP_REJECT_NOT_ENOUGH_SIGS], - ns_v3_responses[GEOIP_REJECT_UNAVAILABLE], - ns_v3_responses[GEOIP_REJECT_NOT_FOUND], - ns_v3_responses[GEOIP_REJECT_NOT_MODIFIED], - ns_v3_responses[GEOIP_REJECT_BUSY], - v3_direct_dl_string ? v3_direct_dl_string : "", - v3_tunneled_dl_string ? v3_tunneled_dl_string : ""); - - /* Free partial strings. */ - tor_free(v3_ips_string); - tor_free(v3_reqs_string); - tor_free(v3_direct_dl_string); - tor_free(v3_tunneled_dl_string); - - return result; -} - -/** If 24 hours have passed since the beginning of the current dirreq - * stats period, write dirreq stats to $DATADIR/stats/dirreq-stats - * (possibly overwriting an existing file) and reset counters. Return - * when we would next want to write dirreq stats or 0 if we never want to - * write. */ -time_t -geoip_dirreq_stats_write(time_t now) -{ - char *str = NULL; - - if (!start_of_dirreq_stats_interval) - return 0; /* Not initialized. */ - if (start_of_dirreq_stats_interval + WRITE_STATS_INTERVAL > now) - goto done; /* Not ready to write. */ - - /* Discard all items in the client history that are too old. */ - geoip_remove_old_clients(start_of_dirreq_stats_interval); - - /* Generate history string .*/ - str = geoip_format_dirreq_stats(now); - if (! str) - goto done; - - /* Write dirreq-stats string to disk. */ - if (!check_or_create_data_subdir("stats")) { - write_to_data_subdir("stats", "dirreq-stats", str, "dirreq statistics"); - /* Reset measurement interval start. */ - geoip_reset_dirreq_stats(now); - } - - done: - tor_free(str); - return start_of_dirreq_stats_interval + WRITE_STATS_INTERVAL; -} - -/** Start time of bridge stats or 0 if we're not collecting bridge - * statistics. */ -static time_t start_of_bridge_stats_interval; - -/** Initialize bridge stats. */ -void -geoip_bridge_stats_init(time_t now) -{ - start_of_bridge_stats_interval = now; -} - -/** Stop collecting bridge stats in a way that we can re-start doing so in - * geoip_bridge_stats_init(). */ -void -geoip_bridge_stats_term(void) -{ - client_history_clear(); - start_of_bridge_stats_interval = 0; -} - -/** Validate a bridge statistics string as it would be written to a - * current extra-info descriptor. Return 1 if the string is valid and - * recent enough, or 0 otherwise. */ -static int -validate_bridge_stats(const char *stats_str, time_t now) -{ - char stats_end_str[ISO_TIME_LEN+1], stats_start_str[ISO_TIME_LEN+1], - *eos; - - const char *BRIDGE_STATS_END = "bridge-stats-end "; - const char *BRIDGE_IPS = "bridge-ips "; - const char *BRIDGE_IPS_EMPTY_LINE = "bridge-ips\n"; - const char *BRIDGE_TRANSPORTS = "bridge-ip-transports "; - const char *BRIDGE_TRANSPORTS_EMPTY_LINE = "bridge-ip-transports\n"; - const char *tmp; - time_t stats_end_time; - int seconds; - tor_assert(stats_str); - - /* Parse timestamp and number of seconds from - "bridge-stats-end YYYY-MM-DD HH:MM:SS (N s)" */ - tmp = find_str_at_start_of_line(stats_str, BRIDGE_STATS_END); - if (!tmp) - return 0; - tmp += strlen(BRIDGE_STATS_END); - - if (strlen(tmp) < ISO_TIME_LEN + 6) - return 0; - strlcpy(stats_end_str, tmp, sizeof(stats_end_str)); - if (parse_iso_time(stats_end_str, &stats_end_time) < 0) - return 0; - if (stats_end_time < now - (25*60*60) || - stats_end_time > now + (1*60*60)) - return 0; - seconds = (int)strtol(tmp + ISO_TIME_LEN + 2, &eos, 10); - if (!eos || seconds < 23*60*60) - return 0; - format_iso_time(stats_start_str, stats_end_time - seconds); - - /* Parse: "bridge-ips CC=N,CC=N,..." */ - tmp = find_str_at_start_of_line(stats_str, BRIDGE_IPS); - if (!tmp) { - /* Look if there is an empty "bridge-ips" line */ - tmp = find_str_at_start_of_line(stats_str, BRIDGE_IPS_EMPTY_LINE); - if (!tmp) - return 0; - } - - /* Parse: "bridge-ip-transports PT=N,PT=N,..." */ - tmp = find_str_at_start_of_line(stats_str, BRIDGE_TRANSPORTS); - if (!tmp) { - /* Look if there is an empty "bridge-ip-transports" line */ - tmp = find_str_at_start_of_line(stats_str, BRIDGE_TRANSPORTS_EMPTY_LINE); - if (!tmp) - return 0; - } - - return 1; -} - -/** Most recent bridge statistics formatted to be written to extra-info - * descriptors. */ -static char *bridge_stats_extrainfo = NULL; - -/** Return a newly allocated string holding our bridge usage stats by country - * in a format suitable for inclusion in an extrainfo document. Return NULL on - * failure. */ -char * -geoip_format_bridge_stats(time_t now) -{ - char *out = NULL; - char *country_data = NULL, *ipver_data = NULL, *transport_data = NULL; - long duration = now - start_of_bridge_stats_interval; - char written[ISO_TIME_LEN+1]; - - if (duration < 0) - return NULL; - if (!start_of_bridge_stats_interval) - return NULL; /* Not initialized. */ - - format_iso_time(written, now); - geoip_get_client_history(GEOIP_CLIENT_CONNECT, &country_data, &ipver_data); - transport_data = geoip_get_transport_history(); - - tor_asprintf(&out, - "bridge-stats-end %s (%ld s)\n" - "bridge-ips %s\n" - "bridge-ip-versions %s\n" - "bridge-ip-transports %s\n", - written, duration, - country_data ? country_data : "", - ipver_data ? ipver_data : "", - transport_data ? transport_data : ""); - tor_free(country_data); - tor_free(ipver_data); - tor_free(transport_data); - - return out; -} - -/** Return a newly allocated string holding our bridge usage stats by country - * in a format suitable for the answer to a controller request. Return NULL on - * failure. */ -static char * -format_bridge_stats_controller(time_t now) -{ - char *out = NULL, *country_data = NULL, *ipver_data = NULL; - char started[ISO_TIME_LEN+1]; - (void) now; - - format_iso_time(started, start_of_bridge_stats_interval); - geoip_get_client_history(GEOIP_CLIENT_CONNECT, &country_data, &ipver_data); - - tor_asprintf(&out, - "TimeStarted=\"%s\" CountrySummary=%s IPVersions=%s", - started, - country_data ? country_data : "", - ipver_data ? ipver_data : ""); - tor_free(country_data); - tor_free(ipver_data); - return out; -} - -/** Return a newly allocated string holding our bridge usage stats by - * country in a format suitable for inclusion in our heartbeat - * message. Return NULL on failure. */ -char * -format_client_stats_heartbeat(time_t now) -{ - const int n_hours = 6; - char *out = NULL; - int n_clients = 0; - clientmap_entry_t **ent; - unsigned cutoff = (unsigned)( (now-n_hours*3600)/60 ); - - if (!start_of_bridge_stats_interval) - return NULL; /* Not initialized. */ - - /* count unique IPs */ - HT_FOREACH(ent, clientmap, &client_history) { - /* only count directly connecting clients */ - if ((*ent)->action != GEOIP_CLIENT_CONNECT) - continue; - if ((*ent)->last_seen_in_minutes < cutoff) - continue; - n_clients++; - } - - tor_asprintf(&out, "Heartbeat: " - "In the last %d hours, I have seen %d unique clients.", - n_hours, - n_clients); - - return out; -} - -/** Write bridge statistics to $DATADIR/stats/bridge-stats and return - * when we should next try to write statistics. */ -time_t -geoip_bridge_stats_write(time_t now) -{ - char *val = NULL; - - /* Check if 24 hours have passed since starting measurements. */ - if (now < start_of_bridge_stats_interval + WRITE_STATS_INTERVAL) - return start_of_bridge_stats_interval + WRITE_STATS_INTERVAL; - - /* Discard all items in the client history that are too old. */ - geoip_remove_old_clients(start_of_bridge_stats_interval); - - /* Generate formatted string */ - val = geoip_format_bridge_stats(now); - if (val == NULL) - goto done; - - /* Update the stored value. */ - tor_free(bridge_stats_extrainfo); - bridge_stats_extrainfo = val; - start_of_bridge_stats_interval = now; - - /* Write it to disk. */ - if (!check_or_create_data_subdir("stats")) { - write_to_data_subdir("stats", "bridge-stats", - bridge_stats_extrainfo, "bridge statistics"); - - /* Tell the controller, "hey, there are clients!" */ - { - char *controller_str = format_bridge_stats_controller(now); - if (controller_str) - control_event_clients_seen(controller_str); - tor_free(controller_str); - } - } - - done: - return start_of_bridge_stats_interval + WRITE_STATS_INTERVAL; -} - -/** Try to load the most recent bridge statistics from disk, unless we - * have finished a measurement interval lately, and check whether they - * are still recent enough. */ -static void -load_bridge_stats(time_t now) -{ - char *fname, *contents; - if (bridge_stats_extrainfo) - return; - - fname = get_datadir_fname2("stats", "bridge-stats"); - contents = read_file_to_str(fname, RFTS_IGNORE_MISSING, NULL); - if (contents && validate_bridge_stats(contents, now)) { - bridge_stats_extrainfo = contents; - } else { - tor_free(contents); - } - - tor_free(fname); -} - -/** Return most recent bridge statistics for inclusion in extra-info - * descriptors, or NULL if we don't have recent bridge statistics. */ -const char * -geoip_get_bridge_stats_extrainfo(time_t now) -{ - load_bridge_stats(now); - return bridge_stats_extrainfo; -} - -/** Return a new string containing the recent bridge statistics to be returned - * to controller clients, or NULL if we don't have any bridge statistics. */ -char * -geoip_get_bridge_stats_controller(time_t now) -{ - return format_bridge_stats_controller(now); -} - -/** Start time of entry stats or 0 if we're not collecting entry - * statistics. */ -static time_t start_of_entry_stats_interval; - -/** Initialize entry stats. */ -void -geoip_entry_stats_init(time_t now) -{ - start_of_entry_stats_interval = now; -} - -/** Reset counters for entry stats. */ -void -geoip_reset_entry_stats(time_t now) -{ - client_history_clear(); - start_of_entry_stats_interval = now; -} - -/** Stop collecting entry stats in a way that we can re-start doing so in - * geoip_entry_stats_init(). */ -void -geoip_entry_stats_term(void) -{ - geoip_reset_entry_stats(0); -} - -/** Return a newly allocated string containing the entry statistics - * until now, or NULL if we're not collecting entry stats. Caller - * must ensure start_of_entry_stats_interval lies in the past. */ -char * -geoip_format_entry_stats(time_t now) -{ - char t[ISO_TIME_LEN+1]; - char *data = NULL; - char *result; - - if (!start_of_entry_stats_interval) - return NULL; /* Not initialized. */ - - tor_assert(now >= start_of_entry_stats_interval); - - geoip_get_client_history(GEOIP_CLIENT_CONNECT, &data, NULL); - format_iso_time(t, now); - tor_asprintf(&result, - "entry-stats-end %s (%u s)\n" - "entry-ips %s\n", - t, (unsigned) (now - start_of_entry_stats_interval), - data ? data : ""); - tor_free(data); - return result; -} - -/** If 24 hours have passed since the beginning of the current entry stats - * period, write entry stats to $DATADIR/stats/entry-stats (possibly - * overwriting an existing file) and reset counters. Return when we would - * next want to write entry stats or 0 if we never want to write. */ -time_t -geoip_entry_stats_write(time_t now) -{ - char *str = NULL; - - if (!start_of_entry_stats_interval) - return 0; /* Not initialized. */ - if (start_of_entry_stats_interval + WRITE_STATS_INTERVAL > now) - goto done; /* Not ready to write. */ - - /* Discard all items in the client history that are too old. */ - geoip_remove_old_clients(start_of_entry_stats_interval); - - /* Generate history string .*/ - str = geoip_format_entry_stats(now); - - /* Write entry-stats string to disk. */ - if (!check_or_create_data_subdir("stats")) { - write_to_data_subdir("stats", "entry-stats", str, "entry statistics"); - - /* Reset measurement interval start. */ - geoip_reset_entry_stats(now); - } - - done: - tor_free(str); - return start_of_entry_stats_interval + WRITE_STATS_INTERVAL; -} - -/** Helper used to implement GETINFO ip-to-country/... controller command. */ -int -getinfo_helper_geoip(control_connection_t *control_conn, - const char *question, char **answer, - const char **errmsg) -{ - (void)control_conn; - if (!strcmpstart(question, "ip-to-country/")) { - int c; - sa_family_t family; - tor_addr_t addr; - question += strlen("ip-to-country/"); - - if (!strcmp(question, "ipv4-available") || - !strcmp(question, "ipv6-available")) { - family = !strcmp(question, "ipv4-available") ? AF_INET : AF_INET6; - const int available = geoip_is_loaded(family); - tor_asprintf(answer, "%d", !! available); - return 0; - } - - family = tor_addr_parse(&addr, question); - if (family != AF_INET && family != AF_INET6) { - *errmsg = "Invalid address family"; - return -1; - } - if (!geoip_is_loaded(family)) { - *errmsg = "GeoIP data not loaded"; - return -1; - } - if (family == AF_INET) - c = geoip_get_country_by_ipv4(tor_addr_to_ipv4h(&addr)); - else /* AF_INET6 */ - c = geoip_get_country_by_ipv6(tor_addr_to_in6(&addr)); - *answer = tor_strdup(geoip_get_country_name(c)); - } - return 0; -} - -/** Release all storage held by the GeoIP databases and country list. */ -STATIC void -clear_geoip_db(void) -{ - if (geoip_countries) { - SMARTLIST_FOREACH(geoip_countries, geoip_country_t *, c, tor_free(c)); - smartlist_free(geoip_countries); - } - - strmap_free(country_idxplus1_by_lc_code, NULL); - if (geoip_ipv4_entries) { - SMARTLIST_FOREACH(geoip_ipv4_entries, geoip_ipv4_entry_t *, ent, - tor_free(ent)); - smartlist_free(geoip_ipv4_entries); - } - if (geoip_ipv6_entries) { - SMARTLIST_FOREACH(geoip_ipv6_entries, geoip_ipv6_entry_t *, ent, - tor_free(ent)); - smartlist_free(geoip_ipv6_entries); - } - geoip_countries = NULL; - country_idxplus1_by_lc_code = NULL; - geoip_ipv4_entries = NULL; - geoip_ipv6_entries = NULL; -} - -/** Release all storage held in this file. */ -void -geoip_free_all(void) -{ - { - clientmap_entry_t **ent, **next, *this; - for (ent = HT_START(clientmap, &client_history); ent != NULL; ent = next) { - this = *ent; - next = HT_NEXT_RMV(clientmap, &client_history, ent); - clientmap_entry_free(this); - } - HT_CLEAR(clientmap, &client_history); - } - { - dirreq_map_entry_t **ent, **next, *this; - for (ent = HT_START(dirreqmap, &dirreq_map); ent != NULL; ent = next) { - this = *ent; - next = HT_NEXT_RMV(dirreqmap, &dirreq_map, ent); - tor_free(this); - } - HT_CLEAR(dirreqmap, &dirreq_map); - } - - clear_geoip_db(); - tor_free(bridge_stats_extrainfo); - tor_free(n_v3_ns_requests); - - memset(geoip_digest, 0, sizeof(geoip_digest)); - memset(geoip6_digest, 0, sizeof(geoip6_digest)); -} +/* Copyright (c) 2007-2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file geoip.c + * \brief Functions related to maintaining an IP-to-country database; + * to summarizing client connections by country to entry guards, bridges, + * and directory servers; and for statistics on answering network status + * requests. + * + * There are two main kinds of functions in this module: geoip functions, + * which map groups of IPv4 and IPv6 addresses to country codes, and + * statistical functions, which collect statistics about different kinds of + * per-country usage. + * + * The geoip lookup tables are implemented as sorted lists of disjoint address + * ranges, each mapping to a singleton geoip_country_t. These country objects + * are also indexed by their names in a hashtable. + * + * The tables are populated from disk at startup by the geoip_load_file() + * function. For more information on the file format they read, see that + * function. See the scripts and the README file in src/config for more + * information about how those files are generated. + * + * Tor uses GeoIP information in order to implement user requests (such as + * ExcludeNodes {cc}), and to keep track of how much usage relays are getting + * for each country. + */ + +#define GEOIP_PRIVATE +#include "core/or/or.h" +#include "ht.h" +#include "lib/container/buffers.h" +#include "app/config/config.h" // XXXX +#include "feature/stats/geoip.h" +#include "feature/nodelist/routerlist.h" // XXXX + +#include "lib/container/order.h" +#include "lib/time/tvdiff.h" + +static void init_geoip_countries(void); + +/** An entry from the GeoIP IPv4 file: maps an IPv4 range to a country. */ +typedef struct geoip_ipv4_entry_t { + uint32_t ip_low; /**< The lowest IP in the range, in host order */ + uint32_t ip_high; /**< The highest IP in the range, in host order */ + intptr_t country; /**< An index into geoip_countries */ +} geoip_ipv4_entry_t; + +/** An entry from the GeoIP IPv6 file: maps an IPv6 range to a country. */ +typedef struct geoip_ipv6_entry_t { + struct in6_addr ip_low; /**< The lowest IP in the range, in host order */ + struct in6_addr ip_high; /**< The highest IP in the range, in host order */ + intptr_t country; /**< An index into geoip_countries */ +} geoip_ipv6_entry_t; + +/** A list of geoip_country_t */ +static smartlist_t *geoip_countries = NULL; +/** A map from lowercased country codes to their position in geoip_countries. + * The index is encoded in the pointer, and 1 is added so that NULL can mean + * not found. */ +static strmap_t *country_idxplus1_by_lc_code = NULL; +/** Lists of all known geoip_ipv4_entry_t and geoip_ipv6_entry_t, sorted + * by their respective ip_low. */ +static smartlist_t *geoip_ipv4_entries = NULL, *geoip_ipv6_entries = NULL; + +/** SHA1 digest of the GeoIP files to include in extra-info descriptors. */ +static char geoip_digest[DIGEST_LEN]; +static char geoip6_digest[DIGEST_LEN]; + +/** Return a list of geoip_country_t for all known countries. */ +const smartlist_t * +geoip_get_countries(void) +{ + if (geoip_countries == NULL) { + init_geoip_countries(); + } + return geoip_countries; +} + +/** Return the index of the country's entry in the GeoIP + * country list if it is a valid 2-letter country code, otherwise + * return -1. */ +MOCK_IMPL(country_t, +geoip_get_country,(const char *country)) +{ + void *idxplus1_; + intptr_t idx; + + idxplus1_ = strmap_get_lc(country_idxplus1_by_lc_code, country); + if (!idxplus1_) + return -1; + + idx = ((uintptr_t)idxplus1_)-1; + return (country_t)idx; +} + +/** Add an entry to a GeoIP table, mapping all IP addresses between low + * and high, inclusive, to the 2-letter country code country. */ +static void +geoip_add_entry(const tor_addr_t *low, const tor_addr_t *high, + const char *country) +{ + intptr_t idx; + void *idxplus1_; + + IF_BUG_ONCE(tor_addr_family(low) != tor_addr_family(high)) + return; + IF_BUG_ONCE(tor_addr_compare(high, low, CMP_EXACT) < 0) + return; + + idxplus1_ = strmap_get_lc(country_idxplus1_by_lc_code, country); + + if (!idxplus1_) { + geoip_country_t *c = tor_malloc_zero(sizeof(geoip_country_t)); + strlcpy(c->countrycode, country, sizeof(c->countrycode)); + tor_strlower(c->countrycode); + smartlist_add(geoip_countries, c); + idx = smartlist_len(geoip_countries) - 1; + strmap_set_lc(country_idxplus1_by_lc_code, country, (void*)(idx+1)); + } else { + idx = ((uintptr_t)idxplus1_)-1; + } + { + geoip_country_t *c = smartlist_get(geoip_countries, (int)idx); + tor_assert(!strcasecmp(c->countrycode, country)); + } + + if (tor_addr_family(low) == AF_INET) { + geoip_ipv4_entry_t *ent = tor_malloc_zero(sizeof(geoip_ipv4_entry_t)); + ent->ip_low = tor_addr_to_ipv4h(low); + ent->ip_high = tor_addr_to_ipv4h(high); + ent->country = idx; + smartlist_add(geoip_ipv4_entries, ent); + } else if (tor_addr_family(low) == AF_INET6) { + geoip_ipv6_entry_t *ent = tor_malloc_zero(sizeof(geoip_ipv6_entry_t)); + ent->ip_low = *tor_addr_to_in6_assert(low); + ent->ip_high = *tor_addr_to_in6_assert(high); + ent->country = idx; + smartlist_add(geoip_ipv6_entries, ent); + } +} + +/** Add an entry to the GeoIP table indicated by family, + * parsing it from line. The format is as for geoip_load_file(). */ +STATIC int +geoip_parse_entry(const char *line, sa_family_t family) +{ + tor_addr_t low_addr, high_addr; + char c[3]; + char *country = NULL; + + if (!geoip_countries) + init_geoip_countries(); + if (family == AF_INET) { + if (!geoip_ipv4_entries) + geoip_ipv4_entries = smartlist_new(); + } else if (family == AF_INET6) { + if (!geoip_ipv6_entries) + geoip_ipv6_entries = smartlist_new(); + } else { + log_warn(LD_GENERAL, "Unsupported family: %d", family); + return -1; + } + + while (TOR_ISSPACE(*line)) + ++line; + if (*line == '#') + return 0; + + char buf[512]; + if (family == AF_INET) { + unsigned int low, high; + if (tor_sscanf(line,"%u,%u,%2s", &low, &high, c) == 3 || + tor_sscanf(line,"\"%u\",\"%u\",\"%2s\",", &low, &high, c) == 3) { + tor_addr_from_ipv4h(&low_addr, low); + tor_addr_from_ipv4h(&high_addr, high); + } else + goto fail; + country = c; + } else { /* AF_INET6 */ + char *low_str, *high_str; + struct in6_addr low, high; + char *strtok_state; + strlcpy(buf, line, sizeof(buf)); + low_str = tor_strtok_r(buf, ",", &strtok_state); + if (!low_str) + goto fail; + high_str = tor_strtok_r(NULL, ",", &strtok_state); + if (!high_str) + goto fail; + country = tor_strtok_r(NULL, "\n", &strtok_state); + if (!country) + goto fail; + if (strlen(country) != 2) + goto fail; + if (tor_inet_pton(AF_INET6, low_str, &low) <= 0) + goto fail; + tor_addr_from_in6(&low_addr, &low); + if (tor_inet_pton(AF_INET6, high_str, &high) <= 0) + goto fail; + tor_addr_from_in6(&high_addr, &high); + } + geoip_add_entry(&low_addr, &high_addr, country); + return 0; + + fail: + log_warn(LD_GENERAL, "Unable to parse line from GEOIP %s file: %s", + family == AF_INET ? "IPv4" : "IPv6", escaped(line)); + return -1; +} + +/** Sorting helper: return -1, 1, or 0 based on comparison of two + * geoip_ipv4_entry_t */ +static int +geoip_ipv4_compare_entries_(const void **_a, const void **_b) +{ + const geoip_ipv4_entry_t *a = *_a, *b = *_b; + if (a->ip_low < b->ip_low) + return -1; + else if (a->ip_low > b->ip_low) + return 1; + else + return 0; +} + +/** bsearch helper: return -1, 1, or 0 based on comparison of an IP (a pointer + * to a uint32_t in host order) to a geoip_ipv4_entry_t */ +static int +geoip_ipv4_compare_key_to_entry_(const void *_key, const void **_member) +{ + /* No alignment issue here, since _key really is a pointer to uint32_t */ + const uint32_t addr = *(uint32_t *)_key; + const geoip_ipv4_entry_t *entry = *_member; + if (addr < entry->ip_low) + return -1; + else if (addr > entry->ip_high) + return 1; + else + return 0; +} + +/** Sorting helper: return -1, 1, or 0 based on comparison of two + * geoip_ipv6_entry_t */ +static int +geoip_ipv6_compare_entries_(const void **_a, const void **_b) +{ + const geoip_ipv6_entry_t *a = *_a, *b = *_b; + return fast_memcmp(a->ip_low.s6_addr, b->ip_low.s6_addr, + sizeof(struct in6_addr)); +} + +/** bsearch helper: return -1, 1, or 0 based on comparison of an IPv6 + * (a pointer to a in6_addr) to a geoip_ipv6_entry_t */ +static int +geoip_ipv6_compare_key_to_entry_(const void *_key, const void **_member) +{ + const struct in6_addr *addr = (struct in6_addr *)_key; + const geoip_ipv6_entry_t *entry = *_member; + + if (fast_memcmp(addr->s6_addr, entry->ip_low.s6_addr, + sizeof(struct in6_addr)) < 0) + return -1; + else if (fast_memcmp(addr->s6_addr, entry->ip_high.s6_addr, + sizeof(struct in6_addr)) > 0) + return 1; + else + return 0; +} + +/** Set up a new list of geoip countries with no countries (yet) set in it, + * except for the unknown country. + */ +static void +init_geoip_countries(void) +{ + geoip_country_t *geoip_unresolved; + geoip_countries = smartlist_new(); + /* Add a geoip_country_t for requests that could not be resolved to a + * country as first element (index 0) to geoip_countries. */ + geoip_unresolved = tor_malloc_zero(sizeof(geoip_country_t)); + strlcpy(geoip_unresolved->countrycode, "??", + sizeof(geoip_unresolved->countrycode)); + smartlist_add(geoip_countries, geoip_unresolved); + country_idxplus1_by_lc_code = strmap_new(); + strmap_set_lc(country_idxplus1_by_lc_code, "??", (void*)(1)); +} + +/** Clear appropriate GeoIP database, based on family, and + * reload it from the file filename. Return 0 on success, -1 on + * failure. + * + * Recognized line formats for IPv4 are: + * INTIPLOW,INTIPHIGH,CC + * and + * "INTIPLOW","INTIPHIGH","CC","CC3","COUNTRY NAME" + * where INTIPLOW and INTIPHIGH are IPv4 addresses encoded as 4-byte unsigned + * integers, and CC is a country code. + * + * Recognized line format for IPv6 is: + * IPV6LOW,IPV6HIGH,CC + * where IPV6LOW and IPV6HIGH are IPv6 addresses and CC is a country code. + * + * It also recognizes, and skips over, blank lines and lines that start + * with '#' (comments). + */ +int +geoip_load_file(sa_family_t family, const char *filename) +{ + FILE *f; + const char *msg = ""; + const or_options_t *options = get_options(); + int severity = options_need_geoip_info(options, &msg) ? LOG_WARN : LOG_INFO; + crypto_digest_t *geoip_digest_env = NULL; + + tor_assert(family == AF_INET || family == AF_INET6); + + if (!(f = tor_fopen_cloexec(filename, "r"))) { + log_fn(severity, LD_GENERAL, "Failed to open GEOIP file %s. %s", + filename, msg); + return -1; + } + if (!geoip_countries) + init_geoip_countries(); + + if (family == AF_INET) { + if (geoip_ipv4_entries) { + SMARTLIST_FOREACH(geoip_ipv4_entries, geoip_ipv4_entry_t *, e, + tor_free(e)); + smartlist_free(geoip_ipv4_entries); + } + geoip_ipv4_entries = smartlist_new(); + } else { /* AF_INET6 */ + if (geoip_ipv6_entries) { + SMARTLIST_FOREACH(geoip_ipv6_entries, geoip_ipv6_entry_t *, e, + tor_free(e)); + smartlist_free(geoip_ipv6_entries); + } + geoip_ipv6_entries = smartlist_new(); + } + geoip_digest_env = crypto_digest_new(); + + log_notice(LD_GENERAL, "Parsing GEOIP %s file %s.", + (family == AF_INET) ? "IPv4" : "IPv6", filename); + while (!feof(f)) { + char buf[512]; + if (fgets(buf, (int)sizeof(buf), f) == NULL) + break; + crypto_digest_add_bytes(geoip_digest_env, buf, strlen(buf)); + /* FFFF track full country name. */ + geoip_parse_entry(buf, family); + } + /*XXXX abort and return -1 if no entries/illformed?*/ + fclose(f); + + /* Sort list and remember file digests so that we can include it in + * our extra-info descriptors. */ + if (family == AF_INET) { + smartlist_sort(geoip_ipv4_entries, geoip_ipv4_compare_entries_); + /* Okay, now we need to maybe change our mind about what is in + * which country. We do this for IPv4 only since that's what we + * store in node->country. */ + refresh_all_country_info(); + crypto_digest_get_digest(geoip_digest_env, geoip_digest, DIGEST_LEN); + } else { + /* AF_INET6 */ + smartlist_sort(geoip_ipv6_entries, geoip_ipv6_compare_entries_); + crypto_digest_get_digest(geoip_digest_env, geoip6_digest, DIGEST_LEN); + } + crypto_digest_free(geoip_digest_env); + + return 0; +} + +/** Given an IP address in host order, return a number representing the + * country to which that address belongs, -1 for "No geoip information + * available", or 0 for the 'unknown country'. The return value will always + * be less than geoip_get_n_countries(). To decode it, call + * geoip_get_country_name(). + */ +int +geoip_get_country_by_ipv4(uint32_t ipaddr) +{ + geoip_ipv4_entry_t *ent; + if (!geoip_ipv4_entries) + return -1; + ent = smartlist_bsearch(geoip_ipv4_entries, &ipaddr, + geoip_ipv4_compare_key_to_entry_); + return ent ? (int)ent->country : 0; +} + +/** Given an IPv6 address, return a number representing the country to + * which that address belongs, -1 for "No geoip information available", or + * 0 for the 'unknown country'. The return value will always be less than + * geoip_get_n_countries(). To decode it, call geoip_get_country_name(). + */ +int +geoip_get_country_by_ipv6(const struct in6_addr *addr) +{ + geoip_ipv6_entry_t *ent; + + if (!geoip_ipv6_entries) + return -1; + ent = smartlist_bsearch(geoip_ipv6_entries, addr, + geoip_ipv6_compare_key_to_entry_); + return ent ? (int)ent->country : 0; +} + +/** Given an IP address, return a number representing the country to which + * that address belongs, -1 for "No geoip information available", or 0 for + * the 'unknown country'. The return value will always be less than + * geoip_get_n_countries(). To decode it, call geoip_get_country_name(). + */ +MOCK_IMPL(int, +geoip_get_country_by_addr,(const tor_addr_t *addr)) +{ + if (tor_addr_family(addr) == AF_INET) { + return geoip_get_country_by_ipv4(tor_addr_to_ipv4h(addr)); + } else if (tor_addr_family(addr) == AF_INET6) { + return geoip_get_country_by_ipv6(tor_addr_to_in6(addr)); + } else { + return -1; + } +} + +/** Return the number of countries recognized by the GeoIP country list. */ +MOCK_IMPL(int, +geoip_get_n_countries,(void)) +{ + if (!geoip_countries) + init_geoip_countries(); + return (int) smartlist_len(geoip_countries); +} + +/** Return the two-letter country code associated with the number num, + * or "??" for an unknown value. */ +const char * +geoip_get_country_name(country_t num) +{ + if (geoip_countries && num >= 0 && num < smartlist_len(geoip_countries)) { + geoip_country_t *c = smartlist_get(geoip_countries, num); + return c->countrycode; + } else + return "??"; +} + +/** Return true iff we have loaded a GeoIP database.*/ +MOCK_IMPL(int, +geoip_is_loaded,(sa_family_t family)) +{ + tor_assert(family == AF_INET || family == AF_INET6); + if (geoip_countries == NULL) + return 0; + if (family == AF_INET) + return geoip_ipv4_entries != NULL; + else /* AF_INET6 */ + return geoip_ipv6_entries != NULL; +} + +/** Return the hex-encoded SHA1 digest of the loaded GeoIP file. The + * result does not need to be deallocated, but will be overwritten by the + * next call of hex_str(). */ +const char * +geoip_db_digest(sa_family_t family) +{ + tor_assert(family == AF_INET || family == AF_INET6); + if (family == AF_INET) + return hex_str(geoip_digest, DIGEST_LEN); + else /* AF_INET6 */ + return hex_str(geoip6_digest, DIGEST_LEN); +} + +/** Release all storage held by the GeoIP databases and country list. */ +STATIC void +clear_geoip_db(void) +{ + if (geoip_countries) { + SMARTLIST_FOREACH(geoip_countries, geoip_country_t *, c, tor_free(c)); + smartlist_free(geoip_countries); + } + + strmap_free(country_idxplus1_by_lc_code, NULL); + if (geoip_ipv4_entries) { + SMARTLIST_FOREACH(geoip_ipv4_entries, geoip_ipv4_entry_t *, ent, + tor_free(ent)); + smartlist_free(geoip_ipv4_entries); + } + if (geoip_ipv6_entries) { + SMARTLIST_FOREACH(geoip_ipv6_entries, geoip_ipv6_entry_t *, ent, + tor_free(ent)); + smartlist_free(geoip_ipv6_entries); + } + geoip_countries = NULL; + country_idxplus1_by_lc_code = NULL; + geoip_ipv4_entries = NULL; + geoip_ipv6_entries = NULL; +} + +/** Release all storage held in this file. */ +void +geoip_free_all(void) +{ + clear_geoip_db(); + + memset(geoip_digest, 0, sizeof(geoip_digest)); + memset(geoip6_digest, 0, sizeof(geoip6_digest)); +} diff --git a/src/feature/stats/geoip.h b/src/feature/stats/geoip.h dissimilarity index 84% index a2175c01e5..4adfc10fba 100644 --- a/src/feature/stats/geoip.h +++ b/src/feature/stats/geoip.h @@ -1,157 +1,42 @@ -/* Copyright (c) 2001 Matej Pfajfar. - * Copyright (c) 2001-2004, Roger Dingledine. - * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ -/* See LICENSE for licensing information */ - -/** - * \file geoip.h - * \brief Header file for geoip.c. - **/ - -#ifndef TOR_GEOIP_H -#define TOR_GEOIP_H - -#include "lib/testsupport/testsupport.h" -#include "core/or/dos.h" - -/** Indicates an action that we might be noting geoip statistics on. - * Note that if we're noticing CONNECT, we're a bridge, and if we're noticing - * the others, we're not. - */ -typedef enum { - /** We've noticed a connection as a bridge relay or entry guard. */ - GEOIP_CLIENT_CONNECT = 0, - /** We've served a networkstatus consensus as a directory server. */ - GEOIP_CLIENT_NETWORKSTATUS = 1, -} geoip_client_action_t; -/** Indicates either a positive reply or a reason for rejectng a network - * status request that will be included in geoip statistics. */ -typedef enum { - /** Request is answered successfully. */ - GEOIP_SUCCESS = 0, - /** V3 network status is not signed by a sufficient number of requested - * authorities. */ - GEOIP_REJECT_NOT_ENOUGH_SIGS = 1, - /** Requested network status object is unavailable. */ - GEOIP_REJECT_UNAVAILABLE = 2, - /** Requested network status not found. */ - GEOIP_REJECT_NOT_FOUND = 3, - /** Network status has not been modified since If-Modified-Since time. */ - GEOIP_REJECT_NOT_MODIFIED = 4, - /** Directory is busy. */ - GEOIP_REJECT_BUSY = 5, -} geoip_ns_response_t; -#define GEOIP_NS_RESPONSE_NUM 6 - -/** Directory requests that we are measuring can be either direct or - * tunneled. */ -typedef enum { - DIRREQ_DIRECT = 0, - DIRREQ_TUNNELED = 1, -} dirreq_type_t; - -/** Possible states for either direct or tunneled directory requests that - * are relevant for determining network status download times. */ -typedef enum { - /** Found that the client requests a network status; applies to both - * direct and tunneled requests; initial state of a request that we are - * measuring. */ - DIRREQ_IS_FOR_NETWORK_STATUS = 0, - /** Finished writing a network status to the directory connection; - * applies to both direct and tunneled requests; completes a direct - * request. */ - DIRREQ_FLUSHING_DIR_CONN_FINISHED = 1, - /** END cell sent to circuit that initiated a tunneled request. */ - DIRREQ_END_CELL_SENT = 2, - /** Flushed last cell from queue of the circuit that initiated a - * tunneled request to the outbuf of the OR connection. */ - DIRREQ_CIRC_QUEUE_FLUSHED = 3, - /** Flushed last byte from buffer of the channel belonging to the - * circuit that initiated a tunneled request; completes a tunneled - * request. */ - DIRREQ_CHANNEL_BUFFER_FLUSHED = 4 -} dirreq_state_t; - -#ifdef GEOIP_PRIVATE -STATIC int geoip_parse_entry(const char *line, sa_family_t family); -STATIC int geoip_get_country_by_ipv4(uint32_t ipaddr); -STATIC int geoip_get_country_by_ipv6(const struct in6_addr *addr); -STATIC void clear_geoip_db(void); -#endif /* defined(GEOIP_PRIVATE) */ - -/** Entry in a map from IP address to the last time we've seen an incoming - * connection from that IP address. Used by bridges only to track which - * countries have them blocked, or the DoS mitigation subsystem if enabled. */ -typedef struct clientmap_entry_t { - HT_ENTRY(clientmap_entry_t) node; - tor_addr_t addr; - /* Name of pluggable transport used by this client. NULL if no - pluggable transport was used. */ - char *transport_name; - - /** Time when we last saw this IP address, in MINUTES since the epoch. - * - * (This will run out of space around 4011 CE. If Tor is still in use around - * 4000 CE, please remember to add more bits to last_seen_in_minutes.) */ - unsigned int last_seen_in_minutes:30; - unsigned int action:2; - - /* This object is used to keep some statistics per client address for the - * DoS mitigation subsystem. */ - dos_client_stats_t dos_stats; -} clientmap_entry_t; - -int should_record_bridge_info(const or_options_t *options); -int geoip_load_file(sa_family_t family, const char *filename); -MOCK_DECL(int, geoip_get_country_by_addr, (const tor_addr_t *addr)); -MOCK_DECL(int, geoip_get_n_countries, (void)); -const char *geoip_get_country_name(country_t num); -MOCK_DECL(int, geoip_is_loaded, (sa_family_t family)); -const char *geoip_db_digest(sa_family_t family); -MOCK_DECL(country_t, geoip_get_country, (const char *countrycode)); - -void geoip_note_client_seen(geoip_client_action_t action, - const tor_addr_t *addr, const char *transport_name, - time_t now); -void geoip_remove_old_clients(time_t cutoff); -clientmap_entry_t *geoip_lookup_client(const tor_addr_t *addr, - const char *transport_name, - geoip_client_action_t action); -size_t geoip_client_cache_total_allocation(void); -size_t geoip_client_cache_handle_oom(time_t now, size_t min_remove_bytes); - -void geoip_note_ns_response(geoip_ns_response_t response); -char *geoip_get_transport_history(void); -int geoip_get_client_history(geoip_client_action_t action, - char **country_str, char **ipver_str); -char *geoip_get_request_history(void); -int getinfo_helper_geoip(control_connection_t *control_conn, - const char *question, char **answer, - const char **errmsg); -void geoip_free_all(void); - -void geoip_start_dirreq(uint64_t dirreq_id, size_t response_size, - dirreq_type_t type); -void geoip_change_dirreq_state(uint64_t dirreq_id, dirreq_type_t type, - dirreq_state_t new_state); - -void geoip_dirreq_stats_init(time_t now); -void geoip_reset_dirreq_stats(time_t now); -char *geoip_format_dirreq_stats(time_t now); -time_t geoip_dirreq_stats_write(time_t now); -void geoip_dirreq_stats_term(void); -void geoip_entry_stats_init(time_t now); -time_t geoip_entry_stats_write(time_t now); -void geoip_entry_stats_term(void); -void geoip_reset_entry_stats(time_t now); -char *geoip_format_entry_stats(time_t now); -void geoip_bridge_stats_init(time_t now); -char *geoip_format_bridge_stats(time_t now); -time_t geoip_bridge_stats_write(time_t now); -void geoip_bridge_stats_term(void); -const char *geoip_get_bridge_stats_extrainfo(time_t); -char *geoip_get_bridge_stats_controller(time_t); -char *format_client_stats_heartbeat(time_t now); - -#endif /* !defined(TOR_GEOIP_H) */ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file geoip.h + * \brief Header file for geoip.c. + **/ + +#ifndef TOR_GEOIP_H +#define TOR_GEOIP_H + +#include "lib/testsupport/testsupport.h" + +#ifdef GEOIP_PRIVATE +STATIC int geoip_parse_entry(const char *line, sa_family_t family); +STATIC void clear_geoip_db(void); +#endif /* defined(GEOIP_PRIVATE) */ + +int geoip_get_country_by_ipv4(uint32_t ipaddr); +int geoip_get_country_by_ipv6(const struct in6_addr *addr); + +/** A per-country GeoIP record. */ +typedef struct geoip_country_t { + char countrycode[3]; +} geoip_country_t; + +const smartlist_t *geoip_get_countries(void); + +int geoip_load_file(sa_family_t family, const char *filename); +MOCK_DECL(int, geoip_get_country_by_addr, (const tor_addr_t *addr)); +MOCK_DECL(int, geoip_get_n_countries, (void)); +const char *geoip_get_country_name(country_t num); +MOCK_DECL(int, geoip_is_loaded, (sa_family_t family)); +const char *geoip_db_digest(sa_family_t family); +MOCK_DECL(country_t, geoip_get_country, (const char *countrycode)); + +void geoip_free_all(void); + +#endif /* !defined(TOR_GEOIP_H) */ diff --git a/src/feature/stats/geoip.c b/src/feature/stats/geoip_stats.c similarity index 74% copy from src/feature/stats/geoip.c copy to src/feature/stats/geoip_stats.c index 7bae448488..4af2ff0ec0 100644 --- a/src/feature/stats/geoip.c +++ b/src/feature/stats/geoip_stats.c @@ -27,8 +27,8 @@ * for each country. */ -#define GEOIP_PRIVATE #include "core/or/or.h" + #include "ht.h" #include "lib/container/buffers.h" #include "app/config/config.h" @@ -36,46 +36,12 @@ #include "feature/client/dnsserv.h" #include "core/or/dos.h" #include "feature/stats/geoip.h" +#include "feature/stats/geoip_stats.h" #include "feature/nodelist/routerlist.h" #include "lib/container/order.h" #include "lib/time/tvdiff.h" -static void init_geoip_countries(void); - -/** An entry from the GeoIP IPv4 file: maps an IPv4 range to a country. */ -typedef struct geoip_ipv4_entry_t { - uint32_t ip_low; /**< The lowest IP in the range, in host order */ - uint32_t ip_high; /**< The highest IP in the range, in host order */ - intptr_t country; /**< An index into geoip_countries */ -} geoip_ipv4_entry_t; - -/** An entry from the GeoIP IPv6 file: maps an IPv6 range to a country. */ -typedef struct geoip_ipv6_entry_t { - struct in6_addr ip_low; /**< The lowest IP in the range, in host order */ - struct in6_addr ip_high; /**< The highest IP in the range, in host order */ - intptr_t country; /**< An index into geoip_countries */ -} geoip_ipv6_entry_t; - -/** A per-country record for GeoIP request history. */ -typedef struct geoip_country_t { - char countrycode[3]; -} geoip_country_t; - -/** A list of geoip_country_t */ -static smartlist_t *geoip_countries = NULL; -/** A map from lowercased country codes to their position in geoip_countries. - * The index is encoded in the pointer, and 1 is added so that NULL can mean - * not found. */ -static strmap_t *country_idxplus1_by_lc_code = NULL; -/** Lists of all known geoip_ipv4_entry_t and geoip_ipv6_entry_t, sorted - * by their respective ip_low. */ -static smartlist_t *geoip_ipv4_entries = NULL, *geoip_ipv6_entries = NULL; - -/** SHA1 digest of the GeoIP files to include in extra-info descriptors. */ -static char geoip_digest[DIGEST_LEN]; -static char geoip6_digest[DIGEST_LEN]; - /** Number of entries in n_v3_ns_requests */ size_t n_v3_ns_requests_len = 0; /** Array, indexed by country index, of number of v3 networkstatus requests @@ -140,196 +106,6 @@ increment_v3_ns_request(country_t country) n_v3_ns_requests[country] += 1; } -/** Return the index of the country's entry in the GeoIP - * country list if it is a valid 2-letter country code, otherwise - * return -1. */ -MOCK_IMPL(country_t, -geoip_get_country,(const char *country)) -{ - void *idxplus1_; - intptr_t idx; - - idxplus1_ = strmap_get_lc(country_idxplus1_by_lc_code, country); - if (!idxplus1_) - return -1; - - idx = ((uintptr_t)idxplus1_)-1; - return (country_t)idx; -} - -/** Add an entry to a GeoIP table, mapping all IP addresses between low - * and high, inclusive, to the 2-letter country code country. */ -static void -geoip_add_entry(const tor_addr_t *low, const tor_addr_t *high, - const char *country) -{ - intptr_t idx; - void *idxplus1_; - - IF_BUG_ONCE(tor_addr_family(low) != tor_addr_family(high)) - return; - IF_BUG_ONCE(tor_addr_compare(high, low, CMP_EXACT) < 0) - return; - - idxplus1_ = strmap_get_lc(country_idxplus1_by_lc_code, country); - - if (!idxplus1_) { - geoip_country_t *c = tor_malloc_zero(sizeof(geoip_country_t)); - strlcpy(c->countrycode, country, sizeof(c->countrycode)); - tor_strlower(c->countrycode); - smartlist_add(geoip_countries, c); - idx = smartlist_len(geoip_countries) - 1; - strmap_set_lc(country_idxplus1_by_lc_code, country, (void*)(idx+1)); - } else { - idx = ((uintptr_t)idxplus1_)-1; - } - { - geoip_country_t *c = smartlist_get(geoip_countries, (int)idx); - tor_assert(!strcasecmp(c->countrycode, country)); - } - - if (tor_addr_family(low) == AF_INET) { - geoip_ipv4_entry_t *ent = tor_malloc_zero(sizeof(geoip_ipv4_entry_t)); - ent->ip_low = tor_addr_to_ipv4h(low); - ent->ip_high = tor_addr_to_ipv4h(high); - ent->country = idx; - smartlist_add(geoip_ipv4_entries, ent); - } else if (tor_addr_family(low) == AF_INET6) { - geoip_ipv6_entry_t *ent = tor_malloc_zero(sizeof(geoip_ipv6_entry_t)); - ent->ip_low = *tor_addr_to_in6_assert(low); - ent->ip_high = *tor_addr_to_in6_assert(high); - ent->country = idx; - smartlist_add(geoip_ipv6_entries, ent); - } -} - -/** Add an entry to the GeoIP table indicated by family, - * parsing it from line. The format is as for geoip_load_file(). */ -STATIC int -geoip_parse_entry(const char *line, sa_family_t family) -{ - tor_addr_t low_addr, high_addr; - char c[3]; - char *country = NULL; - - if (!geoip_countries) - init_geoip_countries(); - if (family == AF_INET) { - if (!geoip_ipv4_entries) - geoip_ipv4_entries = smartlist_new(); - } else if (family == AF_INET6) { - if (!geoip_ipv6_entries) - geoip_ipv6_entries = smartlist_new(); - } else { - log_warn(LD_GENERAL, "Unsupported family: %d", family); - return -1; - } - - while (TOR_ISSPACE(*line)) - ++line; - if (*line == '#') - return 0; - - char buf[512]; - if (family == AF_INET) { - unsigned int low, high; - if (tor_sscanf(line,"%u,%u,%2s", &low, &high, c) == 3 || - tor_sscanf(line,"\"%u\",\"%u\",\"%2s\",", &low, &high, c) == 3) { - tor_addr_from_ipv4h(&low_addr, low); - tor_addr_from_ipv4h(&high_addr, high); - } else - goto fail; - country = c; - } else { /* AF_INET6 */ - char *low_str, *high_str; - struct in6_addr low, high; - char *strtok_state; - strlcpy(buf, line, sizeof(buf)); - low_str = tor_strtok_r(buf, ",", &strtok_state); - if (!low_str) - goto fail; - high_str = tor_strtok_r(NULL, ",", &strtok_state); - if (!high_str) - goto fail; - country = tor_strtok_r(NULL, "\n", &strtok_state); - if (!country) - goto fail; - if (strlen(country) != 2) - goto fail; - if (tor_inet_pton(AF_INET6, low_str, &low) <= 0) - goto fail; - tor_addr_from_in6(&low_addr, &low); - if (tor_inet_pton(AF_INET6, high_str, &high) <= 0) - goto fail; - tor_addr_from_in6(&high_addr, &high); - } - geoip_add_entry(&low_addr, &high_addr, country); - return 0; - - fail: - log_warn(LD_GENERAL, "Unable to parse line from GEOIP %s file: %s", - family == AF_INET ? "IPv4" : "IPv6", escaped(line)); - return -1; -} - -/** Sorting helper: return -1, 1, or 0 based on comparison of two - * geoip_ipv4_entry_t */ -static int -geoip_ipv4_compare_entries_(const void **_a, const void **_b) -{ - const geoip_ipv4_entry_t *a = *_a, *b = *_b; - if (a->ip_low < b->ip_low) - return -1; - else if (a->ip_low > b->ip_low) - return 1; - else - return 0; -} - -/** bsearch helper: return -1, 1, or 0 based on comparison of an IP (a pointer - * to a uint32_t in host order) to a geoip_ipv4_entry_t */ -static int -geoip_ipv4_compare_key_to_entry_(const void *_key, const void **_member) -{ - /* No alignment issue here, since _key really is a pointer to uint32_t */ - const uint32_t addr = *(uint32_t *)_key; - const geoip_ipv4_entry_t *entry = *_member; - if (addr < entry->ip_low) - return -1; - else if (addr > entry->ip_high) - return 1; - else - return 0; -} - -/** Sorting helper: return -1, 1, or 0 based on comparison of two - * geoip_ipv6_entry_t */ -static int -geoip_ipv6_compare_entries_(const void **_a, const void **_b) -{ - const geoip_ipv6_entry_t *a = *_a, *b = *_b; - return fast_memcmp(a->ip_low.s6_addr, b->ip_low.s6_addr, - sizeof(struct in6_addr)); -} - -/** bsearch helper: return -1, 1, or 0 based on comparison of an IPv6 - * (a pointer to a in6_addr) to a geoip_ipv6_entry_t */ -static int -geoip_ipv6_compare_key_to_entry_(const void *_key, const void **_member) -{ - const struct in6_addr *addr = (struct in6_addr *)_key; - const geoip_ipv6_entry_t *entry = *_member; - - if (fast_memcmp(addr->s6_addr, entry->ip_low.s6_addr, - sizeof(struct in6_addr)) < 0) - return -1; - else if (fast_memcmp(addr->s6_addr, entry->ip_high.s6_addr, - sizeof(struct in6_addr)) > 0) - return 1; - else - return 0; -} - /** Return 1 if we should collect geoip stats on bridge users, and * include them in our extrainfo descriptor. Else return 0. */ int @@ -338,208 +114,6 @@ should_record_bridge_info(const or_options_t *options) return options->BridgeRelay && options->BridgeRecordUsageByCountry; } -/** Set up a new list of geoip countries with no countries (yet) set in it, - * except for the unknown country. - */ -static void -init_geoip_countries(void) -{ - geoip_country_t *geoip_unresolved; - geoip_countries = smartlist_new(); - /* Add a geoip_country_t for requests that could not be resolved to a - * country as first element (index 0) to geoip_countries. */ - geoip_unresolved = tor_malloc_zero(sizeof(geoip_country_t)); - strlcpy(geoip_unresolved->countrycode, "??", - sizeof(geoip_unresolved->countrycode)); - smartlist_add(geoip_countries, geoip_unresolved); - country_idxplus1_by_lc_code = strmap_new(); - strmap_set_lc(country_idxplus1_by_lc_code, "??", (void*)(1)); -} - -/** Clear appropriate GeoIP database, based on family, and - * reload it from the file filename. Return 0 on success, -1 on - * failure. - * - * Recognized line formats for IPv4 are: - * INTIPLOW,INTIPHIGH,CC - * and - * "INTIPLOW","INTIPHIGH","CC","CC3","COUNTRY NAME" - * where INTIPLOW and INTIPHIGH are IPv4 addresses encoded as 4-byte unsigned - * integers, and CC is a country code. - * - * Recognized line format for IPv6 is: - * IPV6LOW,IPV6HIGH,CC - * where IPV6LOW and IPV6HIGH are IPv6 addresses and CC is a country code. - * - * It also recognizes, and skips over, blank lines and lines that start - * with '#' (comments). - */ -int -geoip_load_file(sa_family_t family, const char *filename) -{ - FILE *f; - const char *msg = ""; - const or_options_t *options = get_options(); - int severity = options_need_geoip_info(options, &msg) ? LOG_WARN : LOG_INFO; - crypto_digest_t *geoip_digest_env = NULL; - - tor_assert(family == AF_INET || family == AF_INET6); - - if (!(f = tor_fopen_cloexec(filename, "r"))) { - log_fn(severity, LD_GENERAL, "Failed to open GEOIP file %s. %s", - filename, msg); - return -1; - } - if (!geoip_countries) - init_geoip_countries(); - - if (family == AF_INET) { - if (geoip_ipv4_entries) { - SMARTLIST_FOREACH(geoip_ipv4_entries, geoip_ipv4_entry_t *, e, - tor_free(e)); - smartlist_free(geoip_ipv4_entries); - } - geoip_ipv4_entries = smartlist_new(); - } else { /* AF_INET6 */ - if (geoip_ipv6_entries) { - SMARTLIST_FOREACH(geoip_ipv6_entries, geoip_ipv6_entry_t *, e, - tor_free(e)); - smartlist_free(geoip_ipv6_entries); - } - geoip_ipv6_entries = smartlist_new(); - } - geoip_digest_env = crypto_digest_new(); - - log_notice(LD_GENERAL, "Parsing GEOIP %s file %s.", - (family == AF_INET) ? "IPv4" : "IPv6", filename); - while (!feof(f)) { - char buf[512]; - if (fgets(buf, (int)sizeof(buf), f) == NULL) - break; - crypto_digest_add_bytes(geoip_digest_env, buf, strlen(buf)); - /* FFFF track full country name. */ - geoip_parse_entry(buf, family); - } - /*XXXX abort and return -1 if no entries/illformed?*/ - fclose(f); - - /* Sort list and remember file digests so that we can include it in - * our extra-info descriptors. */ - if (family == AF_INET) { - smartlist_sort(geoip_ipv4_entries, geoip_ipv4_compare_entries_); - /* Okay, now we need to maybe change our mind about what is in - * which country. We do this for IPv4 only since that's what we - * store in node->country. */ - refresh_all_country_info(); - crypto_digest_get_digest(geoip_digest_env, geoip_digest, DIGEST_LEN); - } else { - /* AF_INET6 */ - smartlist_sort(geoip_ipv6_entries, geoip_ipv6_compare_entries_); - crypto_digest_get_digest(geoip_digest_env, geoip6_digest, DIGEST_LEN); - } - crypto_digest_free(geoip_digest_env); - - return 0; -} - -/** Given an IP address in host order, return a number representing the - * country to which that address belongs, -1 for "No geoip information - * available", or 0 for the 'unknown country'. The return value will always - * be less than geoip_get_n_countries(). To decode it, call - * geoip_get_country_name(). - */ -STATIC int -geoip_get_country_by_ipv4(uint32_t ipaddr) -{ - geoip_ipv4_entry_t *ent; - if (!geoip_ipv4_entries) - return -1; - ent = smartlist_bsearch(geoip_ipv4_entries, &ipaddr, - geoip_ipv4_compare_key_to_entry_); - return ent ? (int)ent->country : 0; -} - -/** Given an IPv6 address, return a number representing the country to - * which that address belongs, -1 for "No geoip information available", or - * 0 for the 'unknown country'. The return value will always be less than - * geoip_get_n_countries(). To decode it, call geoip_get_country_name(). - */ -STATIC int -geoip_get_country_by_ipv6(const struct in6_addr *addr) -{ - geoip_ipv6_entry_t *ent; - - if (!geoip_ipv6_entries) - return -1; - ent = smartlist_bsearch(geoip_ipv6_entries, addr, - geoip_ipv6_compare_key_to_entry_); - return ent ? (int)ent->country : 0; -} - -/** Given an IP address, return a number representing the country to which - * that address belongs, -1 for "No geoip information available", or 0 for - * the 'unknown country'. The return value will always be less than - * geoip_get_n_countries(). To decode it, call geoip_get_country_name(). - */ -MOCK_IMPL(int, -geoip_get_country_by_addr,(const tor_addr_t *addr)) -{ - if (tor_addr_family(addr) == AF_INET) { - return geoip_get_country_by_ipv4(tor_addr_to_ipv4h(addr)); - } else if (tor_addr_family(addr) == AF_INET6) { - return geoip_get_country_by_ipv6(tor_addr_to_in6(addr)); - } else { - return -1; - } -} - -/** Return the number of countries recognized by the GeoIP country list. */ -MOCK_IMPL(int, -geoip_get_n_countries,(void)) -{ - if (!geoip_countries) - init_geoip_countries(); - return (int) smartlist_len(geoip_countries); -} - -/** Return the two-letter country code associated with the number num, - * or "??" for an unknown value. */ -const char * -geoip_get_country_name(country_t num) -{ - if (geoip_countries && num >= 0 && num < smartlist_len(geoip_countries)) { - geoip_country_t *c = smartlist_get(geoip_countries, num); - return c->countrycode; - } else - return "??"; -} - -/** Return true iff we have loaded a GeoIP database.*/ -MOCK_IMPL(int, -geoip_is_loaded,(sa_family_t family)) -{ - tor_assert(family == AF_INET || family == AF_INET6); - if (geoip_countries == NULL) - return 0; - if (family == AF_INET) - return geoip_ipv4_entries != NULL; - else /* AF_INET6 */ - return geoip_ipv6_entries != NULL; -} - -/** Return the hex-encoded SHA1 digest of the loaded GeoIP file. The - * result does not need to be deallocated, but will be overwritten by the - * next call of hex_str(). */ -const char * -geoip_db_digest(sa_family_t family) -{ - tor_assert(family == AF_INET || family == AF_INET6); - if (family == AF_INET) - return hex_str(geoip_digest, DIGEST_LEN); - else /* AF_INET6 */ - return hex_str(geoip6_digest, DIGEST_LEN); -} - /** Largest allowable value for last_seen_in_minutes. (It's a 30-bit field, * so it can hold up to (1u<<30)-1, or 0x3fffffffu. */ @@ -1303,11 +877,8 @@ geoip_get_request_history(void) char *result; unsigned granularity = IP_GRANULARITY; - if (!geoip_countries) - return NULL; - entries = smartlist_new(); - SMARTLIST_FOREACH_BEGIN(geoip_countries, geoip_country_t *, c) { + SMARTLIST_FOREACH_BEGIN(geoip_get_countries(), const geoip_country_t *, c) { uint32_t tot = 0; c_hist_t *ent; if ((size_t)c_sl_idx < n_v3_ns_requests_len) @@ -1823,74 +1394,9 @@ geoip_entry_stats_write(time_t now) return start_of_entry_stats_interval + WRITE_STATS_INTERVAL; } -/** Helper used to implement GETINFO ip-to-country/... controller command. */ -int -getinfo_helper_geoip(control_connection_t *control_conn, - const char *question, char **answer, - const char **errmsg) -{ - (void)control_conn; - if (!strcmpstart(question, "ip-to-country/")) { - int c; - sa_family_t family; - tor_addr_t addr; - question += strlen("ip-to-country/"); - - if (!strcmp(question, "ipv4-available") || - !strcmp(question, "ipv6-available")) { - family = !strcmp(question, "ipv4-available") ? AF_INET : AF_INET6; - const int available = geoip_is_loaded(family); - tor_asprintf(answer, "%d", !! available); - return 0; - } - - family = tor_addr_parse(&addr, question); - if (family != AF_INET && family != AF_INET6) { - *errmsg = "Invalid address family"; - return -1; - } - if (!geoip_is_loaded(family)) { - *errmsg = "GeoIP data not loaded"; - return -1; - } - if (family == AF_INET) - c = geoip_get_country_by_ipv4(tor_addr_to_ipv4h(&addr)); - else /* AF_INET6 */ - c = geoip_get_country_by_ipv6(tor_addr_to_in6(&addr)); - *answer = tor_strdup(geoip_get_country_name(c)); - } - return 0; -} - -/** Release all storage held by the GeoIP databases and country list. */ -STATIC void -clear_geoip_db(void) -{ - if (geoip_countries) { - SMARTLIST_FOREACH(geoip_countries, geoip_country_t *, c, tor_free(c)); - smartlist_free(geoip_countries); - } - - strmap_free(country_idxplus1_by_lc_code, NULL); - if (geoip_ipv4_entries) { - SMARTLIST_FOREACH(geoip_ipv4_entries, geoip_ipv4_entry_t *, ent, - tor_free(ent)); - smartlist_free(geoip_ipv4_entries); - } - if (geoip_ipv6_entries) { - SMARTLIST_FOREACH(geoip_ipv6_entries, geoip_ipv6_entry_t *, ent, - tor_free(ent)); - smartlist_free(geoip_ipv6_entries); - } - geoip_countries = NULL; - country_idxplus1_by_lc_code = NULL; - geoip_ipv4_entries = NULL; - geoip_ipv6_entries = NULL; -} - /** Release all storage held in this file. */ void -geoip_free_all(void) +geoip_stats_free_all(void) { { clientmap_entry_t **ent, **next, *this; @@ -1911,10 +1417,6 @@ geoip_free_all(void) HT_CLEAR(dirreqmap, &dirreq_map); } - clear_geoip_db(); tor_free(bridge_stats_extrainfo); tor_free(n_v3_ns_requests); - - memset(geoip_digest, 0, sizeof(geoip_digest)); - memset(geoip6_digest, 0, sizeof(geoip6_digest)); } diff --git a/src/feature/stats/geoip.h b/src/feature/stats/geoip_stats.h similarity index 83% copy from src/feature/stats/geoip.h copy to src/feature/stats/geoip_stats.h index a2175c01e5..97011d62ce 100644 --- a/src/feature/stats/geoip.h +++ b/src/feature/stats/geoip_stats.h @@ -5,14 +5,13 @@ /* See LICENSE for licensing information */ /** - * \file geoip.h - * \brief Header file for geoip.c. + * \file geoip_stats.h + * \brief Header file for geoip_stats.c. **/ -#ifndef TOR_GEOIP_H -#define TOR_GEOIP_H +#ifndef TOR_GEOIP_STATS_H +#define TOR_GEOIP_STATS_H -#include "lib/testsupport/testsupport.h" #include "core/or/dos.h" /** Indicates an action that we might be noting geoip statistics on. @@ -73,13 +72,6 @@ typedef enum { DIRREQ_CHANNEL_BUFFER_FLUSHED = 4 } dirreq_state_t; -#ifdef GEOIP_PRIVATE -STATIC int geoip_parse_entry(const char *line, sa_family_t family); -STATIC int geoip_get_country_by_ipv4(uint32_t ipaddr); -STATIC int geoip_get_country_by_ipv6(const struct in6_addr *addr); -STATIC void clear_geoip_db(void); -#endif /* defined(GEOIP_PRIVATE) */ - /** Entry in a map from IP address to the last time we've seen an incoming * connection from that IP address. Used by bridges only to track which * countries have them blocked, or the DoS mitigation subsystem if enabled. */ @@ -103,13 +95,6 @@ typedef struct clientmap_entry_t { } clientmap_entry_t; int should_record_bridge_info(const or_options_t *options); -int geoip_load_file(sa_family_t family, const char *filename); -MOCK_DECL(int, geoip_get_country_by_addr, (const tor_addr_t *addr)); -MOCK_DECL(int, geoip_get_n_countries, (void)); -const char *geoip_get_country_name(country_t num); -MOCK_DECL(int, geoip_is_loaded, (sa_family_t family)); -const char *geoip_db_digest(sa_family_t family); -MOCK_DECL(country_t, geoip_get_country, (const char *countrycode)); void geoip_note_client_seen(geoip_client_action_t action, const tor_addr_t *addr, const char *transport_name, @@ -126,10 +111,7 @@ char *geoip_get_transport_history(void); int geoip_get_client_history(geoip_client_action_t action, char **country_str, char **ipver_str); char *geoip_get_request_history(void); -int getinfo_helper_geoip(control_connection_t *control_conn, - const char *question, char **answer, - const char **errmsg); -void geoip_free_all(void); +void geoip_stats_free_all(void); void geoip_start_dirreq(uint64_t dirreq_id, size_t response_size, dirreq_type_t type); @@ -154,4 +136,4 @@ const char *geoip_get_bridge_stats_extrainfo(time_t); char *geoip_get_bridge_stats_controller(time_t); char *format_client_stats_heartbeat(time_t now); -#endif /* !defined(TOR_GEOIP_H) */ +#endif /* !defined(TOR_GEOIP_STATS_H) */ diff --git a/src/test/test_dir_handle_get.c b/src/test/test_dir_handle_get.c index f05b0fc79f..7f7a00abde 100644 --- a/src/test/test_dir_handle_get.c +++ b/src/test/test_dir_handle_get.c @@ -33,6 +33,7 @@ #include "feature/nodelist/networkstatus.h" #include "core/proto/proto_http.h" #include "feature/stats/geoip.h" +#include "feature/stats/geoip_stats.h" #include "feature/dircache/dirserv.h" #include "feature/dirauth/dirvote.h" #include "test/log_test_helpers.h" diff --git a/src/test/test_dos.c b/src/test/test_dos.c index b411e7b38a..40a4c6ba29 100644 --- a/src/test/test_dos.c +++ b/src/test/test_dos.c @@ -9,7 +9,7 @@ #include "core/or/dos.h" #include "core/or/circuitlist.h" #include "lib/crypt_ops/crypto_rand.h" -#include "feature/stats/geoip.h" +#include "feature/stats/geoip_stats.h" #include "core/or/channel.h" #include "feature/nodelist/microdesc.h" #include "feature/nodelist/networkstatus.h" @@ -500,4 +500,3 @@ struct testcase_t dos_tests[] = { NULL, NULL }, END_OF_TESTCASES }; - diff --git a/src/test/test_geoip.c b/src/test/test_geoip.c index 9df8ea7988..8f13939dbd 100644 --- a/src/test/test_geoip.c +++ b/src/test/test_geoip.c @@ -11,6 +11,7 @@ #include "core/or/or.h" #include "app/config/config.h" #include "feature/stats/geoip.h" +#include "feature/stats/geoip_stats.h" #include "test/test.h" /* Record odd numbered fake-IPs using ipv6, even numbered fake-IPs -- 2.11.4.GIT