1 /* Copyright (c) 2009-2011, The Tor Project, Inc. */
2 /* See LICENSE for licensing information */
7 #include "routerparse.h"
9 /** A data structure to hold a bunch of cached microdescriptors. There are
10 * two active files in the cache: a "cache file" that we mmap, and a "journal
11 * file" that we append to. Periodically, we rebuild the cache file to hold
12 * only the microdescriptors that we want to keep */
13 struct microdesc_cache_t
{
14 /** Map from sha256-digest to microdesc_t for every microdesc_t in the
16 HT_HEAD(microdesc_map
, microdesc_t
) map
;
18 /** Name of the cache file. */
20 /** Name of the journal file. */
22 /** Mmap'd contents of the cache file, or NULL if there is none. */
23 tor_mmap_t
*cache_content
;
24 /** Number of bytes used in the journal file. */
27 /** Total bytes of microdescriptor bodies we have added to this cache */
28 uint64_t total_len_seen
;
29 /** Total number of microdescriptors we have added to this cache */
33 /** Helper: computes a hash of <b>md</b> to place it in a hash table. */
34 static INLINE
unsigned int
35 _microdesc_hash(microdesc_t
*md
)
37 unsigned *d
= (unsigned*)md
->digest
;
39 return d
[0] ^ d
[1] ^ d
[2] ^ d
[3] ^ d
[4] ^ d
[5] ^ d
[6] ^ d
[7];
41 return d
[0] ^ d
[1] ^ d
[2] ^ d
[3];
45 /** Helper: compares <b>a</b> and </b> for equality for hash-table purposes. */
47 _microdesc_eq(microdesc_t
*a
, microdesc_t
*b
)
49 return !memcmp(a
->digest
, b
->digest
, DIGEST256_LEN
);
52 HT_PROTOTYPE(microdesc_map
, microdesc_t
, node
,
53 _microdesc_hash
, _microdesc_eq
);
54 HT_GENERATE(microdesc_map
, microdesc_t
, node
,
55 _microdesc_hash
, _microdesc_eq
, 0.6,
56 malloc
, realloc
, free
);
58 /** Write the body of <b>md</b> into <b>f</b>, with appropriate annotations.
59 * On success, return the total number of bytes written, and set
60 * *<b>annotation_len_out</b> to the number of bytes written as
63 dump_microdescriptor(FILE *f
, microdesc_t
*md
, size_t *annotation_len_out
)
67 /* XXXX drops unkown annotations. */
68 if (md
->last_listed
) {
69 char buf
[ISO_TIME_LEN
+1];
70 char annotation
[ISO_TIME_LEN
+32];
71 format_iso_time(buf
, md
->last_listed
);
72 tor_snprintf(annotation
, sizeof(annotation
), "@last-listed %s\n", buf
);
74 r
+= strlen(annotation
);
75 *annotation_len_out
= r
;
77 *annotation_len_out
= 0;
80 md
->off
= (off_t
) ftell(f
);
81 written
= fwrite(md
->body
, 1, md
->bodylen
, f
);
82 if (written
!= md
->bodylen
) {
84 "Couldn't dump microdescriptor (wrote %lu out of %lu): %s",
85 (unsigned long)written
, (unsigned long)md
->bodylen
,
93 /** Holds a pointer to the current microdesc_cache_t object, or NULL if no
94 * such object has been allocated. */
95 static microdesc_cache_t
*the_microdesc_cache
= NULL
;
97 /** Return a pointer to the microdescriptor cache, loading it if necessary. */
99 get_microdesc_cache(void)
101 if (PREDICT_UNLIKELY(the_microdesc_cache
==NULL
)) {
102 microdesc_cache_t
*cache
= tor_malloc_zero(sizeof(microdesc_cache_t
));
103 HT_INIT(microdesc_map
, &cache
->map
);
104 cache
->cache_fname
= get_datadir_fname("cached-microdescs");
105 cache
->journal_fname
= get_datadir_fname("cached-microdescs.new");
106 microdesc_cache_reload(cache
);
107 the_microdesc_cache
= cache
;
109 return the_microdesc_cache
;
112 /* There are three sources of microdescriptors:
113 1) Generated by us while acting as a directory authority.
114 2) Loaded from the cache on disk.
118 /** Decode the microdescriptors from the string starting at <b>s</b> and
119 * ending at <b>eos</b>, and store them in <b>cache</b>. If <b>no-save</b>,
120 * mark them as non-writable to disk. If <b>where</b> is SAVED_IN_CACHE,
121 * leave their bodies as pointers to the mmap'd cache. If where is
122 * <b>SAVED_NOWHERE</b>, do not allow annotations. Return a list of the added
123 * microdescriptors. */
125 microdescs_add_to_cache(microdesc_cache_t
*cache
,
126 const char *s
, const char *eos
, saved_location_t where
,
129 /*XXXX need an argument that sets last_listed as appropriate. */
131 smartlist_t
*descriptors
, *added
;
132 const int allow_annotations
= (where
!= SAVED_NOWHERE
);
133 const int copy_body
= (where
!= SAVED_IN_CACHE
);
135 descriptors
= microdescs_parse_from_string(s
, eos
,
139 added
= microdescs_add_list_to_cache(cache
, descriptors
, where
, no_save
);
140 smartlist_free(descriptors
);
144 /* As microdescs_add_to_cache, but takes a list of micrdescriptors instead of
145 * a string to encode. Frees any members of <b>descriptors</b> that it does
148 microdescs_add_list_to_cache(microdesc_cache_t
*cache
,
149 smartlist_t
*descriptors
, saved_location_t where
,
153 open_file_t
*open_file
= NULL
;
158 if (where
== SAVED_NOWHERE
&& !no_save
) {
159 f
= start_writing_to_stdio_file(cache
->journal_fname
,
160 OPEN_FLAGS_APPEND
|O_BINARY
,
163 log_warn(LD_DIR
, "Couldn't append to journal in %s: %s",
164 cache
->journal_fname
, strerror(errno
));
169 added
= smartlist_create();
170 SMARTLIST_FOREACH_BEGIN(descriptors
, microdesc_t
*, md
) {
172 md2
= HT_FIND(microdesc_map
, &cache
->map
, md
);
174 /* We already had this one. */
175 if (md2
->last_listed
< md
->last_listed
)
176 md2
->last_listed
= md
->last_listed
;
181 /* Okay, it's a new one. */
183 size_t annotation_len
;
184 size
= dump_microdescriptor(f
, md
, &annotation_len
);
186 /* XXX handle errors from dump_microdescriptor() */
187 /* log? return -1? die? coredump the universe? */
190 md
->saved_location
= SAVED_IN_JOURNAL
;
191 cache
->journal_len
+= size
;
193 md
->saved_location
= where
;
196 md
->no_save
= no_save
;
198 HT_INSERT(microdesc_map
, &cache
->map
, md
);
199 smartlist_add(added
, md
);
201 cache
->total_len_seen
+= md
->bodylen
;
202 } SMARTLIST_FOREACH_END(md
);
205 finish_writing_to_file(open_file
); /*XXX Check me.*/
208 size_t old_content_len
=
209 cache
->cache_content
? cache
->cache_content
->size
: 0;
210 if (cache
->journal_len
> 16384 + old_content_len
&&
211 cache
->journal_len
> old_content_len
* 2) {
212 microdesc_cache_rebuild(cache
);
219 /** Remove every microdescriptor in <b>cache</b>. */
221 microdesc_cache_clear(microdesc_cache_t
*cache
)
223 microdesc_t
**entry
, **next
;
224 for (entry
= HT_START(microdesc_map
, &cache
->map
); entry
; entry
= next
) {
225 microdesc_t
*md
= *entry
;
226 next
= HT_NEXT_RMV(microdesc_map
, &cache
->map
, entry
);
229 HT_CLEAR(microdesc_map
, &cache
->map
);
230 if (cache
->cache_content
) {
231 tor_munmap_file(cache
->cache_content
);
232 cache
->cache_content
= NULL
;
234 cache
->total_len_seen
= 0;
238 /** Reload the contents of <b>cache</b> from disk. If it is empty, load it
239 * for the first time. Return 0 on success, -1 on failure. */
241 microdesc_cache_reload(microdesc_cache_t
*cache
)
244 char *journal_content
;
249 microdesc_cache_clear(cache
);
251 mm
= cache
->cache_content
= tor_mmap_file(cache
->cache_fname
);
253 added
= microdescs_add_to_cache(cache
, mm
->data
, mm
->data
+mm
->size
,
256 total
+= smartlist_len(added
);
257 smartlist_free(added
);
261 journal_content
= read_file_to_str(cache
->journal_fname
,
262 RFTS_IGNORE_MISSING
, &st
);
263 if (journal_content
) {
264 added
= microdescs_add_to_cache(cache
, journal_content
,
265 journal_content
+st
.st_size
,
266 SAVED_IN_JOURNAL
, 0);
268 total
+= smartlist_len(added
);
269 smartlist_free(added
);
271 tor_free(journal_content
);
273 log_notice(LD_DIR
, "Reloaded microdescriptor cache. Found %d descriptors.",
278 /** Regenerate the main cache file for <b>cache</b>, clear the journal file,
279 * and update every microdesc_t in the cache with pointers to its new
282 microdesc_cache_rebuild(microdesc_cache_t
*cache
)
284 open_file_t
*open_file
;
290 int orig_size
, new_size
;
292 log_info(LD_DIR
, "Rebuilding the microdescriptor cache...");
293 orig_size
= (int)(cache
->cache_content
? cache
->cache_content
->size
: 0);
294 orig_size
+= (int)cache
->journal_len
;
296 f
= start_writing_to_stdio_file(cache
->cache_fname
,
297 OPEN_FLAGS_REPLACE
|O_BINARY
,
302 wrote
= smartlist_create();
304 HT_FOREACH(mdp
, microdesc_map
, &cache
->map
) {
305 microdesc_t
*md
= *mdp
;
306 size_t annotation_len
;
310 size
= dump_microdescriptor(f
, md
, &annotation_len
);
312 /* XXX handle errors from dump_microdescriptor() */
313 /* log? return -1? die? coredump the universe? */
316 md
->off
= off
+ annotation_len
;
318 if (md
->saved_location
!= SAVED_IN_CACHE
) {
320 md
->saved_location
= SAVED_IN_CACHE
;
322 smartlist_add(wrote
, md
);
325 finish_writing_to_file(open_file
); /*XXX Check me.*/
327 if (cache
->cache_content
)
328 tor_munmap_file(cache
->cache_content
);
329 cache
->cache_content
= tor_mmap_file(cache
->cache_fname
);
331 if (!cache
->cache_content
&& smartlist_len(wrote
)) {
332 log_err(LD_DIR
, "Couldn't map file that we just wrote to %s!",
334 smartlist_free(wrote
);
337 SMARTLIST_FOREACH_BEGIN(wrote
, microdesc_t
*, md
) {
338 tor_assert(md
->saved_location
== SAVED_IN_CACHE
);
339 md
->body
= (char*)cache
->cache_content
->data
+ md
->off
;
340 tor_assert(!memcmp(md
->body
, "onion-key", 9));
341 } SMARTLIST_FOREACH_END(md
);
343 smartlist_free(wrote
);
345 write_str_to_file(cache
->journal_fname
, "", 1);
346 cache
->journal_len
= 0;
348 new_size
= (int)cache
->cache_content
->size
;
349 log_info(LD_DIR
, "Done rebuilding microdesc cache. "
350 "Saved %d bytes; %d still used.",
351 orig_size
-new_size
, new_size
);
356 /** Deallocate a single microdescriptor. Note: the microdescriptor MUST have
357 * previously been removed from the cache if it had ever been inserted. */
359 microdesc_free(microdesc_t
*md
)
363 /* Must be removed from hash table! */
365 crypto_free_pk_env(md
->onion_pkey
);
366 if (md
->body
&& md
->saved_location
!= SAVED_IN_CACHE
)
370 SMARTLIST_FOREACH(md
->family
, char *, cp
, tor_free(cp
));
371 smartlist_free(md
->family
);
373 tor_free(md
->exitsummary
);
378 /** Free all storage held in the microdesc.c module. */
380 microdesc_free_all(void)
382 if (the_microdesc_cache
) {
383 microdesc_cache_clear(the_microdesc_cache
);
384 tor_free(the_microdesc_cache
->cache_fname
);
385 tor_free(the_microdesc_cache
->journal_fname
);
386 tor_free(the_microdesc_cache
);
390 /** If there is a microdescriptor in <b>cache</b> whose sha256 digest is
391 * <b>d</b>, return it. Otherwise return NULL. */
393 microdesc_cache_lookup_by_digest256(microdesc_cache_t
*cache
, const char *d
)
395 microdesc_t
*md
, search
;
397 cache
= get_microdesc_cache();
398 memcpy(search
.digest
, d
, DIGEST256_LEN
);
399 md
= HT_FIND(microdesc_map
, &cache
->map
, &search
);
403 /** Return the mean size of decriptors added to <b>cache</b> since it was last
404 * cleared. Used to estimate the size of large downloads. */
406 microdesc_average_size(microdesc_cache_t
*cache
)
409 cache
= get_microdesc_cache();
412 return (size_t)(cache
->total_len_seen
/ cache
->n_seen
);