1 /* Copyright (c) 2009, The Tor Project, Inc. */
2 /* See LICENSE for licensing information */
6 /** A data structure to hold a bunch of cached microdescriptors. There are
7 * two active files in the cache: a "cache file" that we mmap, and a "journal
8 * file" that we append to. Periodically, we rebuild the cache file to hold
9 * only the microdescriptors that we want to keep */
10 struct microdesc_cache_t
{
11 /** Map from sha256-digest to microdesc_t for every microdesc_t in the
13 HT_HEAD(microdesc_map
, microdesc_t
) map
;
15 /** Name of the cache file. */
17 /** Name of the journal file. */
19 /** Mmap'd contents of the cache file, or NULL if there is none. */
20 tor_mmap_t
*cache_content
;
21 /** Number of bytes used in the journal file. */
24 /** Total bytes of microdescriptor bodies we have added to this cache */
25 uint64_t total_len_seen
;
26 /** Total number of microdescriptors we have added to this cache */
30 /** Helper: computes a hash of <b>md</b> to place it in a hash table. */
31 static INLINE
unsigned int
32 _microdesc_hash(microdesc_t
*md
)
34 unsigned *d
= (unsigned*)md
->digest
;
36 return d
[0] ^ d
[1] ^ d
[2] ^ d
[3] ^ d
[4] ^ d
[5] ^ d
[6] ^ d
[7];
38 return d
[0] ^ d
[1] ^ d
[2] ^ d
[3];
42 /** Helper: compares <b>a</b> and </b> for equality for hash-table purposes. */
44 _microdesc_eq(microdesc_t
*a
, microdesc_t
*b
)
46 return !memcmp(a
->digest
, b
->digest
, DIGEST256_LEN
);
49 HT_PROTOTYPE(microdesc_map
, microdesc_t
, node
,
50 _microdesc_hash
, _microdesc_eq
);
51 HT_GENERATE(microdesc_map
, microdesc_t
, node
,
52 _microdesc_hash
, _microdesc_eq
, 0.6,
53 _tor_malloc
, _tor_realloc
, _tor_free
);
55 /** Write the body of <b>md</b> into <b>f</b>, with appropriate annotations.
56 * On success, return the total number of bytes written, and set
57 * *<b>annotation_len_out</b> to the number of bytes written as
60 dump_microdescriptor(FILE *f
, microdesc_t
*md
, size_t *annotation_len_out
)
64 /* XXXX drops unkown annotations. */
65 if (md
->last_listed
) {
66 char buf
[ISO_TIME_LEN
+1];
67 char annotation
[ISO_TIME_LEN
+32];
68 format_iso_time(buf
, md
->last_listed
);
69 tor_snprintf(annotation
, sizeof(annotation
), "@last-listed %s\n", buf
);
71 r
+= strlen(annotation
);
72 *annotation_len_out
= r
;
74 *annotation_len_out
= 0;
77 md
->off
= (off_t
) ftell(f
);
78 written
= fwrite(md
->body
, 1, md
->bodylen
, f
);
79 if (written
!= md
->bodylen
) {
81 "Couldn't dump microdescriptor (wrote %lu out of %lu): %s",
82 (unsigned long)written
, (unsigned long)md
->bodylen
,
90 /** Holds a pointer to the current microdesc_cache_t object, or NULL if no
91 * such object has been allocated. */
92 static microdesc_cache_t
*the_microdesc_cache
= NULL
;
94 /** Return a pointer to the microdescriptor cache, loading it if necessary. */
96 get_microdesc_cache(void)
98 if (PREDICT_UNLIKELY(the_microdesc_cache
==NULL
)) {
99 microdesc_cache_t
*cache
= tor_malloc_zero(sizeof(microdesc_cache_t
));
100 HT_INIT(microdesc_map
, &cache
->map
);
101 cache
->cache_fname
= get_datadir_fname("cached-microdescs");
102 cache
->journal_fname
= get_datadir_fname("cached-microdescs.new");
103 microdesc_cache_reload(cache
);
104 the_microdesc_cache
= cache
;
106 return the_microdesc_cache
;
109 /* There are three sources of microdescriptors:
110 1) Generated by us while acting as a directory authority.
111 2) Loaded from the cache on disk.
115 /** Decode the microdescriptors from the string starting at <b>s</b> and
116 * ending at <b>eos</b>, and store them in <b>cache</b>. If <b>no-save</b>,
117 * mark them as non-writable to disk. If <b>where</b> is SAVED_IN_CACHE,
118 * leave their bodies as pointers to the mmap'd cache. If where is
119 * <b>SAVED_NOWHERE</b>, do not allow annotations. Return a list of the added
120 * microdescriptors. */
122 microdescs_add_to_cache(microdesc_cache_t
*cache
,
123 const char *s
, const char *eos
, saved_location_t where
,
126 /*XXXX need an argument that sets last_listed as appropriate. */
128 smartlist_t
*descriptors
, *added
;
129 const int allow_annotations
= (where
!= SAVED_NOWHERE
);
130 const int copy_body
= (where
!= SAVED_IN_CACHE
);
132 descriptors
= microdescs_parse_from_string(s
, eos
,
136 added
= microdescs_add_list_to_cache(cache
, descriptors
, where
, no_save
);
137 smartlist_free(descriptors
);
141 /* As microdescs_add_to_cache, but takes a list of micrdescriptors instead of
142 * a string to encode. Frees any members of <b>descriptors</b> that it does
145 microdescs_add_list_to_cache(microdesc_cache_t
*cache
,
146 smartlist_t
*descriptors
, saved_location_t where
,
150 open_file_t
*open_file
= NULL
;
155 if (where
== SAVED_NOWHERE
&& !no_save
) {
156 f
= start_writing_to_stdio_file(cache
->journal_fname
,
157 OPEN_FLAGS_APPEND
|O_BINARY
,
160 log_warn(LD_DIR
, "Couldn't append to journal in %s: %s",
161 cache
->journal_fname
, strerror(errno
));
166 added
= smartlist_create();
167 SMARTLIST_FOREACH_BEGIN(descriptors
, microdesc_t
*, md
) {
169 md2
= HT_FIND(microdesc_map
, &cache
->map
, md
);
171 /* We already had this one. */
172 if (md2
->last_listed
< md
->last_listed
)
173 md2
->last_listed
= md
->last_listed
;
178 /* Okay, it's a new one. */
180 size_t annotation_len
;
181 size
= dump_microdescriptor(f
, md
, &annotation_len
);
183 /* XXX handle errors from dump_microdescriptor() */
184 /* log? return -1? die? coredump the universe? */
187 md
->saved_location
= SAVED_IN_JOURNAL
;
188 cache
->journal_len
+= size
;
190 md
->saved_location
= where
;
193 md
->no_save
= no_save
;
195 HT_INSERT(microdesc_map
, &cache
->map
, md
);
196 smartlist_add(added
, md
);
198 cache
->total_len_seen
+= md
->bodylen
;
199 } SMARTLIST_FOREACH_END(md
);
202 finish_writing_to_file(open_file
); /*XXX Check me.*/
205 size_t old_content_len
=
206 cache
->cache_content
? cache
->cache_content
->size
: 0;
207 if (cache
->journal_len
> 16384 + old_content_len
&&
208 cache
->journal_len
> old_content_len
* 2) {
209 microdesc_cache_rebuild(cache
);
216 /** Remove every microdescriptor in <b>cache</b>. */
218 microdesc_cache_clear(microdesc_cache_t
*cache
)
220 microdesc_t
**entry
, **next
;
221 for (entry
= HT_START(microdesc_map
, &cache
->map
); entry
; entry
= next
) {
222 microdesc_t
*md
= *entry
;
223 next
= HT_NEXT_RMV(microdesc_map
, &cache
->map
, entry
);
226 HT_CLEAR(microdesc_map
, &cache
->map
);
227 if (cache
->cache_content
) {
228 tor_munmap_file(cache
->cache_content
);
229 cache
->cache_content
= NULL
;
231 cache
->total_len_seen
= 0;
235 /** Reload the contents of <b>cache</b> from disk. If it is empty, load it
236 * for the first time. Return 0 on success, -1 on failure. */
238 microdesc_cache_reload(microdesc_cache_t
*cache
)
241 char *journal_content
;
246 microdesc_cache_clear(cache
);
248 mm
= cache
->cache_content
= tor_mmap_file(cache
->cache_fname
);
250 added
= microdescs_add_to_cache(cache
, mm
->data
, mm
->data
+mm
->size
,
253 total
+= smartlist_len(added
);
254 smartlist_free(added
);
258 journal_content
= read_file_to_str(cache
->journal_fname
,
259 RFTS_IGNORE_MISSING
, &st
);
260 if (journal_content
) {
261 added
= microdescs_add_to_cache(cache
, journal_content
,
262 journal_content
+st
.st_size
,
263 SAVED_IN_JOURNAL
, 0);
265 total
+= smartlist_len(added
);
266 smartlist_free(added
);
268 tor_free(journal_content
);
270 log_notice(LD_DIR
, "Reloaded microdescriptor cache. Found %d descriptors.",
275 /** Regenerate the main cache file for <b>cache</b>, clear the journal file,
276 * and update every microdesc_t in the cache with pointers to its new
279 microdesc_cache_rebuild(microdesc_cache_t
*cache
)
281 open_file_t
*open_file
;
287 int orig_size
, new_size
;
289 log_info(LD_DIR
, "Rebuilding the microdescriptor cache...");
290 orig_size
= (int)(cache
->cache_content
? cache
->cache_content
->size
: 0);
291 orig_size
+= (int)cache
->journal_len
;
293 f
= start_writing_to_stdio_file(cache
->cache_fname
,
294 OPEN_FLAGS_REPLACE
|O_BINARY
,
299 wrote
= smartlist_create();
301 HT_FOREACH(mdp
, microdesc_map
, &cache
->map
) {
302 microdesc_t
*md
= *mdp
;
303 size_t annotation_len
;
307 size
= dump_microdescriptor(f
, md
, &annotation_len
);
309 /* XXX handle errors from dump_microdescriptor() */
310 /* log? return -1? die? coredump the universe? */
313 md
->off
= off
+ annotation_len
;
315 if (md
->saved_location
!= SAVED_IN_CACHE
) {
317 md
->saved_location
= SAVED_IN_CACHE
;
319 smartlist_add(wrote
, md
);
322 finish_writing_to_file(open_file
); /*XXX Check me.*/
324 if (cache
->cache_content
)
325 tor_munmap_file(cache
->cache_content
);
326 cache
->cache_content
= tor_mmap_file(cache
->cache_fname
);
328 if (!cache
->cache_content
&& smartlist_len(wrote
)) {
329 log_err(LD_DIR
, "Couldn't map file that we just wrote to %s!",
331 smartlist_free(wrote
);
334 SMARTLIST_FOREACH_BEGIN(wrote
, microdesc_t
*, md
) {
335 tor_assert(md
->saved_location
== SAVED_IN_CACHE
);
336 md
->body
= (char*)cache
->cache_content
->data
+ md
->off
;
337 tor_assert(!memcmp(md
->body
, "onion-key", 9));
338 } SMARTLIST_FOREACH_END(md
);
340 smartlist_free(wrote
);
342 write_str_to_file(cache
->journal_fname
, "", 1);
343 cache
->journal_len
= 0;
345 new_size
= (int)cache
->cache_content
->size
;
346 log_info(LD_DIR
, "Done rebuilding microdesc cache. "
347 "Saved %d bytes; %d still used.",
348 orig_size
-new_size
, new_size
);
353 /** Deallocate a single microdescriptor. Note: the microdescriptor MUST have
354 * previously been removed from the cache if it had ever been inserted. */
356 microdesc_free(microdesc_t
*md
)
360 /* Must be removed from hash table! */
362 crypto_free_pk_env(md
->onion_pkey
);
363 if (md
->body
&& md
->saved_location
!= SAVED_IN_CACHE
)
367 SMARTLIST_FOREACH(md
->family
, char *, cp
, tor_free(cp
));
368 smartlist_free(md
->family
);
370 tor_free(md
->exitsummary
);
375 /** Free all storage held in the microdesc.c module. */
377 microdesc_free_all(void)
379 if (the_microdesc_cache
) {
380 microdesc_cache_clear(the_microdesc_cache
);
381 tor_free(the_microdesc_cache
->cache_fname
);
382 tor_free(the_microdesc_cache
->journal_fname
);
383 tor_free(the_microdesc_cache
);
387 /** If there is a microdescriptor in <b>cache</b> whose sha256 digest is
388 * <b>d</b>, return it. Otherwise return NULL. */
390 microdesc_cache_lookup_by_digest256(microdesc_cache_t
*cache
, const char *d
)
392 microdesc_t
*md
, search
;
394 cache
= get_microdesc_cache();
395 memcpy(search
.digest
, d
, DIGEST256_LEN
);
396 md
= HT_FIND(microdesc_map
, &cache
->map
, &search
);
400 /** Return the mean size of decriptors added to <b>cache</b> since it was last
401 * cleared. Used to estimate the size of large downloads. */
403 microdesc_average_size(microdesc_cache_t
*cache
)
406 cache
= get_microdesc_cache();
409 return (size_t)(cache
->total_len_seen
/ cache
->n_seen
);