1 /* Copyright (c) 2017, The Tor Project, Inc. */
2 /* See LICENSE for licensing information */
7 * \brief consensus diff manager functions
9 * This module is run by directory authorities and caches in order
10 * to remember a number of past consensus documents, and to generate
11 * and serve the diffs from those documents to the latest consensus.
14 #define CONSDIFFMGR_PRIVATE
18 #include "conscache.h"
20 #include "consdiffmgr.h"
21 #include "cpuworker.h"
22 #include "networkstatus.h"
23 #include "routerparse.h"
24 #include "workqueue.h"
27 * Labels to apply to items in the conscache object.
31 /* One of DOCTYPE_CONSENSUS or DOCTYPE_CONSENSUS_DIFF */
32 #define LABEL_DOCTYPE "document-type"
33 /* The valid-after time for a consensus (or for the target consensus of a
34 * diff), encoded as ISO UTC. */
35 #define LABEL_VALID_AFTER "consensus-valid-after"
36 /* The fresh-until time for a consensus (or for the target consensus of a
37 * diff), encoded as ISO UTC. */
38 #define LABEL_FRESH_UNTIL "consensus-fresh-until"
39 /* The valid-until time for a consensus (or for the target consensus of a
40 * diff), encoded as ISO UTC. */
41 #define LABEL_VALID_UNTIL "consensus-valid-until"
42 /* Comma-separated list of hex-encoded identity digests for the voting
44 #define LABEL_SIGNATORIES "consensus-signatories"
45 /* A hex encoded SHA3 digest of the object, as compressed (if any) */
46 #define LABEL_SHA3_DIGEST "sha3-digest"
47 /* A hex encoded SHA3 digest of the object before compression. */
48 #define LABEL_SHA3_DIGEST_UNCOMPRESSED "sha3-digest-uncompressed"
49 /* A hex encoded SHA3 digest-as-signed of a consensus */
50 #define LABEL_SHA3_DIGEST_AS_SIGNED "sha3-digest-as-signed"
51 /* The flavor of the consensus or consensuses diff */
52 #define LABEL_FLAVOR "consensus-flavor"
53 /* Diff only: the SHA3 digest-as-signed of the source consensus. */
54 #define LABEL_FROM_SHA3_DIGEST "from-sha3-digest"
55 /* Diff only: the SHA3 digest-in-full of the target consensus. */
56 #define LABEL_TARGET_SHA3_DIGEST "target-sha3-digest"
57 /* Diff only: the valid-after date of the source consensus. */
58 #define LABEL_FROM_VALID_AFTER "from-valid-after"
59 /* What kind of compression was used? */
60 #define LABEL_COMPRESSION_TYPE "compression"
63 #define DOCTYPE_CONSENSUS "consensus"
64 #define DOCTYPE_CONSENSUS_DIFF "consensus-diff"
67 * Underlying directory that stores consensuses and consensus diffs. Don't
68 * use this directly: use cdm_cache_get() instead.
70 static consensus_cache_t
*cons_diff_cache
= NULL
;
72 * If true, we have learned at least one new consensus since the
73 * consensus cache was last up-to-date.
75 static int cdm_cache_dirty
= 0;
77 * If true, we have scanned the cache to update our hashtable of diffs.
79 static int cdm_cache_loaded
= 0;
82 * Possible status values for cdm_diff_t.cdm_diff_status
84 typedef enum cdm_diff_status_t
{
86 CDM_DIFF_IN_PROGRESS
=2,
90 /** Which methods do we use for precompressing diffs? */
91 static const compress_method_t compress_diffs_with
[] = {
102 /** How many different methods will we try to use for diff compression? */
104 n_diff_compression_methods(void)
106 return ARRAY_LENGTH(compress_diffs_with
);
109 /** Which methods do we use for precompressing consensuses? */
110 static const compress_method_t compress_consensus_with
[] = {
120 /** How many different methods will we try to use for diff compression? */
122 n_consensus_compression_methods(void)
124 return ARRAY_LENGTH(compress_consensus_with
);
127 /** For which compression method do we retain old consensuses? There's no
128 * need to keep all of them, since we won't be serving them. We'll
129 * go with ZLIB_METHOD because it's pretty fast and everyone has it.
131 #define RETAIN_CONSENSUS_COMPRESSED_WITH_METHOD ZLIB_METHOD
133 /** Handles pointing to the latest consensus entries as compressed and
135 static consensus_cache_entry_handle_t
*
136 latest_consensus
[N_CONSENSUS_FLAVORS
]
137 [ARRAY_LENGTH(compress_consensus_with
)];
139 /** Hashtable node used to remember the current status of the diff
140 * from a given sha3 digest to the current consensus. */
141 typedef struct cdm_diff_t
{
142 HT_ENTRY(cdm_diff_t
) node
;
144 /** Consensus flavor for this diff (part of ht key) */
145 consensus_flavor_t flavor
;
146 /** SHA3-256 digest of the consensus that this diff is _from_. (part of the
148 uint8_t from_sha3
[DIGEST256_LEN
];
149 /** Method by which the diff is compressed. (part of the ht key */
150 compress_method_t compress_method
;
152 /** One of the CDM_DIFF_* values, depending on whether this diff
153 * is available, in progress, or impossible to compute. */
154 cdm_diff_status_t cdm_diff_status
;
155 /** SHA3-256 digest of the consensus that this diff is _to. */
156 uint8_t target_sha3
[DIGEST256_LEN
];
158 /** Handle to the cache entry for this diff, if any. We use a handle here
159 * to avoid thinking too hard about cache entry lifetime issues. */
160 consensus_cache_entry_handle_t
*entry
;
163 /** Hashtable mapping flavor and source consensus digest to status. */
164 static HT_HEAD(cdm_diff_ht
, cdm_diff_t
) cdm_diff_ht
= HT_INITIALIZER();
167 * Configuration for this module
169 static consdiff_cfg_t consdiff_cfg
= {
170 // XXXX I'd like to make this number bigger, but it interferes with the
171 // XXXX seccomp2 syscall filter, which tops out at BPF_MAXINS (4096)
173 /* .cache_max_num = */ 128
176 static int consdiffmgr_ensure_space_for_files(int n
);
177 static int consensus_queue_compression_work(const char *consensus
,
178 const networkstatus_t
*as_parsed
);
179 static int consensus_diff_queue_diff_work(consensus_cache_entry_t
*diff_from
,
180 consensus_cache_entry_t
*diff_to
);
181 static void consdiffmgr_set_cache_flags(void);
187 /** Helper: hash the key of a cdm_diff_t. */
189 cdm_diff_hash(const cdm_diff_t
*diff
)
191 uint8_t tmp
[DIGEST256_LEN
+ 2];
192 memcpy(tmp
, diff
->from_sha3
, DIGEST256_LEN
);
193 tmp
[DIGEST256_LEN
] = (uint8_t) diff
->flavor
;
194 tmp
[DIGEST256_LEN
+1] = (uint8_t) diff
->compress_method
;
195 return (unsigned) siphash24g(tmp
, sizeof(tmp
));
197 /** Helper: compare two cdm_diff_t objects for key equality */
199 cdm_diff_eq(const cdm_diff_t
*diff1
, const cdm_diff_t
*diff2
)
201 return fast_memeq(diff1
->from_sha3
, diff2
->from_sha3
, DIGEST256_LEN
) &&
202 diff1
->flavor
== diff2
->flavor
&&
203 diff1
->compress_method
== diff2
->compress_method
;
206 HT_PROTOTYPE(cdm_diff_ht
, cdm_diff_t
, node
, cdm_diff_hash
, cdm_diff_eq
)
207 HT_GENERATE2(cdm_diff_ht
, cdm_diff_t
, node
, cdm_diff_hash
, cdm_diff_eq
,
208 0.6, tor_reallocarray
, tor_free_
)
210 /** Release all storage held in <b>diff</b>. */
212 cdm_diff_free(cdm_diff_t
*diff
)
216 consensus_cache_entry_handle_free(diff
->entry
);
220 /** Create and return a new cdm_diff_t with the given values. Does not
221 * add it to the hashtable. */
223 cdm_diff_new(consensus_flavor_t flav
,
224 const uint8_t *from_sha3
,
225 const uint8_t *target_sha3
,
226 compress_method_t method
)
229 ent
= tor_malloc_zero(sizeof(cdm_diff_t
));
231 memcpy(ent
->from_sha3
, from_sha3
, DIGEST256_LEN
);
232 memcpy(ent
->target_sha3
, target_sha3
, DIGEST256_LEN
);
233 ent
->compress_method
= method
;
238 * Examine the diff hashtable to see whether we know anything about computing
239 * a diff of type <b>flav</b> between consensuses with the two provided
240 * SHA3-256 digests. If a computation is in progress, or if the computation
241 * has already been tried and failed, return 1. Otherwise, note the
242 * computation as "in progress" so that we don't reattempt it later, and
246 cdm_diff_ht_check_and_note_pending(consensus_flavor_t flav
,
247 const uint8_t *from_sha3
,
248 const uint8_t *target_sha3
)
250 struct cdm_diff_t search
, *ent
;
253 for (u
= 0; u
< n_diff_compression_methods(); ++u
) {
254 compress_method_t method
= compress_diffs_with
[u
];
255 memset(&search
, 0, sizeof(cdm_diff_t
));
256 search
.flavor
= flav
;
257 search
.compress_method
= method
;
258 memcpy(search
.from_sha3
, from_sha3
, DIGEST256_LEN
);
259 ent
= HT_FIND(cdm_diff_ht
, &cdm_diff_ht
, &search
);
261 tor_assert_nonfatal(ent
->cdm_diff_status
!= CDM_DIFF_PRESENT
);
265 ent
= cdm_diff_new(flav
, from_sha3
, target_sha3
, method
);
266 ent
->cdm_diff_status
= CDM_DIFF_IN_PROGRESS
;
267 HT_INSERT(cdm_diff_ht
, &cdm_diff_ht
, ent
);
273 * Update the status of the diff of type <b>flav</b> between consensuses with
274 * the two provided SHA3-256 digests, so that its status becomes
275 * <b>status</b>, and its value becomes the <b>handle</b>. If <b>handle</b>
276 * is NULL, then the old handle (if any) is freed, and replaced with NULL.
279 cdm_diff_ht_set_status(consensus_flavor_t flav
,
280 const uint8_t *from_sha3
,
281 const uint8_t *to_sha3
,
282 compress_method_t method
,
284 consensus_cache_entry_handle_t
*handle
)
286 struct cdm_diff_t search
, *ent
;
287 memset(&search
, 0, sizeof(cdm_diff_t
));
288 search
.flavor
= flav
;
289 search
.compress_method
= method
,
290 memcpy(search
.from_sha3
, from_sha3
, DIGEST256_LEN
);
291 ent
= HT_FIND(cdm_diff_ht
, &cdm_diff_ht
, &search
);
293 ent
= cdm_diff_new(flav
, from_sha3
, to_sha3
, method
);
294 ent
->cdm_diff_status
= CDM_DIFF_IN_PROGRESS
;
295 HT_INSERT(cdm_diff_ht
, &cdm_diff_ht
, ent
);
296 } else if (fast_memneq(ent
->target_sha3
, to_sha3
, DIGEST256_LEN
)) {
297 // This can happen under certain really pathological conditions
298 // if we decide we don't care about a diff before it is actually
303 tor_assert_nonfatal(ent
->cdm_diff_status
== CDM_DIFF_IN_PROGRESS
);
305 ent
->cdm_diff_status
= status
;
306 consensus_cache_entry_handle_free(ent
->entry
);
311 * Helper: Remove from the hash table every present (actually computed) diff
312 * of type <b>flav</b> whose target digest does not match
313 * <b>unless_target_sha3_matches</b>.
315 * This function is used for the hash table to throw away references to diffs
316 * that do not lead to the most given consensus of a given flavor.
319 cdm_diff_ht_purge(consensus_flavor_t flav
,
320 const uint8_t *unless_target_sha3_matches
)
322 cdm_diff_t
**diff
, **next
;
323 for (diff
= HT_START(cdm_diff_ht
, &cdm_diff_ht
); diff
; diff
= next
) {
324 cdm_diff_t
*this = *diff
;
326 if ((*diff
)->cdm_diff_status
== CDM_DIFF_PRESENT
&&
327 flav
== (*diff
)->flavor
) {
329 if (BUG((*diff
)->entry
== NULL
) ||
330 consensus_cache_entry_handle_get((*diff
)->entry
) == NULL
) {
331 /* the underlying entry has gone away; drop this. */
332 next
= HT_NEXT_RMV(cdm_diff_ht
, &cdm_diff_ht
, diff
);
337 if (unless_target_sha3_matches
&&
338 fast_memneq(unless_target_sha3_matches
, (*diff
)->target_sha3
,
340 /* target hash doesn't match; drop this. */
341 next
= HT_NEXT_RMV(cdm_diff_ht
, &cdm_diff_ht
, diff
);
346 next
= HT_NEXT(cdm_diff_ht
, &cdm_diff_ht
, diff
);
351 * Helper: initialize <b>cons_diff_cache</b>.
356 unsigned n_entries
= consdiff_cfg
.cache_max_num
* 2;
358 tor_assert(cons_diff_cache
== NULL
);
359 cons_diff_cache
= consensus_cache_open("diff-cache", n_entries
);
360 if (cons_diff_cache
== NULL
) {
362 log_err(LD_FS
, "Error: Couldn't open storage for consensus diffs.");
363 tor_assert_unreached();
366 consdiffmgr_set_cache_flags();
369 cdm_cache_loaded
= 0;
373 * Helper: return the consensus_cache_t * that backs this manager,
374 * initializing it if needed.
376 STATIC consensus_cache_t
*
379 if (PREDICT_UNLIKELY(cons_diff_cache
== NULL
)) {
382 return cons_diff_cache
;
386 * Helper: given a list of labels, prepend the hex-encoded SHA3 digest
387 * of the <b>bodylen</b>-byte object at <b>body</b> to those labels,
388 * with <b>label</b> as its label.
391 cdm_labels_prepend_sha3(config_line_t
**labels
,
396 uint8_t sha3_digest
[DIGEST256_LEN
];
397 char hexdigest
[HEX_DIGEST256_LEN
+1];
398 crypto_digest256((char *)sha3_digest
,
399 (const char *)body
, bodylen
, DIGEST_SHA3_256
);
400 base16_encode(hexdigest
, sizeof(hexdigest
),
401 (const char *)sha3_digest
, sizeof(sha3_digest
));
403 config_line_prepend(labels
, label
, hexdigest
);
406 /** Helper: if there is a sha3-256 hex-encoded digest in <b>ent</b> with the
407 * given label, set <b>digest_out</b> to that value (decoded), and return 0.
409 * Return -1 if there is no such label, and -2 if it is badly formatted. */
411 cdm_entry_get_sha3_value(uint8_t *digest_out
,
412 consensus_cache_entry_t
*ent
,
418 const char *hex
= consensus_cache_entry_get_value(ent
, label
);
422 int n
= base16_decode((char*)digest_out
, DIGEST256_LEN
, hex
, strlen(hex
));
423 if (n
!= DIGEST256_LEN
)
430 * Helper: look for a consensus with the given <b>flavor</b> and
431 * <b>valid_after</b> time in the cache. Return that consensus if it's
432 * present, or NULL if it's missing.
434 STATIC consensus_cache_entry_t
*
435 cdm_cache_lookup_consensus(consensus_flavor_t flavor
, time_t valid_after
)
437 char formatted_time
[ISO_TIME_LEN
+1];
438 format_iso_time_nospace(formatted_time
, valid_after
);
439 const char *flavname
= networkstatus_get_flavor_name(flavor
);
441 /* We'll filter by valid-after time first, since that should
442 * match the fewest documents. */
443 /* We could add an extra hashtable here, but since we only do this scan
444 * when adding a new consensus, it probably doesn't matter much. */
445 smartlist_t
*matches
= smartlist_new();
446 consensus_cache_find_all(matches
, cdm_cache_get(),
447 LABEL_VALID_AFTER
, formatted_time
);
448 consensus_cache_filter_list(matches
, LABEL_FLAVOR
, flavname
);
449 consensus_cache_filter_list(matches
, LABEL_DOCTYPE
, DOCTYPE_CONSENSUS
);
451 consensus_cache_entry_t
*result
= NULL
;
452 if (smartlist_len(matches
)) {
453 result
= smartlist_get(matches
, 0);
455 smartlist_free(matches
);
460 /** Return the maximum age (in seconds) of consensuses that we should consider
461 * storing. The available space in the directory may impose additional limits
462 * on how much we store. */
464 get_max_age_to_cache(void)
466 const int32_t DEFAULT_MAX_AGE_TO_CACHE
= 8192;
467 const int32_t MIN_MAX_AGE_TO_CACHE
= 0;
468 const int32_t MAX_MAX_AGE_TO_CACHE
= 8192;
469 const char MAX_AGE_TO_CACHE_NAME
[] = "max-consensus-age-to-cache-for-diff";
471 const or_options_t
*options
= get_options();
473 if (options
->MaxConsensusAgeForDiffs
) {
474 const int v
= options
->MaxConsensusAgeForDiffs
;
475 if (v
>= MAX_MAX_AGE_TO_CACHE
* 3600)
476 return MAX_MAX_AGE_TO_CACHE
;
481 /* The parameter is in hours, so we multiply */
482 return 3600 * networkstatus_get_param(NULL
,
483 MAX_AGE_TO_CACHE_NAME
,
484 DEFAULT_MAX_AGE_TO_CACHE
,
485 MIN_MAX_AGE_TO_CACHE
,
486 MAX_MAX_AGE_TO_CACHE
);
490 * Given a string containing a networkstatus consensus, and the results of
491 * having parsed that consensus, add that consensus to the cache if it is not
492 * already present and not too old. Create new consensus diffs from or to
493 * that consensus as appropriate.
495 * Return 0 on success and -1 on failure.
498 consdiffmgr_add_consensus(const char *consensus
,
499 const networkstatus_t
*as_parsed
)
501 if (BUG(consensus
== NULL
) || BUG(as_parsed
== NULL
))
502 return -1; // LCOV_EXCL_LINE
503 if (BUG(as_parsed
->type
!= NS_TYPE_CONSENSUS
))
504 return -1; // LCOV_EXCL_LINE
506 const consensus_flavor_t flavor
= as_parsed
->flavor
;
507 const time_t valid_after
= as_parsed
->valid_after
;
509 if (valid_after
< approx_time() - get_max_age_to_cache()) {
510 log_info(LD_DIRSERV
, "We don't care about this consensus document; it's "
515 /* Do we already have this one? */
516 consensus_cache_entry_t
*entry
=
517 cdm_cache_lookup_consensus(flavor
, valid_after
);
519 log_info(LD_DIRSERV
, "We already have a copy of that consensus");
523 /* We don't have it. Add it to the cache. */
524 return consensus_queue_compression_work(consensus
, as_parsed
);
528 * Helper: used to sort two smartlists of consensus_cache_entry_t by their
529 * LABEL_VALID_AFTER labels.
532 compare_by_valid_after_(const void **a
, const void **b
)
534 const consensus_cache_entry_t
*e1
= *a
;
535 const consensus_cache_entry_t
*e2
= *b
;
536 /* We're in luck here: sorting UTC iso-encoded values lexically will work
537 * fine (until 9999). */
538 return strcmp_opt(consensus_cache_entry_get_value(e1
, LABEL_VALID_AFTER
),
539 consensus_cache_entry_get_value(e2
, LABEL_VALID_AFTER
));
543 * Helper: Sort <b>lst</b> by LABEL_VALID_AFTER and return the most recent
546 static consensus_cache_entry_t
*
547 sort_and_find_most_recent(smartlist_t
*lst
)
549 smartlist_sort(lst
, compare_by_valid_after_
);
550 if (smartlist_len(lst
)) {
551 return smartlist_get(lst
, smartlist_len(lst
) - 1);
557 /** Return i such that compress_consensus_with[i] == method. Return
558 * -1 if no such i exists. */
560 consensus_compression_method_pos(compress_method_t method
)
563 for (i
= 0; i
< n_consensus_compression_methods(); ++i
) {
564 if (compress_consensus_with
[i
] == method
) {
572 * If we know a consensus with the flavor <b>flavor</b> compressed with
573 * <b>method</b>, set *<b>entry_out</b> to that value. Return values are as
574 * for consdiffmgr_find_diff_from().
577 consdiffmgr_find_consensus(struct consensus_cache_entry_t
**entry_out
,
578 consensus_flavor_t flavor
,
579 compress_method_t method
)
581 tor_assert(entry_out
);
582 tor_assert((int)flavor
< N_CONSENSUS_FLAVORS
);
584 int pos
= consensus_compression_method_pos(method
);
586 // We don't compress consensuses with this method.
587 return CONSDIFF_NOT_FOUND
;
589 consensus_cache_entry_handle_t
*handle
= latest_consensus
[flavor
][pos
];
591 return CONSDIFF_NOT_FOUND
;
592 *entry_out
= consensus_cache_entry_handle_get(handle
);
594 return CONSDIFF_AVAILABLE
;
596 return CONSDIFF_NOT_FOUND
;
600 * Look up consensus_cache_entry_t for the consensus of type <b>flavor</b>,
601 * from the source consensus with the specified digest (which must be SHA3).
603 * If the diff is present, store it into *<b>entry_out</b> and return
604 * CONSDIFF_AVAILABLE. Otherwise return CONSDIFF_NOT_FOUND or
605 * CONSDIFF_IN_PROGRESS.
608 consdiffmgr_find_diff_from(consensus_cache_entry_t
**entry_out
,
609 consensus_flavor_t flavor
,
611 const uint8_t *digest
,
613 compress_method_t method
)
615 if (BUG(digest_type
!= DIGEST_SHA3_256
) ||
616 BUG(digestlen
!= DIGEST256_LEN
)) {
617 return CONSDIFF_NOT_FOUND
; // LCOV_EXCL_LINE
620 // Try to look up the entry in the hashtable.
621 cdm_diff_t search
, *ent
;
622 memset(&search
, 0, sizeof(search
));
623 search
.flavor
= flavor
;
624 search
.compress_method
= method
;
625 memcpy(search
.from_sha3
, digest
, DIGEST256_LEN
);
626 ent
= HT_FIND(cdm_diff_ht
, &cdm_diff_ht
, &search
);
629 ent
->cdm_diff_status
== CDM_DIFF_ERROR
) {
630 return CONSDIFF_NOT_FOUND
;
631 } else if (ent
->cdm_diff_status
== CDM_DIFF_IN_PROGRESS
) {
632 return CONSDIFF_IN_PROGRESS
;
633 } else if (BUG(ent
->cdm_diff_status
!= CDM_DIFF_PRESENT
)) {
634 return CONSDIFF_IN_PROGRESS
;
637 if (BUG(ent
->entry
== NULL
)) {
638 return CONSDIFF_NOT_FOUND
;
640 *entry_out
= consensus_cache_entry_handle_get(ent
->entry
);
641 return (*entry_out
) ? CONSDIFF_AVAILABLE
: CONSDIFF_NOT_FOUND
;
644 // XXXX Remove this. I'm keeping it around for now in case we need to
645 // XXXX debug issues in the hashtable.
646 char hex
[HEX_DIGEST256_LEN
+1];
647 base16_encode(hex
, sizeof(hex
), (const char *)digest
, digestlen
);
648 const char *flavname
= networkstatus_get_flavor_name(flavor
);
650 smartlist_t
*matches
= smartlist_new();
651 consensus_cache_find_all(matches
, cdm_cache_get(),
652 LABEL_FROM_SHA3_DIGEST
, hex
);
653 consensus_cache_filter_list(matches
, LABEL_FLAVOR
, flavname
);
654 consensus_cache_filter_list(matches
, LABEL_DOCTYPE
, DOCTYPE_CONSENSUS_DIFF
);
656 *entry_out
= sort_and_find_most_recent(matches
);
657 consdiff_status_t result
=
658 (*entry_out
) ? CONSDIFF_AVAILABLE
: CONSDIFF_NOT_FOUND
;
659 smartlist_free(matches
);
666 * Perform periodic cleanup tasks on the consensus diff cache. Return
667 * the number of objects marked for deletion.
670 consdiffmgr_cleanup(void)
672 smartlist_t
*objects
= smartlist_new();
673 smartlist_t
*consensuses
= smartlist_new();
674 smartlist_t
*diffs
= smartlist_new();
677 log_debug(LD_DIRSERV
, "Looking for consdiffmgr entries to remove");
679 // 1. Delete any consensus or diff or anything whose valid_after is too old.
680 const time_t valid_after_cutoff
= approx_time() - get_max_age_to_cache();
682 consensus_cache_find_all(objects
, cdm_cache_get(),
684 SMARTLIST_FOREACH_BEGIN(objects
, consensus_cache_entry_t
*, ent
) {
685 const char *lv_valid_after
=
686 consensus_cache_entry_get_value(ent
, LABEL_VALID_AFTER
);
687 if (! lv_valid_after
) {
688 log_debug(LD_DIRSERV
, "Ignoring entry because it had no %s label",
692 time_t valid_after
= 0;
693 if (parse_iso_time_nospace(lv_valid_after
, &valid_after
) < 0) {
694 log_debug(LD_DIRSERV
, "Ignoring entry because its %s value (%s) was "
695 "unparseable", LABEL_VALID_AFTER
, escaped(lv_valid_after
));
698 if (valid_after
< valid_after_cutoff
) {
699 log_debug(LD_DIRSERV
, "Deleting entry because its %s value (%s) was "
700 "too old", LABEL_VALID_AFTER
, lv_valid_after
);
701 consensus_cache_entry_mark_for_removal(ent
);
704 } SMARTLIST_FOREACH_END(ent
);
706 // 2. Delete all diffs that lead to a consensus whose valid-after is not the
708 for (int flav
= 0; flav
< N_CONSENSUS_FLAVORS
; ++flav
) {
709 const char *flavname
= networkstatus_get_flavor_name(flav
);
710 /* Determine the most recent consensus of this flavor */
711 consensus_cache_find_all(consensuses
, cdm_cache_get(),
712 LABEL_DOCTYPE
, DOCTYPE_CONSENSUS
);
713 consensus_cache_filter_list(consensuses
, LABEL_FLAVOR
, flavname
);
714 consensus_cache_entry_t
*most_recent
=
715 sort_and_find_most_recent(consensuses
);
716 if (most_recent
== NULL
)
718 const char *most_recent_sha3
=
719 consensus_cache_entry_get_value(most_recent
,
720 LABEL_SHA3_DIGEST_UNCOMPRESSED
);
721 if (BUG(most_recent_sha3
== NULL
))
722 continue; // LCOV_EXCL_LINE
724 /* consider all such-flavored diffs, and look to see if they match. */
725 consensus_cache_find_all(diffs
, cdm_cache_get(),
726 LABEL_DOCTYPE
, DOCTYPE_CONSENSUS_DIFF
);
727 consensus_cache_filter_list(diffs
, LABEL_FLAVOR
, flavname
);
728 SMARTLIST_FOREACH_BEGIN(diffs
, consensus_cache_entry_t
*, diff
) {
729 const char *this_diff_target_sha3
=
730 consensus_cache_entry_get_value(diff
, LABEL_TARGET_SHA3_DIGEST
);
731 if (!this_diff_target_sha3
)
733 if (strcmp(this_diff_target_sha3
, most_recent_sha3
)) {
734 consensus_cache_entry_mark_for_removal(diff
);
737 } SMARTLIST_FOREACH_END(diff
);
738 smartlist_clear(consensuses
);
739 smartlist_clear(diffs
);
742 // 3. Delete all consensuses except the most recent that are compressed with
743 // an un-preferred method.
744 for (int flav
= 0; flav
< N_CONSENSUS_FLAVORS
; ++flav
) {
745 const char *flavname
= networkstatus_get_flavor_name(flav
);
746 /* Determine the most recent consensus of this flavor */
747 consensus_cache_find_all(consensuses
, cdm_cache_get(),
748 LABEL_DOCTYPE
, DOCTYPE_CONSENSUS
);
749 consensus_cache_filter_list(consensuses
, LABEL_FLAVOR
, flavname
);
750 consensus_cache_entry_t
*most_recent
=
751 sort_and_find_most_recent(consensuses
);
752 if (most_recent
== NULL
)
754 const char *most_recent_sha3_uncompressed
=
755 consensus_cache_entry_get_value(most_recent
,
756 LABEL_SHA3_DIGEST_UNCOMPRESSED
);
757 const char *retain_methodname
= compression_method_get_name(
758 RETAIN_CONSENSUS_COMPRESSED_WITH_METHOD
);
760 if (BUG(most_recent_sha3_uncompressed
== NULL
))
762 SMARTLIST_FOREACH_BEGIN(consensuses
, consensus_cache_entry_t
*, ent
) {
763 const char *lv_sha3_uncompressed
=
764 consensus_cache_entry_get_value(ent
, LABEL_SHA3_DIGEST_UNCOMPRESSED
);
765 if (BUG(! lv_sha3_uncompressed
))
767 if (!strcmp(lv_sha3_uncompressed
, most_recent_sha3_uncompressed
))
768 continue; // This _is_ the most recent.
769 const char *lv_methodname
=
770 consensus_cache_entry_get_value(ent
, LABEL_COMPRESSION_TYPE
);
771 if (! lv_methodname
|| strcmp(lv_methodname
, retain_methodname
)) {
772 consensus_cache_entry_mark_for_removal(ent
);
775 } SMARTLIST_FOREACH_END(ent
);
778 smartlist_free(objects
);
779 smartlist_free(consensuses
);
780 smartlist_free(diffs
);
782 // Actually remove files, if they're not used.
783 consensus_cache_delete_pending(cdm_cache_get(), 0);
788 * Initialize the consensus diff manager and its cache, and configure
789 * its parameters based on the latest torrc and networkstatus parameters.
792 consdiffmgr_configure(const consdiff_cfg_t
*cfg
)
795 memcpy(&consdiff_cfg
, cfg
, sizeof(consdiff_cfg
));
797 (void) cdm_cache_get();
801 * Tell the sandbox (if any) configured by <b>cfg</b> to allow the
802 * operations that the consensus diff manager will need.
805 consdiffmgr_register_with_sandbox(struct sandbox_cfg_elem
**cfg
)
807 return consensus_cache_register_with_sandbox(cdm_cache_get(), cfg
);
811 * Scan the consensus diff manager's cache for any grossly malformed entries,
812 * and mark them as deletable. Return 0 if no problems were found; 1
813 * if problems were found and fixed.
816 consdiffmgr_validate(void)
818 /* Right now, we only check for entries that have bad sha3 values */
821 smartlist_t
*objects
= smartlist_new();
822 consensus_cache_find_all(objects
, cdm_cache_get(),
824 SMARTLIST_FOREACH_BEGIN(objects
, consensus_cache_entry_t
*, obj
) {
825 uint8_t sha3_expected
[DIGEST256_LEN
];
826 uint8_t sha3_received
[DIGEST256_LEN
];
827 int r
= cdm_entry_get_sha3_value(sha3_expected
, obj
, LABEL_SHA3_DIGEST
);
829 /* digest isn't there; that's allowed */
831 } else if (r
== -2) {
832 /* digest is malformed; that's not allowed */
834 consensus_cache_entry_mark_for_removal(obj
);
839 consensus_cache_entry_incref(obj
);
840 r
= consensus_cache_entry_get_body(obj
, &body
, &bodylen
);
842 crypto_digest256((char *)sha3_received
, (const char *)body
, bodylen
,
845 consensus_cache_entry_decref(obj
);
849 // Deconfuse coverity about the possibility of sha3_received being
853 if (fast_memneq(sha3_received
, sha3_expected
, DIGEST256_LEN
)) {
855 consensus_cache_entry_mark_for_removal(obj
);
859 } SMARTLIST_FOREACH_END(obj
);
860 smartlist_free(objects
);
865 * Helper: build new diffs of <b>flavor</b> as needed
868 consdiffmgr_rescan_flavor_(consensus_flavor_t flavor
)
870 smartlist_t
*matches
= NULL
;
871 smartlist_t
*diffs
= NULL
;
872 smartlist_t
*compute_diffs_from
= NULL
;
873 strmap_t
*have_diff_from
= NULL
;
875 // look for the most recent consensus, and for all previous in-range
876 // consensuses. Do they all have diffs to it?
877 const char *flavname
= networkstatus_get_flavor_name(flavor
);
879 // 1. find the most recent consensus, and the ones that we might want
881 const char *methodname
= compression_method_get_name(
882 RETAIN_CONSENSUS_COMPRESSED_WITH_METHOD
);
884 matches
= smartlist_new();
885 consensus_cache_find_all(matches
, cdm_cache_get(),
886 LABEL_FLAVOR
, flavname
);
887 consensus_cache_filter_list(matches
, LABEL_DOCTYPE
, DOCTYPE_CONSENSUS
);
888 consensus_cache_filter_list(matches
, LABEL_COMPRESSION_TYPE
, methodname
);
889 consensus_cache_entry_t
*most_recent
= sort_and_find_most_recent(matches
);
891 log_info(LD_DIRSERV
, "No 'most recent' %s consensus found; "
892 "not making diffs", flavname
);
895 tor_assert(smartlist_len(matches
));
896 smartlist_del(matches
, smartlist_len(matches
) - 1);
898 const char *most_recent_valid_after
=
899 consensus_cache_entry_get_value(most_recent
, LABEL_VALID_AFTER
);
900 if (BUG(most_recent_valid_after
== NULL
))
901 goto done
; //LCOV_EXCL_LINE
902 uint8_t most_recent_sha3
[DIGEST256_LEN
];
903 if (BUG(cdm_entry_get_sha3_value(most_recent_sha3
, most_recent
,
904 LABEL_SHA3_DIGEST_UNCOMPRESSED
) < 0))
905 goto done
; //LCOV_EXCL_LINE
907 // 2. Find all the relevant diffs _to_ this consensus. These are ones
908 // that we don't need to compute.
909 diffs
= smartlist_new();
910 consensus_cache_find_all(diffs
, cdm_cache_get(),
911 LABEL_VALID_AFTER
, most_recent_valid_after
);
912 consensus_cache_filter_list(diffs
, LABEL_DOCTYPE
, DOCTYPE_CONSENSUS_DIFF
);
913 consensus_cache_filter_list(diffs
, LABEL_FLAVOR
, flavname
);
914 have_diff_from
= strmap_new();
915 SMARTLIST_FOREACH_BEGIN(diffs
, consensus_cache_entry_t
*, diff
) {
916 const char *va
= consensus_cache_entry_get_value(diff
,
917 LABEL_FROM_VALID_AFTER
);
919 continue; // LCOV_EXCL_LINE
920 strmap_set(have_diff_from
, va
, diff
);
921 } SMARTLIST_FOREACH_END(diff
);
923 // 3. See which consensuses in 'matches' don't have diffs yet.
924 smartlist_reverse(matches
); // from newest to oldest.
925 compute_diffs_from
= smartlist_new();
926 SMARTLIST_FOREACH_BEGIN(matches
, consensus_cache_entry_t
*, ent
) {
927 const char *va
= consensus_cache_entry_get_value(ent
, LABEL_VALID_AFTER
);
929 continue; // LCOV_EXCL_LINE
930 if (strmap_get(have_diff_from
, va
) != NULL
)
931 continue; /* we already have this one. */
932 smartlist_add(compute_diffs_from
, ent
);
933 /* Since we are not going to serve this as the most recent consensus
934 * any more, we should stop keeping it mmap'd when it's not in use.
936 consensus_cache_entry_mark_for_aggressive_release(ent
);
937 } SMARTLIST_FOREACH_END(ent
);
940 "The most recent %s consensus is valid-after %s. We have diffs to "
941 "this consensus for %d/%d older %s consensuses. Generating diffs "
944 most_recent_valid_after
,
945 smartlist_len(matches
) - smartlist_len(compute_diffs_from
),
946 smartlist_len(matches
),
948 smartlist_len(compute_diffs_from
));
950 // 4. Update the hashtable; remove entries in this flavor to other
951 // target consensuses.
952 cdm_diff_ht_purge(flavor
, most_recent_sha3
);
954 // 5. Actually launch the requests.
955 SMARTLIST_FOREACH_BEGIN(compute_diffs_from
, consensus_cache_entry_t
*, c
) {
956 if (BUG(c
== most_recent
))
957 continue; // LCOV_EXCL_LINE
959 uint8_t this_sha3
[DIGEST256_LEN
];
960 if (cdm_entry_get_sha3_value(this_sha3
, c
,
961 LABEL_SHA3_DIGEST_AS_SIGNED
)<0) {
962 // Not actually a bug, since we might be running with a directory
963 // with stale files from before the #22143 fixes.
966 if (cdm_diff_ht_check_and_note_pending(flavor
,
967 this_sha3
, most_recent_sha3
)) {
968 // This is already pending, or we encountered an error.
971 consensus_diff_queue_diff_work(c
, most_recent
);
972 } SMARTLIST_FOREACH_END(c
);
975 smartlist_free(matches
);
976 smartlist_free(diffs
);
977 smartlist_free(compute_diffs_from
);
978 strmap_free(have_diff_from
, NULL
);
982 * Scan the cache for the latest consensuses and add their handles to
986 consdiffmgr_consensus_load(void)
988 smartlist_t
*matches
= smartlist_new();
989 for (int flav
= 0; flav
< N_CONSENSUS_FLAVORS
; ++flav
) {
990 const char *flavname
= networkstatus_get_flavor_name(flav
);
991 smartlist_clear(matches
);
992 consensus_cache_find_all(matches
, cdm_cache_get(),
993 LABEL_FLAVOR
, flavname
);
994 consensus_cache_filter_list(matches
, LABEL_DOCTYPE
, DOCTYPE_CONSENSUS
);
995 consensus_cache_entry_t
*most_recent
= sort_and_find_most_recent(matches
);
997 continue; // no consensuses.
998 const char *most_recent_sha3
=
999 consensus_cache_entry_get_value(most_recent
,
1000 LABEL_SHA3_DIGEST_UNCOMPRESSED
);
1001 if (BUG(most_recent_sha3
== NULL
))
1002 continue; // LCOV_EXCL_LINE
1003 consensus_cache_filter_list(matches
, LABEL_SHA3_DIGEST_UNCOMPRESSED
,
1006 // Everything that remains matches the most recent consensus of this
1008 SMARTLIST_FOREACH_BEGIN(matches
, consensus_cache_entry_t
*, ent
) {
1009 const char *lv_compression
=
1010 consensus_cache_entry_get_value(ent
, LABEL_COMPRESSION_TYPE
);
1011 compress_method_t method
=
1012 compression_method_get_by_name(lv_compression
);
1013 int pos
= consensus_compression_method_pos(method
);
1016 consensus_cache_entry_handle_free(latest_consensus
[flav
][pos
]);
1017 latest_consensus
[flav
][pos
] = consensus_cache_entry_handle_new(ent
);
1018 } SMARTLIST_FOREACH_END(ent
);
1020 smartlist_free(matches
);
1024 * Scan the cache for diffs, and add them to the hashtable.
1027 consdiffmgr_diffs_load(void)
1029 smartlist_t
*diffs
= smartlist_new();
1030 consensus_cache_find_all(diffs
, cdm_cache_get(),
1031 LABEL_DOCTYPE
, DOCTYPE_CONSENSUS_DIFF
);
1032 SMARTLIST_FOREACH_BEGIN(diffs
, consensus_cache_entry_t
*, diff
) {
1033 const char *lv_flavor
=
1034 consensus_cache_entry_get_value(diff
, LABEL_FLAVOR
);
1037 int flavor
= networkstatus_parse_flavor_name(lv_flavor
);
1040 const char *lv_compression
=
1041 consensus_cache_entry_get_value(diff
, LABEL_COMPRESSION_TYPE
);
1042 compress_method_t method
= NO_METHOD
;
1043 if (lv_compression
) {
1044 method
= compression_method_get_by_name(lv_compression
);
1045 if (method
== UNKNOWN_METHOD
) {
1050 uint8_t from_sha3
[DIGEST256_LEN
];
1051 uint8_t to_sha3
[DIGEST256_LEN
];
1052 if (cdm_entry_get_sha3_value(from_sha3
, diff
, LABEL_FROM_SHA3_DIGEST
)<0)
1054 if (cdm_entry_get_sha3_value(to_sha3
, diff
, LABEL_TARGET_SHA3_DIGEST
)<0)
1057 cdm_diff_ht_set_status(flavor
, from_sha3
, to_sha3
,
1060 consensus_cache_entry_handle_new(diff
));
1061 } SMARTLIST_FOREACH_END(diff
);
1062 smartlist_free(diffs
);
1066 * Build new diffs as needed.
1069 consdiffmgr_rescan(void)
1071 if (cdm_cache_dirty
== 0)
1074 // Clean up here to make room for new diffs, and to ensure that older
1075 // consensuses do not have any entries.
1076 consdiffmgr_cleanup();
1078 if (cdm_cache_loaded
== 0) {
1079 consdiffmgr_diffs_load();
1080 consdiffmgr_consensus_load();
1081 cdm_cache_loaded
= 1;
1084 for (int flav
= 0; flav
< N_CONSENSUS_FLAVORS
; ++flav
) {
1085 consdiffmgr_rescan_flavor_((consensus_flavor_t
) flav
);
1088 cdm_cache_dirty
= 0;
1092 * Helper: compare two files by their from-valid-after and valid-after labels,
1093 * trying to sort in ascending order by from-valid-after (when present) and
1094 * valid-after (when not). Place everything that has neither label first in
1098 compare_by_staleness_(const void **a
, const void **b
)
1100 const consensus_cache_entry_t
*e1
= *a
;
1101 const consensus_cache_entry_t
*e2
= *b
;
1102 const char *va1
, *fva1
, *va2
, *fva2
;
1103 va1
= consensus_cache_entry_get_value(e1
, LABEL_VALID_AFTER
);
1104 va2
= consensus_cache_entry_get_value(e2
, LABEL_VALID_AFTER
);
1105 fva1
= consensus_cache_entry_get_value(e1
, LABEL_FROM_VALID_AFTER
);
1106 fva2
= consensus_cache_entry_get_value(e2
, LABEL_FROM_VALID_AFTER
);
1113 /* See note about iso-encoded values in compare_by_valid_after_. Also note
1114 * that missing dates will get placed first. */
1115 return strcmp_opt(va1
, va2
);
1118 /** If there are not enough unused filenames to store <b>n</b> files, then
1119 * delete old consensuses until there are. (We have to keep track of the
1120 * number of filenames because of the way that the seccomp2 cache works.)
1122 * Return 0 on success, -1 on failure.
1125 consdiffmgr_ensure_space_for_files(int n
)
1127 consensus_cache_t
*cache
= cdm_cache_get();
1128 if (consensus_cache_get_n_filenames_available(cache
) >= n
) {
1129 // there are already enough unused filenames.
1132 // Try a cheap deletion of stuff that's waiting to get deleted.
1133 consensus_cache_delete_pending(cache
, 0);
1134 if (consensus_cache_get_n_filenames_available(cache
) >= n
) {
1135 // okay, _that_ made enough filenames available.
1138 // Let's get more assertive: clean out unused stuff, and force-remove
1139 // the files that we can.
1140 consdiffmgr_cleanup();
1141 consensus_cache_delete_pending(cache
, 1);
1142 const int n_to_remove
= n
- consensus_cache_get_n_filenames_available(cache
);
1143 if (n_to_remove
<= 0) {
1148 // At this point, we're going to have to throw out objects that will be
1150 smartlist_t
*objects
= smartlist_new();
1151 consensus_cache_find_all(objects
, cache
, NULL
, NULL
);
1152 smartlist_sort(objects
, compare_by_staleness_
);
1154 SMARTLIST_FOREACH_BEGIN(objects
, consensus_cache_entry_t
*, ent
) {
1155 consensus_cache_entry_mark_for_removal(ent
);
1156 if (++n_marked
>= n_to_remove
)
1158 } SMARTLIST_FOREACH_END(ent
);
1159 smartlist_free(objects
);
1161 consensus_cache_delete_pending(cache
, 1);
1163 if (consensus_cache_may_overallocate(cache
)) {
1164 /* If we're allowed to throw extra files into the cache, let's do so
1165 * rather getting upset.
1170 if (BUG(n_marked
< n_to_remove
))
1177 * Set consensus cache flags on the objects in this consdiffmgr.
1180 consdiffmgr_set_cache_flags(void)
1182 /* Right now, we just mark the consensus objects for aggressive release,
1183 * so that they get mmapped for as little time as possible. */
1184 smartlist_t
*objects
= smartlist_new();
1185 consensus_cache_find_all(objects
, cdm_cache_get(), LABEL_DOCTYPE
,
1187 SMARTLIST_FOREACH_BEGIN(objects
, consensus_cache_entry_t
*, ent
) {
1188 consensus_cache_entry_mark_for_aggressive_release(ent
);
1189 } SMARTLIST_FOREACH_END(ent
);
1190 smartlist_free(objects
);
1194 * Called before shutdown: drop all storage held by the consdiffmgr.c module.
1197 consdiffmgr_free_all(void)
1199 cdm_diff_t
**diff
, **next
;
1200 for (diff
= HT_START(cdm_diff_ht
, &cdm_diff_ht
); diff
; diff
= next
) {
1201 cdm_diff_t
*this = *diff
;
1202 next
= HT_NEXT_RMV(cdm_diff_ht
, &cdm_diff_ht
, diff
);
1203 cdm_diff_free(this);
1207 for (i
= 0; i
< N_CONSENSUS_FLAVORS
; ++i
) {
1208 for (j
= 0; j
< n_consensus_compression_methods(); ++j
) {
1209 consensus_cache_entry_handle_free(latest_consensus
[i
][j
]);
1212 memset(latest_consensus
, 0, sizeof(latest_consensus
));
1213 consensus_cache_free(cons_diff_cache
);
1214 cons_diff_cache
= NULL
;
1221 typedef struct compressed_result_t
{
1222 config_line_t
*labels
;
1224 * Output: Body of the diff, as compressed.
1228 * Output: length of body_out
1231 } compressed_result_t
;
1234 * Compress the bytestring <b>input</b> of length <b>len</b> using the
1235 * <n>n_methods</b> compression methods listed in the array <b>methods</b>.
1237 * For each successful compression, set the fields in the <b>results_out</b>
1238 * array in the position corresponding to the compression method. Use
1239 * <b>labels_in</b> as a basis for the labels of the result.
1241 * Return 0 if all compression succeeded; -1 if any failed.
1244 compress_multiple(compressed_result_t
*results_out
, int n_methods
,
1245 const compress_method_t
*methods
,
1246 const uint8_t *input
, size_t len
,
1247 const config_line_t
*labels_in
)
1251 for (i
= 0; i
< n_methods
; ++i
) {
1252 compress_method_t method
= methods
[i
];
1253 const char *methodname
= compression_method_get_name(method
);
1256 if (0 == tor_compress(&result
, &sz
, (const char*)input
, len
, method
)) {
1257 results_out
[i
].body
= (uint8_t*)result
;
1258 results_out
[i
].bodylen
= sz
;
1259 results_out
[i
].labels
= config_lines_dup(labels_in
);
1260 cdm_labels_prepend_sha3(&results_out
[i
].labels
, LABEL_SHA3_DIGEST
,
1261 results_out
[i
].body
,
1262 results_out
[i
].bodylen
);
1263 config_line_prepend(&results_out
[i
].labels
,
1264 LABEL_COMPRESSION_TYPE
,
1274 * Given an array of <b>n</b> compressed_result_t in <b>results</b>,
1275 * as produced by compress_multiple, store them all into the
1276 * consdiffmgr, and store handles to them in the <b>handles_out</b>
1279 * Return CDM_DIFF_PRESENT if any was stored, and CDM_DIFF_ERROR if none
1282 static cdm_diff_status_t
1283 store_multiple(consensus_cache_entry_handle_t
**handles_out
,
1285 const compress_method_t
*methods
,
1286 const compressed_result_t
*results
,
1287 const char *description
)
1289 cdm_diff_status_t status
= CDM_DIFF_ERROR
;
1290 consdiffmgr_ensure_space_for_files(n
);
1293 for (i
= 0; i
< n
; ++i
) {
1294 compress_method_t method
= methods
[i
];
1295 uint8_t *body_out
= results
[i
].body
;
1296 size_t bodylen_out
= results
[i
].bodylen
;
1297 config_line_t
*labels
= results
[i
].labels
;
1298 const char *methodname
= compression_method_get_name(method
);
1299 if (body_out
&& bodylen_out
&& labels
) {
1300 /* Success! Store the results */
1301 log_info(LD_DIRSERV
, "Adding %s, compressed with %s",
1302 description
, methodname
);
1304 consensus_cache_entry_t
*ent
=
1305 consensus_cache_add(cdm_cache_get(),
1309 if (BUG(ent
== NULL
))
1312 status
= CDM_DIFF_PRESENT
;
1313 handles_out
[i
] = consensus_cache_entry_handle_new(ent
);
1314 consensus_cache_entry_decref(ent
);
1321 * An object passed to a worker thread that will try to produce a consensus
1324 typedef struct consensus_diff_worker_job_t
{
1326 * Input: The consensus to compute the diff from. Holds a reference to the
1327 * cache entry, which must not be released until the job is passed back to
1328 * the main thread. The body must be mapped into memory in the main thread.
1330 consensus_cache_entry_t
*diff_from
;
1332 * Input: The consensus to compute the diff to. Holds a reference to the
1333 * cache entry, which must not be released until the job is passed back to
1334 * the main thread. The body must be mapped into memory in the main thread.
1336 consensus_cache_entry_t
*diff_to
;
1338 /** Output: labels and bodies */
1339 compressed_result_t out
[ARRAY_LENGTH(compress_diffs_with
)];
1340 } consensus_diff_worker_job_t
;
1342 /** Given a consensus_cache_entry_t, check whether it has a label claiming
1343 * that it was compressed. If so, uncompress its contents into <b>out</b> and
1344 * set <b>outlen</b> to hold their size. If not, just copy the body into
1345 * <b>out</b> and set <b>outlen</b> to its length. Return 0 on success,
1348 * In all cases, the output is nul-terminated. */
1350 uncompress_or_copy(char **out
, size_t *outlen
,
1351 consensus_cache_entry_t
*ent
)
1353 const uint8_t *body
;
1356 if (consensus_cache_entry_get_body(ent
, &body
, &bodylen
) < 0)
1359 const char *lv_compression
=
1360 consensus_cache_entry_get_value(ent
, LABEL_COMPRESSION_TYPE
);
1361 compress_method_t method
= NO_METHOD
;
1364 method
= compression_method_get_by_name(lv_compression
);
1366 return tor_uncompress(out
, outlen
, (const char *)body
, bodylen
,
1367 method
, 1, LOG_WARN
);
1371 * Worker function. This function runs inside a worker thread and receives
1372 * a consensus_diff_worker_job_t as its input.
1374 static workqueue_reply_t
1375 consensus_diff_worker_threadfn(void *state_
, void *work_
)
1378 consensus_diff_worker_job_t
*job
= work_
;
1379 const uint8_t *diff_from
, *diff_to
;
1380 size_t len_from
, len_to
;
1382 /* We need to have the body already mapped into RAM here.
1384 r
= consensus_cache_entry_get_body(job
->diff_from
, &diff_from
, &len_from
);
1386 return WQ_RPL_REPLY
; // LCOV_EXCL_LINE
1387 r
= consensus_cache_entry_get_body(job
->diff_to
, &diff_to
, &len_to
);
1389 return WQ_RPL_REPLY
; // LCOV_EXCL_LINE
1391 const char *lv_to_valid_after
=
1392 consensus_cache_entry_get_value(job
->diff_to
, LABEL_VALID_AFTER
);
1393 const char *lv_to_fresh_until
=
1394 consensus_cache_entry_get_value(job
->diff_to
, LABEL_FRESH_UNTIL
);
1395 const char *lv_to_valid_until
=
1396 consensus_cache_entry_get_value(job
->diff_to
, LABEL_VALID_UNTIL
);
1397 const char *lv_to_signatories
=
1398 consensus_cache_entry_get_value(job
->diff_to
, LABEL_SIGNATORIES
);
1399 const char *lv_from_valid_after
=
1400 consensus_cache_entry_get_value(job
->diff_from
, LABEL_VALID_AFTER
);
1401 const char *lv_from_digest
=
1402 consensus_cache_entry_get_value(job
->diff_from
,
1403 LABEL_SHA3_DIGEST_AS_SIGNED
);
1404 const char *lv_from_flavor
=
1405 consensus_cache_entry_get_value(job
->diff_from
, LABEL_FLAVOR
);
1406 const char *lv_to_flavor
=
1407 consensus_cache_entry_get_value(job
->diff_to
, LABEL_FLAVOR
);
1408 const char *lv_to_digest
=
1409 consensus_cache_entry_get_value(job
->diff_to
,
1410 LABEL_SHA3_DIGEST_UNCOMPRESSED
);
1412 if (! lv_from_digest
) {
1413 /* This isn't a bug right now, since it can happen if you're migrating
1414 * from an older version of master to a newer one. The older ones didn't
1415 * annotate their stored consensus objects with sha3-digest-as-signed.
1417 return WQ_RPL_REPLY
; // LCOV_EXCL_LINE
1420 /* All these values are mandatory on the input */
1421 if (BUG(!lv_to_valid_after
) ||
1422 BUG(!lv_from_valid_after
) ||
1423 BUG(!lv_from_flavor
) ||
1424 BUG(!lv_to_flavor
)) {
1425 return WQ_RPL_REPLY
; // LCOV_EXCL_LINE
1427 /* The flavors need to match */
1428 if (BUG(strcmp(lv_from_flavor
, lv_to_flavor
))) {
1429 return WQ_RPL_REPLY
; // LCOV_EXCL_LINE
1432 char *consensus_diff
;
1434 char *diff_from_nt
= NULL
, *diff_to_nt
= NULL
;
1435 size_t diff_from_nt_len
, diff_to_nt_len
;
1437 if (uncompress_or_copy(&diff_from_nt
, &diff_from_nt_len
,
1438 job
->diff_from
) < 0) {
1439 return WQ_RPL_REPLY
;
1441 if (uncompress_or_copy(&diff_to_nt
, &diff_to_nt_len
,
1442 job
->diff_to
) < 0) {
1443 tor_free(diff_from_nt
);
1444 return WQ_RPL_REPLY
;
1446 tor_assert(diff_from_nt
);
1447 tor_assert(diff_to_nt
);
1449 // XXXX ugh; this is going to calculate the SHA3 of both its
1450 // XXXX inputs again, even though we already have that. Maybe it's time
1451 // XXXX to change the API here?
1452 consensus_diff
= consensus_diff_generate(diff_from_nt
, diff_to_nt
);
1453 tor_free(diff_from_nt
);
1454 tor_free(diff_to_nt
);
1456 if (!consensus_diff
) {
1457 /* Couldn't generate consensus; we'll leave the reply blank. */
1458 return WQ_RPL_REPLY
;
1461 /* Compress the results and send the reply */
1462 tor_assert(compress_diffs_with
[0] == NO_METHOD
);
1463 size_t difflen
= strlen(consensus_diff
);
1464 job
->out
[0].body
= (uint8_t *) consensus_diff
;
1465 job
->out
[0].bodylen
= difflen
;
1467 config_line_t
*common_labels
= NULL
;
1468 if (lv_to_valid_until
)
1469 config_line_prepend(&common_labels
, LABEL_VALID_UNTIL
, lv_to_valid_until
);
1470 if (lv_to_fresh_until
)
1471 config_line_prepend(&common_labels
, LABEL_FRESH_UNTIL
, lv_to_fresh_until
);
1472 if (lv_to_signatories
)
1473 config_line_prepend(&common_labels
, LABEL_SIGNATORIES
, lv_to_signatories
);
1474 cdm_labels_prepend_sha3(&common_labels
,
1475 LABEL_SHA3_DIGEST_UNCOMPRESSED
,
1477 job
->out
[0].bodylen
);
1478 config_line_prepend(&common_labels
, LABEL_FROM_VALID_AFTER
,
1479 lv_from_valid_after
);
1480 config_line_prepend(&common_labels
, LABEL_VALID_AFTER
,
1482 config_line_prepend(&common_labels
, LABEL_FLAVOR
, lv_from_flavor
);
1483 config_line_prepend(&common_labels
, LABEL_FROM_SHA3_DIGEST
,
1485 config_line_prepend(&common_labels
, LABEL_TARGET_SHA3_DIGEST
,
1487 config_line_prepend(&common_labels
, LABEL_DOCTYPE
,
1488 DOCTYPE_CONSENSUS_DIFF
);
1490 job
->out
[0].labels
= config_lines_dup(common_labels
);
1491 cdm_labels_prepend_sha3(&job
->out
[0].labels
,
1494 job
->out
[0].bodylen
);
1496 compress_multiple(job
->out
+1,
1497 n_diff_compression_methods()-1,
1498 compress_diffs_with
+1,
1499 (const uint8_t*)consensus_diff
, difflen
, common_labels
);
1501 config_free_lines(common_labels
);
1502 return WQ_RPL_REPLY
;
1506 * Helper: release all storage held in <b>job</b>.
1509 consensus_diff_worker_job_free(consensus_diff_worker_job_t
*job
)
1514 for (u
= 0; u
< n_diff_compression_methods(); ++u
) {
1515 config_free_lines(job
->out
[u
].labels
);
1516 tor_free(job
->out
[u
].body
);
1518 consensus_cache_entry_decref(job
->diff_from
);
1519 consensus_cache_entry_decref(job
->diff_to
);
1524 * Worker function: This function runs in the main thread, and receives
1525 * a consensus_diff_worker_job_t that the worker thread has already
1529 consensus_diff_worker_replyfn(void *work_
)
1531 tor_assert(in_main_thread());
1534 consensus_diff_worker_job_t
*job
= work_
;
1536 const char *lv_from_digest
=
1537 consensus_cache_entry_get_value(job
->diff_from
,
1538 LABEL_SHA3_DIGEST_AS_SIGNED
);
1539 const char *lv_to_digest
=
1540 consensus_cache_entry_get_value(job
->diff_to
,
1541 LABEL_SHA3_DIGEST_UNCOMPRESSED
);
1542 const char *lv_flavor
=
1543 consensus_cache_entry_get_value(job
->diff_to
, LABEL_FLAVOR
);
1544 if (BUG(lv_from_digest
== NULL
))
1545 lv_from_digest
= "???"; // LCOV_EXCL_LINE
1546 if (BUG(lv_to_digest
== NULL
))
1547 lv_to_digest
= "???"; // LCOV_EXCL_LINE
1549 uint8_t from_sha3
[DIGEST256_LEN
];
1550 uint8_t to_sha3
[DIGEST256_LEN
];
1553 if (BUG(cdm_entry_get_sha3_value(from_sha3
, job
->diff_from
,
1554 LABEL_SHA3_DIGEST_AS_SIGNED
) < 0))
1556 if (BUG(cdm_entry_get_sha3_value(to_sha3
, job
->diff_to
,
1557 LABEL_SHA3_DIGEST_UNCOMPRESSED
) < 0))
1559 if (BUG(lv_flavor
== NULL
)) {
1561 } else if ((flav
= networkstatus_parse_flavor_name(lv_flavor
)) < 0) {
1565 consensus_cache_entry_handle_t
*handles
[ARRAY_LENGTH(compress_diffs_with
)];
1566 memset(handles
, 0, sizeof(handles
));
1568 char description
[128];
1569 tor_snprintf(description
, sizeof(description
),
1570 "consensus diff from %s to %s",
1571 lv_from_digest
, lv_to_digest
);
1573 int status
= store_multiple(handles
,
1574 n_diff_compression_methods(),
1575 compress_diffs_with
,
1579 if (status
!= CDM_DIFF_PRESENT
) {
1580 /* Failure! Nothing to do but complain */
1581 log_warn(LD_DIRSERV
,
1582 "Worker was unable to compute consensus diff "
1583 "from %s to %s", lv_from_digest
, lv_to_digest
);
1584 /* Cache this error so we don't try to compute this one again. */
1585 status
= CDM_DIFF_ERROR
;
1589 for (u
= 0; u
< ARRAY_LENGTH(handles
); ++u
) {
1590 compress_method_t method
= compress_diffs_with
[u
];
1592 cdm_diff_ht_set_status(flav
, from_sha3
, to_sha3
, method
, status
,
1595 consensus_cache_entry_handle_free(handles
[u
]);
1599 consensus_diff_worker_job_free(job
);
1603 * Queue the job of computing the diff from <b>diff_from</b> to <b>diff_to</b>
1604 * in a worker thread.
1607 consensus_diff_queue_diff_work(consensus_cache_entry_t
*diff_from
,
1608 consensus_cache_entry_t
*diff_to
)
1610 tor_assert(in_main_thread());
1612 consensus_cache_entry_incref(diff_from
);
1613 consensus_cache_entry_incref(diff_to
);
1615 consensus_diff_worker_job_t
*job
= tor_malloc_zero(sizeof(*job
));
1616 job
->diff_from
= diff_from
;
1617 job
->diff_to
= diff_to
;
1619 /* Make sure body is mapped. */
1620 const uint8_t *body
;
1622 int r1
= consensus_cache_entry_get_body(diff_from
, &body
, &bodylen
);
1623 int r2
= consensus_cache_entry_get_body(diff_to
, &body
, &bodylen
);
1624 if (r1
< 0 || r2
< 0)
1627 workqueue_entry_t
*work
;
1628 work
= cpuworker_queue_work(WQ_PRI_LOW
,
1629 consensus_diff_worker_threadfn
,
1630 consensus_diff_worker_replyfn
,
1637 consensus_diff_worker_job_free(job
); // includes decrefs.
1642 * Holds requests and replies for consensus_compress_workers.
1644 typedef struct consensus_compress_worker_job_t
{
1646 size_t consensus_len
;
1647 consensus_flavor_t flavor
;
1648 config_line_t
*labels_in
;
1649 compressed_result_t out
[ARRAY_LENGTH(compress_consensus_with
)];
1650 } consensus_compress_worker_job_t
;
1653 * Free all resources held in <b>job</b>
1656 consensus_compress_worker_job_free(consensus_compress_worker_job_t
*job
)
1660 tor_free(job
->consensus
);
1661 config_free_lines(job
->labels_in
);
1663 for (u
= 0; u
< n_consensus_compression_methods(); ++u
) {
1664 config_free_lines(job
->out
[u
].labels
);
1665 tor_free(job
->out
[u
].body
);
1670 * Worker function. This function runs inside a worker thread and receives
1671 * a consensus_compress_worker_job_t as its input.
1673 static workqueue_reply_t
1674 consensus_compress_worker_threadfn(void *state_
, void *work_
)
1677 consensus_compress_worker_job_t
*job
= work_
;
1678 consensus_flavor_t flavor
= job
->flavor
;
1679 const char *consensus
= job
->consensus
;
1680 size_t bodylen
= job
->consensus_len
;
1682 config_line_t
*labels
= config_lines_dup(job
->labels_in
);
1683 const char *flavname
= networkstatus_get_flavor_name(flavor
);
1685 cdm_labels_prepend_sha3(&labels
, LABEL_SHA3_DIGEST_UNCOMPRESSED
,
1686 (const uint8_t *)consensus
, bodylen
);
1688 const char *start
, *end
;
1689 if (router_get_networkstatus_v3_signed_boundaries(consensus
,
1690 &start
, &end
) < 0) {
1692 end
= consensus
+bodylen
;
1694 cdm_labels_prepend_sha3(&labels
, LABEL_SHA3_DIGEST_AS_SIGNED
,
1695 (const uint8_t *)start
,
1698 config_line_prepend(&labels
, LABEL_FLAVOR
, flavname
);
1699 config_line_prepend(&labels
, LABEL_DOCTYPE
, DOCTYPE_CONSENSUS
);
1701 compress_multiple(job
->out
,
1702 n_consensus_compression_methods(),
1703 compress_consensus_with
,
1704 (const uint8_t*)consensus
, bodylen
, labels
);
1705 config_free_lines(labels
);
1706 return WQ_RPL_REPLY
;
1710 * Worker function: This function runs in the main thread, and receives
1711 * a consensus_diff_compress_job_t that the worker thread has already
1715 consensus_compress_worker_replyfn(void *work_
)
1717 consensus_compress_worker_job_t
*job
= work_
;
1719 consensus_cache_entry_handle_t
*handles
[
1720 ARRAY_LENGTH(compress_consensus_with
)];
1721 memset(handles
, 0, sizeof(handles
));
1723 store_multiple(handles
,
1724 n_consensus_compression_methods(),
1725 compress_consensus_with
,
1728 cdm_cache_dirty
= 1;
1731 consensus_flavor_t f
= job
->flavor
;
1732 tor_assert((int)f
< N_CONSENSUS_FLAVORS
);
1733 for (u
= 0; u
< ARRAY_LENGTH(handles
); ++u
) {
1734 if (handles
[u
] == NULL
)
1736 consensus_cache_entry_handle_free(latest_consensus
[f
][u
]);
1737 latest_consensus
[f
][u
] = handles
[u
];
1740 consensus_compress_worker_job_free(job
);
1744 * If true, we compress in worker threads.
1746 static int background_compression
= 0;
1749 * Queue a job to compress <b>consensus</b> and store its compressed
1750 * text in the cache.
1753 consensus_queue_compression_work(const char *consensus
,
1754 const networkstatus_t
*as_parsed
)
1756 tor_assert(consensus
);
1757 tor_assert(as_parsed
);
1759 consensus_compress_worker_job_t
*job
= tor_malloc_zero(sizeof(*job
));
1760 job
->consensus
= tor_strdup(consensus
);
1761 job
->consensus_len
= strlen(consensus
);
1762 job
->flavor
= as_parsed
->flavor
;
1764 char va_str
[ISO_TIME_LEN
+1];
1765 char vu_str
[ISO_TIME_LEN
+1];
1766 char fu_str
[ISO_TIME_LEN
+1];
1767 format_iso_time_nospace(va_str
, as_parsed
->valid_after
);
1768 format_iso_time_nospace(fu_str
, as_parsed
->fresh_until
);
1769 format_iso_time_nospace(vu_str
, as_parsed
->valid_until
);
1770 config_line_append(&job
->labels_in
, LABEL_VALID_AFTER
, va_str
);
1771 config_line_append(&job
->labels_in
, LABEL_FRESH_UNTIL
, fu_str
);
1772 config_line_append(&job
->labels_in
, LABEL_VALID_UNTIL
, vu_str
);
1773 if (as_parsed
->voters
) {
1774 smartlist_t
*hexvoters
= smartlist_new();
1775 SMARTLIST_FOREACH_BEGIN(as_parsed
->voters
,
1776 networkstatus_voter_info_t
*, vi
) {
1777 if (smartlist_len(vi
->sigs
) == 0)
1778 continue; // didn't sign.
1779 char d
[HEX_DIGEST_LEN
+1];
1780 base16_encode(d
, sizeof(d
), vi
->identity_digest
, DIGEST_LEN
);
1781 smartlist_add_strdup(hexvoters
, d
);
1782 } SMARTLIST_FOREACH_END(vi
);
1783 char *signers
= smartlist_join_strings(hexvoters
, ",", 0, NULL
);
1784 config_line_prepend(&job
->labels_in
, LABEL_SIGNATORIES
, signers
);
1786 SMARTLIST_FOREACH(hexvoters
, char *, cp
, tor_free(cp
));
1787 smartlist_free(hexvoters
);
1790 if (background_compression
) {
1791 workqueue_entry_t
*work
;
1792 work
= cpuworker_queue_work(WQ_PRI_LOW
,
1793 consensus_compress_worker_threadfn
,
1794 consensus_compress_worker_replyfn
,
1797 consensus_compress_worker_job_free(job
);
1803 consensus_compress_worker_threadfn(NULL
, job
);
1804 consensus_compress_worker_replyfn(job
);
1810 * Tell the consdiffmgr backend to compress consensuses in worker threads.
1813 consdiffmgr_enable_background_compression(void)
1815 // This isn't the default behavior because it would break unit tests.
1816 background_compression
= 1;
1819 /** Read the set of voters from the cached object <b>ent</b> into
1820 * <b>out</b>, as a list of hex-encoded digests. Return 0 on success,
1821 * -1 if no signatories were recorded. */
1823 consensus_cache_entry_get_voter_id_digests(const consensus_cache_entry_t
*ent
,
1829 s
= consensus_cache_entry_get_value(ent
, LABEL_SIGNATORIES
);
1832 smartlist_split_string(out
, s
, ",", SPLIT_SKIP_SPACE
|SPLIT_STRIP_SPACE
, 0);
1836 /** Read the fresh-until time of cached object <b>ent</b> into *<b>out</b>
1837 * and return 0, or return -1 if no such time was recorded. */
1839 consensus_cache_entry_get_fresh_until(const consensus_cache_entry_t
*ent
,
1845 s
= consensus_cache_entry_get_value(ent
, LABEL_FRESH_UNTIL
);
1846 if (s
== NULL
|| parse_iso_time_nospace(s
, out
) < 0)
1852 /** Read the valid until timestamp from the cached object <b>ent</b> into
1853 * *<b>out</b> and return 0, or return -1 if no such time was recorded. */
1855 consensus_cache_entry_get_valid_until(const consensus_cache_entry_t
*ent
,
1862 s
= consensus_cache_entry_get_value(ent
, LABEL_VALID_UNTIL
);
1863 if (s
== NULL
|| parse_iso_time_nospace(s
, out
) < 0)
1869 /** Read the valid after timestamp from the cached object <b>ent</b> into
1870 * *<b>out</b> and return 0, or return -1 if no such time was recorded. */
1872 consensus_cache_entry_get_valid_after(const consensus_cache_entry_t
*ent
,
1879 s
= consensus_cache_entry_get_value(ent
, LABEL_VALID_AFTER
);
1881 if (s
== NULL
|| parse_iso_time_nospace(s
, out
) < 0)