1 /* Copyright (c) 2021, The Tor Project, Inc. */
2 /* See LICENSE for licensing information */
6 * \brief Conflux circuit pool management
9 #define TOR_CONFLUX_PRIVATE
10 #define CONFLUX_CELL_PRIVATE
12 #include "core/or/or.h"
14 #include "app/config/config.h"
16 #include "core/or/circuitbuild.h"
17 #include "core/or/circuitlist.h"
18 #include "core/or/circuitstats.h"
19 #include "core/or/circuituse.h"
20 #include "core/or/congestion_control_st.h"
21 #include "core/or/conflux.h"
22 #include "core/or/conflux_cell.h"
23 #include "trunnel/conflux.h"
24 #include "core/or/conflux_params.h"
25 #include "core/or/conflux_pool.h"
26 #include "core/or/conflux_util.h"
27 #include "core/or/relay.h"
28 #include "core/or/connection_edge.h"
29 #include "core/or/edge_connection_st.h"
31 #include "core/or/crypt_path_st.h"
32 #include "core/or/or_circuit_st.h"
33 #include "core/or/origin_circuit_st.h"
34 #include "core/or/extend_info_st.h"
35 #include "core/or/conflux_st.h"
37 #include "feature/nodelist/nodelist.h"
38 #include "feature/client/bridges.h"
39 #include "app/config/config.h"
41 #include "lib/crypt_ops/crypto_rand.h"
42 #include "lib/crypt_ops/crypto_util.h"
44 /* Indicate if we are shutting down. This is used so we avoid recovering a
45 * conflux set on total shutdown. */
46 static bool shutting_down
= false;
48 /** The pool of client-side conflux_t that are built, linked, and ready
49 * to be used. Indexed by nonce. */
50 static digest256map_t
*client_linked_pool
;
52 /** The pool of origin unlinked_circuits_t indexed by nonce. */
53 static digest256map_t
*client_unlinked_pool
;
55 /** The pool of relay conflux_t indexed by nonce. We call these "server"
56 * because they could be onion-service side too (even though we likely will
57 * only implement onion service conflux in Arti). The code is littered with
58 * asserts to ensure there are no origin circuits in here for now, too. */
59 static digest256map_t
*server_linked_pool
;
61 /** The pool of relay unlinked_circuits_t indexed by nonce. */
62 static digest256map_t
*server_unlinked_pool
;
64 /* A leg is essentially a circuit for a conflux set. We use this object for the
66 typedef struct leg_t
{
67 /* The circuit of the leg. */
70 /* The LINK cell content which is used to put the information back in the
71 * conflux_t object once all legs have linked and validate the ack. */
72 conflux_cell_link_t
*link
;
74 /* Indicate if the leg has received the LINKED or the LINKED_ACK cell
75 * depending on its side of the circuit. When all legs are linked, we then
76 * finalize the conflux_t object and move it to the linked pool. */
79 /* What time did we send the LINK/LINKED (depending on which side) so we can
80 * calculate the RTT. */
81 uint64_t link_sent_usec
;
83 /* The RTT value in usec takend from the LINK <--> LINKED round trip. */
87 /* Object used to track unlinked circuits which are kept in the unlinked pool
88 * until they are linked and moved to the linked pool and global circuit set.
90 typedef struct unlinked_circuits_t
{
91 /* If true, indicate that this unlinked set is client side as in the legs are
92 * origin circuits. Else, it is on the exit side and thus or circuits. */
95 /* If true, indicate if the conflux_t is related to a linked set. */
96 bool is_for_linked_set
;
98 /* Conflux object that will be set in each leg once all linked. */
103 } unlinked_circuits_t
;
105 /** Error code used when linking circuits. Based on those, we decide to
106 * relaunch or not. */
107 typedef enum link_circ_err_t
{
108 /* Linking was successful. */
109 ERR_LINK_CIRC_OK
= 0,
110 /* The RTT was not acceptable. */
111 ERR_LINK_CIRC_BAD_RTT
= 1,
112 /* The leg can't be found. */
113 ERR_LINK_CIRC_MISSING_LEG
= 2,
114 /* The set can't be found. */
115 ERR_LINK_CIRC_MISSING_SET
= 3,
116 /* Invalid leg as in not pass validation. */
117 ERR_LINK_CIRC_INVALID_LEG
= 4,
120 #ifdef TOR_UNIT_TESTS
122 get_unlinked_pool(bool is_client
)
124 return is_client
? client_unlinked_pool
: server_unlinked_pool
;
128 get_linked_pool(bool is_client
)
130 return is_client
? client_linked_pool
: server_linked_pool
;
134 /* For unit tests only: please treat these exactly as the defines in the
136 STATIC
uint8_t DEFAULT_CLIENT_UX
= CONFLUX_UX_HIGH_THROUGHPUT
;
137 STATIC
uint8_t DEFAULT_EXIT_UX
= CONFLUX_UX_MIN_LATENCY
;
139 /** Helper: Format at 8 bytes the nonce for logging. */
140 static inline const char *
141 fmt_nonce(const uint8_t *nonce
)
143 return hex_str((char *) nonce
, 8);
147 * Return the conflux algorithm for a desired UX value.
150 conflux_choose_algorithm(uint8_t desired_ux
)
152 switch (desired_ux
) {
153 case CONFLUX_UX_NO_OPINION
:
154 return CONFLUX_ALG_LOWRTT
;
155 case CONFLUX_UX_MIN_LATENCY
:
156 return CONFLUX_ALG_MINRTT
;
157 case CONFLUX_UX_HIGH_THROUGHPUT
:
158 return CONFLUX_ALG_LOWRTT
;
159 /* For now, we have no low mem algs, so use minRTT since it should
160 * switch less and thus use less mem */
161 /* TODO-329-TUNING: Pick better algs here*/
162 case CONFLUX_UX_LOW_MEM_THROUGHPUT
:
163 case CONFLUX_UX_LOW_MEM_LATENCY
:
164 return CONFLUX_ALG_MINRTT
;
166 /* Trunnel should protect us from this */
167 tor_assert_nonfatal_unreached();
168 return CONFLUX_ALG_LOWRTT
;
172 /** Return a newly allocated conflux_t object. */
176 conflux_t
*cfx
= tor_malloc_zero(sizeof(*cfx
));
178 cfx
->ooo_q
= smartlist_new();
179 cfx
->legs
= smartlist_new();
185 conflux_free_(conflux_t
*cfx
)
191 SMARTLIST_FOREACH_BEGIN(cfx
->legs
, conflux_leg_t
*, leg
) {
192 SMARTLIST_DEL_CURRENT(cfx
->legs
, leg
);
194 } SMARTLIST_FOREACH_END(leg
);
195 smartlist_free(cfx
->legs
);
197 SMARTLIST_FOREACH(cfx
->ooo_q
, conflux_cell_t
*, cell
, tor_free(cell
));
198 smartlist_free(cfx
->ooo_q
);
200 memwipe(cfx
->nonce
, 0, sizeof(cfx
->nonce
));
204 /** Wrapper for the free function, set the cfx pointer to NULL after free */
205 #define conflux_free(cfx) \
206 FREE_AND_NULL(conflux_t, conflux_free_, cfx)
208 /** Helper: Free function for the digest256map_free(). */
210 free_conflux_void_(void *ptr
)
212 conflux_t
*cfx
= (conflux_t
*)ptr
;
216 /** Return a newly allocated leg object containing the given circuit and link
217 * pointer (no copy). */
219 leg_new(circuit_t
*circ
, conflux_cell_link_t
*link
)
221 leg_t
*leg
= tor_malloc_zero(sizeof(*leg
));
227 /** Free the given leg object. Passing NULL is safe. */
235 tor_free(leg
->circ
->conflux_pending_nonce
);
236 leg
->circ
->conflux_pending_nonce
= NULL
;
242 /** Return a newly allocated unlinked set object for the given nonce. A new
243 * conflux object is also created. */
244 static unlinked_circuits_t
*
245 unlinked_new(const uint8_t *nonce
, bool is_client
)
247 unlinked_circuits_t
*unlinked
= tor_malloc_zero(sizeof(*unlinked
));
248 unlinked
->cfx
= conflux_new();
249 unlinked
->legs
= smartlist_new();
250 unlinked
->is_client
= is_client
;
251 memcpy(unlinked
->cfx
->nonce
, nonce
, sizeof(unlinked
->cfx
->nonce
));
256 /** Free the given unlinked object. */
258 unlinked_free(unlinked_circuits_t
*unlinked
)
263 /* This cfx is pointing to a linked set. */
264 if (!unlinked
->is_for_linked_set
) {
265 conflux_free(unlinked
->cfx
);
267 SMARTLIST_FOREACH(unlinked
->legs
, leg_t
*, leg
, leg_free(leg
));
268 smartlist_free(unlinked
->legs
);
272 /** Add the given unlinked object to the unlinked pool. */
274 unlinked_pool_add(unlinked_circuits_t
*unlinked
, bool is_client
)
276 tor_assert(unlinked
);
278 digest256map_set(client_unlinked_pool
, unlinked
->cfx
->nonce
, unlinked
);
280 digest256map_set(server_unlinked_pool
, unlinked
->cfx
->nonce
, unlinked
);
284 /** Delete the given unlinked object from the unlinked pool. */
286 unlinked_pool_del(unlinked_circuits_t
*unlinked
, bool is_client
)
288 tor_assert(unlinked
);
291 digest256map_remove(client_unlinked_pool
, unlinked
->cfx
->nonce
);
293 digest256map_remove(server_unlinked_pool
, unlinked
->cfx
->nonce
);
297 /** Return an unlinked object for the given nonce else NULL. */
298 static unlinked_circuits_t
*
299 unlinked_pool_get(const uint8_t *nonce
, bool is_client
)
303 return digest256map_get(client_unlinked_pool
, nonce
);
305 return digest256map_get(server_unlinked_pool
, nonce
);
309 /** Delete from the pool and free the given unlinked object. */
311 unlinked_pool_del_and_free(unlinked_circuits_t
*unlinked
, bool is_client
)
313 tor_assert(unlinked
);
314 unlinked_pool_del(unlinked
, is_client
);
315 unlinked_free(unlinked
);
318 /** Add the given conflux object to the linked conflux set. */
320 linked_pool_add(conflux_t
*cfx
, bool is_client
)
324 digest256map_set(client_linked_pool
, cfx
->nonce
, cfx
);
326 digest256map_set(server_linked_pool
, cfx
->nonce
, cfx
);
330 /** Delete from the linked conflux set the given nonce. */
332 linked_pool_del(const uint8_t *nonce
, bool is_client
)
336 digest256map_remove(client_linked_pool
, nonce
);
338 digest256map_remove(server_linked_pool
, nonce
);
342 /** Return a conflux_t object for the given nonce from the linked set. */
344 linked_pool_get(const uint8_t *nonce
, bool is_client
)
348 return digest256map_get(client_linked_pool
, nonce
);
350 return digest256map_get(server_linked_pool
, nonce
);
354 /** Add the given leg to the given unlinked object. */
356 unlinked_leg_add(unlinked_circuits_t
*unlinked
, leg_t
*leg
)
358 tor_assert(unlinked
);
361 smartlist_add(unlinked
->legs
, leg
);
364 /** Return an unlinked leg for the given unlinked object and for the given
366 static inline leg_t
*
367 leg_find(const unlinked_circuits_t
*unlinked
, const circuit_t
*circ
)
369 SMARTLIST_FOREACH_BEGIN(unlinked
->legs
, leg_t
*, leg
) {
370 if (leg
->circ
== circ
) {
373 } SMARTLIST_FOREACH_END(leg
);
377 /** Return the given circuit leg from its unlinked set (if any). */
379 unlinked_leg_find(const circuit_t
*circ
, bool is_client
)
381 unlinked_circuits_t
*unlinked
=
382 unlinked_pool_get(circ
->conflux_pending_nonce
, is_client
);
386 return leg_find(unlinked
, circ
);
390 unlinked_leg_del_and_free(unlinked_circuits_t
*unlinked
,
391 const circuit_t
*circ
)
394 tor_assert(unlinked
);
396 SMARTLIST_FOREACH_BEGIN(unlinked
->legs
, leg_t
*, leg
) {
397 if (leg
->circ
== circ
) {
398 SMARTLIST_DEL_CURRENT(unlinked
->legs
, leg
);
402 } SMARTLIST_FOREACH_END(leg
);
406 * Ensure that the given circuit has no attached streams.
408 * This validation function is called at various stages for
409 * unlinked circuits, to make sure they have no streams.
412 validate_circ_has_no_streams(circuit_t
*circ
)
414 if (CIRCUIT_IS_ORIGIN(circ
)) {
415 origin_circuit_t
*ocirc
= TO_ORIGIN_CIRCUIT(circ
);
416 if (BUG(ocirc
->p_streams
)) {
418 "Unlinked Conflux circuit %u has attached streams.",
419 ocirc
->global_identifier
);
420 ocirc
->p_streams
= NULL
;
422 if (BUG(ocirc
->half_streams
)) {
424 "Unlinked conflux circ %u has half streams.",
425 ocirc
->global_identifier
);
426 ocirc
->half_streams
= NULL
;
429 or_circuit_t
*orcirc
= TO_OR_CIRCUIT(circ
);
430 if (BUG(orcirc
->n_streams
)) {
432 "Unlinked conflux circuit has attached streams.");
433 orcirc
->n_streams
= NULL
;
435 if (BUG(orcirc
->resolving_streams
)) {
437 "Unlinked conflux circuit has resolving streams.");
438 orcirc
->resolving_streams
= NULL
;
443 /** Return true iff the legs in the given unlinked set are valid and coherent
444 * to be a linked set. */
446 validate_unlinked_legs(unlinked_circuits_t
*unlinked
)
450 uint8_t *nonce
= NULL
;
452 tor_assert(unlinked
);
454 SMARTLIST_FOREACH_BEGIN(unlinked
->legs
, const leg_t
*, leg
) {
456 nonce
= leg
->link
->nonce
;
457 version
= leg
->link
->version
;
459 /* Version and nonce must match in all legs. */
460 valid
&= (leg
->link
->version
== version
&&
461 tor_memeq(leg
->link
->nonce
, nonce
, sizeof(leg
->link
->nonce
)));
464 // If the other ends last sent sequence number is higher than the
465 // last sequence number we delivered, we have data loss, and cannot link.
466 if (leg
->link
->last_seqno_sent
> unlinked
->cfx
->last_seq_delivered
) {
467 log_fn(unlinked
->is_client
? LOG_NOTICE
: LOG_PROTOCOL_WARN
, LD_CIRC
,
468 "Data loss detected while trying to add a conflux leg.");
471 // TODO-329-ARTI: Instead of closing the set here, we could
472 // immediately send a SWITCH cell and re-send the missing data.
473 // To do this, though, we would need to constantly buffer at least
474 // a cwnd worth of sent data to retransmit. We're not going to try
475 // this in C-Tor, but arti could consider it.
477 validate_circ_has_no_streams(leg
->circ
);
478 } SMARTLIST_FOREACH_END(leg
);
480 /* Note that if no legs, it validates. */
485 /** Add up a new leg to the given conflux object. */
487 cfx_add_leg(conflux_t
*cfx
, leg_t
*leg
)
491 tor_assert(leg
->link
);
493 /* Big trouble if we add a leg to the wrong set. */
494 tor_assert(tor_memeq(cfx
->nonce
, leg
->link
->nonce
, sizeof(cfx
->nonce
)));
496 if (BUG(CONFLUX_NUM_LEGS(cfx
) > CONFLUX_MAX_CIRCS
)) {
500 conflux_leg_t
*cleg
= tor_malloc_zero(sizeof(*cleg
));
501 cleg
->circ
= leg
->circ
;
502 // TODO-329-ARTI: Blindly copying the values from the cell. Is this correct?
503 // I think no... When adding new legs, switching to this leg is
504 // likely to break, unless the sender tracks what link cell it sent..
505 // Is that the best option? Or should we use the max of our legs, here?
506 // (It seems the other side will have no idea what our current maxes
507 /// are, so this option seems better right now)
508 cleg
->last_seq_recv
= leg
->link
->last_seqno_sent
;
509 cleg
->last_seq_sent
= leg
->link
->last_seqno_recv
;
510 cleg
->circ_rtts_usec
= leg
->rtt_usec
;
511 cleg
->linked_sent_usec
= leg
->link_sent_usec
;
513 cfx
->params
.alg
= conflux_choose_algorithm(leg
->link
->desired_ux
);
515 /* Add leg to given conflux. */
516 smartlist_add(cfx
->legs
, cleg
);
518 /* Ensure the new circuit has no streams. */
519 validate_circ_has_no_streams(leg
->circ
);
521 /* If this is not the first leg, get the first leg, and get
522 * the reference streams from it. */
523 if (CONFLUX_NUM_LEGS(cfx
) > 0) {
524 conflux_leg_t
*first_leg
= smartlist_get(cfx
->legs
, 0);
525 if (CIRCUIT_IS_ORIGIN(first_leg
->circ
)) {
526 origin_circuit_t
*old_circ
= TO_ORIGIN_CIRCUIT(first_leg
->circ
);
527 origin_circuit_t
*new_circ
= TO_ORIGIN_CIRCUIT(leg
->circ
);
529 new_circ
->p_streams
= old_circ
->p_streams
;
530 new_circ
->half_streams
= old_circ
->half_streams
;
531 /* Sync all legs with the new stream(s). */
532 conflux_sync_circ_fields(cfx
, old_circ
);
534 or_circuit_t
*old_circ
= TO_OR_CIRCUIT(first_leg
->circ
);
535 or_circuit_t
*new_circ
= TO_OR_CIRCUIT(leg
->circ
);
536 new_circ
->n_streams
= old_circ
->n_streams
;
537 new_circ
->resolving_streams
= old_circ
->resolving_streams
;
541 if (CIRCUIT_IS_ORIGIN(cleg
->circ
)) {
542 tor_assert_nonfatal(cleg
->circ
->purpose
==
543 CIRCUIT_PURPOSE_CONFLUX_UNLINKED
);
544 circuit_change_purpose(cleg
->circ
, CIRCUIT_PURPOSE_CONFLUX_LINKED
);
546 conflux_validate_stream_lists(cfx
);
550 * Clean up a circuit from its conflux_t object.
552 * Return true if closing this circuit should tear down the entire set,
556 cfx_del_leg(conflux_t
*cfx
, const circuit_t
*circ
)
559 bool full_teardown
= false;
564 leg
= conflux_get_leg(cfx
, circ
);
569 // If the circuit still has inflight data, teardown
570 const struct congestion_control_t
*cc
= circuit_ccontrol(circ
);
572 tor_assert(cc
->sendme_inc
);
573 if (cc
->inflight
>= cc
->sendme_inc
) {
574 full_teardown
= true;
575 log_info(LD_CIRC
, "Conflux current circuit has closed with "
576 "data in flight, tearing down entire set.");
579 /* Remove it from the cfx. */
580 smartlist_remove(cfx
->legs
, leg
);
582 /* After removal, if this leg had the highest sent (or recv)
583 * sequence number, it was in active use by us (or the other side).
584 * We need to tear down the entire set. */
585 // TODO-329-ARTI: If we support resumption, we don't need this.
586 if (CONFLUX_NUM_LEGS(cfx
) > 0) {
587 if (conflux_get_max_seq_sent(cfx
) < leg
->last_seq_sent
||
588 conflux_get_max_seq_recv(cfx
) < leg
->last_seq_recv
) {
589 full_teardown
= true;
590 log_info(LD_CIRC
, "Conflux sequence number check failed, "
591 "tearing down entire set.");
595 /* Cleanup any reference to leg. */
596 if (cfx
->curr_leg
== leg
) {
597 cfx
->curr_leg
= NULL
;
598 full_teardown
= true;
599 log_info(LD_CIRC
, "Conflux current circuit has closed, "
600 "tearing down entire set.");
602 if (cfx
->prev_leg
== leg
) {
603 cfx
->prev_leg
= NULL
;
609 return full_teardown
;
612 /** Close the circuit of each legs of the given unlinked object. */
614 unlinked_close_all_legs(unlinked_circuits_t
*unlinked
)
616 smartlist_t
*circ_to_close
= NULL
;
618 tor_assert(unlinked
);
620 /* Small optimization here, avoid this work if no legs. */
621 if (smartlist_len(unlinked
->legs
) == 0) {
625 /* We will iterate over all legs and put the circuit in its own list and then
626 * mark them for close. The unlinked object gets freed opportunistically once
627 * there is no more legs attached to it and so we can't hold a reference
628 * while closing circuits. */
629 circ_to_close
= smartlist_new();
631 SMARTLIST_FOREACH(unlinked
->legs
, leg_t
*, leg
,
632 smartlist_add(circ_to_close
, leg
->circ
));
635 /* The leg gets cleaned up in the circuit close. */
636 SMARTLIST_FOREACH_BEGIN(circ_to_close
, circuit_t
*, circ
) {
637 if (CIRCUIT_IS_ORIGIN(circ
)) {
638 tor_assert_nonfatal(circ
->purpose
== CIRCUIT_PURPOSE_CONFLUX_UNLINKED
);
640 if (!circ
->marked_for_close
) {
641 circuit_mark_for_close(circ
, END_CIRC_REASON_INTERNAL
);
643 } SMARTLIST_FOREACH_END(circ
);
645 /* Drop the list and ignore its content, we don't have ownership. */
646 smartlist_free(circ_to_close
);
649 /** Either closee all legs of the given unlinked set or delete it from the pool
650 * and free its memory.
652 * Important: The unlinked object is freed opportunistically when legs are
653 * removed until the point none remains. And so, it is only safe to free the
654 * object if no more legs exist.
657 unlinked_close_or_free(unlinked_circuits_t
*unlinked
)
663 /* If we have legs, the circuit close will trigger the unlinked object to be
664 * opportunistically freed. Else, we do it explicitly. */
665 if (smartlist_len(unlinked
->legs
) > 0) {
666 unlinked_close_all_legs(unlinked
);
668 unlinked_pool_del_and_free(unlinked
, unlinked
->is_client
);
670 /* Either the unlinked object has been freed or the last leg close will free
671 * it so from this point on, nullify for safety reasons. */
675 /** Upon an error condition or a close of an in-use circuit, we must close all
676 * linked and unlinked circuits associated with a set. When the last leg of
677 * each set is closed, the set is removed from the pool. */
679 conflux_mark_all_for_close(const uint8_t *nonce
, bool is_client
, int reason
)
681 /* It is possible that for a nonce we have both an unlinked set and a linked
682 * set. This happens if there is a recovery leg launched for an existing
685 /* Close the unlinked set. */
686 unlinked_circuits_t
*unlinked
= unlinked_pool_get(nonce
, is_client
);
688 unlinked_close_or_free(unlinked
);
690 /* In case it gets freed, be safe here. */
693 /* Close the linked set. It will free itself upon the close of
695 conflux_t
*linked
= linked_pool_get(nonce
, is_client
);
697 if (linked
->in_full_teardown
) {
700 linked
->in_full_teardown
= true;
702 smartlist_t
*circ_to_close
= smartlist_new();
704 SMARTLIST_FOREACH(linked
->legs
, conflux_leg_t
*, leg
,
705 smartlist_add(circ_to_close
, leg
->circ
));
707 SMARTLIST_FOREACH(circ_to_close
, circuit_t
*, circ
,
708 circuit_mark_for_close(circ
, reason
));
710 /* Drop the list and ignore its content, we don't have ownership. */
711 smartlist_free(circ_to_close
);
715 /** Helper: Free function taking a void pointer for the digest256map_free. */
717 free_unlinked_void_(void *ptr
)
719 unlinked_circuits_t
*unlinked
= ptr
;
720 unlinked_pool_del_and_free(unlinked
, unlinked
->is_client
);
723 /** Attempt to finalize the unlinked set to become a linked set and be put in
726 * If this finalized successfully, the given unlinked object is freed. */
727 static link_circ_err_t
728 try_finalize_set(unlinked_circuits_t
*unlinked
)
730 link_circ_err_t err
= ERR_LINK_CIRC_OK
;
733 tor_assert(unlinked
);
735 /* Without legs, this is not ready to become a linked set. */
736 if (BUG(smartlist_len(unlinked
->legs
) == 0)) {
737 err
= ERR_LINK_CIRC_MISSING_LEG
;
741 /* Validate that all legs are coherent and parameters match. On failure, we
742 * teardown the whole unlinked set because this means we either have a code
743 * flow problem or the Exit is trying to trick us. */
744 if (!validate_unlinked_legs(unlinked
)) {
745 log_fn(LOG_PROTOCOL_WARN
, LD_CIRC
,
746 "Conflux unlinked set legs are not validating. Tearing it down.");
747 conflux_mark_all_for_close(unlinked
->cfx
->nonce
, unlinked
->is_client
,
748 END_CIRC_REASON_TORPROTOCOL
);
749 err
= ERR_LINK_CIRC_INVALID_LEG
;
753 /* Check all linked status. All need to be true in order to finalize the set
754 * and move it to the linked pool. */
755 SMARTLIST_FOREACH_BEGIN(unlinked
->legs
, const leg_t
*, leg
) {
756 /* We are still waiting on a leg. */
758 log_info(LD_CIRC
, "Can't finalize conflux set, still waiting on at "
759 "least one leg to link up.");
763 } SMARTLIST_FOREACH_END(leg
);
765 /* Finalize the cfx object by adding all legs into it. */
766 SMARTLIST_FOREACH_BEGIN(unlinked
->legs
, leg_t
*, leg
) {
767 /* Removing the leg from the list is important so we avoid ending up with a
768 * leg in the unlinked list that is set with LINKED purpose. */
769 SMARTLIST_DEL_CURRENT(unlinked
->legs
, leg
);
771 /* We are ready to attach the leg to the cfx object now. */
772 cfx_add_leg(unlinked
->cfx
, leg
);
774 /* Clean the pending nonce and set the conflux object in the circuit. */
775 leg
->circ
->conflux
= unlinked
->cfx
;
777 /* We are done with this leg object. */
779 } SMARTLIST_FOREACH_END(leg
);
781 is_client
= unlinked
->is_client
;
783 /* Add the conflux object to the linked pool. For an existing linked cfx
784 * object, we'll simply replace it with itself. */
785 linked_pool_add(unlinked
->cfx
, is_client
);
787 /* Remove it from the unlinked pool. */
788 unlinked_pool_del(unlinked
, is_client
);
790 /* We don't recover a leg when it is linked but if we would like to support
791 * session ressumption, this would be very important in order to allow new
792 * legs to be created/recovered. */
793 unlinked
->cfx
->num_leg_launch
= 0;
795 /* Nullify because we are about to free the unlinked object and the cfx has
796 * moved to all circuits. */
797 unlinked
->cfx
= NULL
;
798 unlinked_free(unlinked
);
801 "Successfully linked a conflux %s set which is now usable.",
802 is_client
? "client" : "relay");
808 /** Record the RTT for this client circuit.
810 * Return the RTT value. UINT64_MAX is returned if we couldn't find the initial
811 * measurement of when the cell was sent or if the leg is missing. */
813 record_rtt_client(const circuit_t
*circ
)
816 tor_assert(circ
->conflux_pending_nonce
);
817 tor_assert(CIRCUIT_IS_ORIGIN(circ
));
819 leg_t
*leg
= unlinked_leg_find(circ
, true);
821 if (BUG(!leg
|| leg
->link_sent_usec
== 0)) {
823 "Conflux: Trying to record client RTT without a timestamp");
827 uint64_t now
= monotime_absolute_usec();
828 tor_assert_nonfatal(now
>= leg
->link_sent_usec
);
829 leg
->rtt_usec
= now
- leg
->link_sent_usec
;
830 if (leg
->rtt_usec
== 0) {
831 log_warn(LD_CIRC
, "Clock appears stalled for conflux.");
832 // TODO-329-TUNING: For now, let's accept this case. We need to do
833 // tuning and clean up the tests such that they use RTT in order to
837 return leg
->rtt_usec
;
840 // Avoid using this leg until a timestamp comes in
842 leg
->rtt_usec
= UINT64_MAX
;
846 /** Record the RTT for this Exit circuit.
848 * Return the RTT value. UINT64_MAX is returned if we couldn't find the initial
849 * measurement of when the cell was sent or if the leg is missing. */
852 record_rtt_exit(const circuit_t
*circ
)
855 tor_assert(circ
->conflux
);
856 tor_assert(CIRCUIT_IS_ORCIRC(circ
));
858 conflux_leg_t
*leg
= conflux_get_leg(circ
->conflux
, circ
);
860 if (BUG(!leg
|| leg
->linked_sent_usec
== 0)) {
862 "Conflux: Trying to record exit RTT without a timestamp");
866 uint64_t now
= monotime_absolute_usec();
867 tor_assert_nonfatal(now
>= leg
->linked_sent_usec
);
868 leg
->circ_rtts_usec
= now
- leg
->linked_sent_usec
;
870 if (leg
->circ_rtts_usec
== 0) {
871 log_warn(LD_CIRC
, "Clock appears stalled for conflux.");
874 return leg
->circ_rtts_usec
;
878 leg
->circ_rtts_usec
= UINT64_MAX
;
882 /** For the given circuit, record the RTT from when the LINK or LINKED cell was
883 * sent that is this function works for either client or Exit.
885 * Return false if the RTT is too high for our standard else true. */
887 record_rtt(const circuit_t
*circ
, bool is_client
)
894 rtt_usec
= record_rtt_client(circ
);
896 if (rtt_usec
== UINT64_MAX
)
899 if (rtt_usec
>= get_circuit_build_timeout_ms()*1000) {
900 log_info(LD_CIRC
, "Conflux leg RTT is above circuit build time out "
901 "currently at %f msec. Relaunching.",
902 get_circuit_build_timeout_ms());
906 rtt_usec
= record_rtt_exit(circ
);
912 /** Link the given circuit within its unlinked set. This is called when either
913 * the LINKED or LINKED_ACK is received depending on which side of the circuit
916 * It attempts to finalize the unlinked set as well which, if successful, puts
917 * it in the linked pool. */
918 static link_circ_err_t
919 link_circuit(circuit_t
*circ
)
921 link_circ_err_t err
= ERR_LINK_CIRC_OK
;
922 unlinked_circuits_t
*unlinked
= NULL
;
923 bool is_client
= false;
926 if (CIRCUIT_IS_ORIGIN(circ
)) {
927 tor_assert_nonfatal(circ
->purpose
== CIRCUIT_PURPOSE_CONFLUX_UNLINKED
);
931 unlinked
= unlinked_pool_get(circ
->conflux_pending_nonce
, is_client
);
932 if (BUG(!unlinked
)) {
933 log_warn(LD_BUG
, "Failed to find the unlinked set %s when linking. "
935 fmt_nonce(circ
->conflux_pending_nonce
));
936 err
= ERR_LINK_CIRC_MISSING_SET
;
940 leg_t
*leg
= leg_find(unlinked
, circ
);
942 /* Failure to find the leg when linking a circuit is an important problem
943 * so log loudly and error. */
944 log_warn(LD_BUG
, "Failed to find leg for the unlinked set %s when "
945 "linking. Closing circuit.",
946 fmt_nonce(unlinked
->cfx
->nonce
));
947 err
= ERR_LINK_CIRC_MISSING_LEG
;
951 /* Successful link. Attempt to finalize the set in case this was the last
952 * LINKED or LINKED_ACK cell to receive. */
954 err
= try_finalize_set(unlinked
);
960 /** Launch a brand new set.
962 * Return true if all legs successfully launched or false if one failed. */
964 launch_new_set(int num_legs
)
966 uint8_t nonce
[DIGEST256_LEN
];
968 /* Brand new nonce for this set. */
969 crypto_rand((char *) nonce
, sizeof(nonce
));
971 /* Launch all legs. */
972 for (int i
= 0; i
< num_legs
; i
++) {
973 if (!conflux_launch_leg(nonce
)) {
974 /* This function cleans up entirely the unlinked set if a leg is unable
975 * to be launched. The recovery would be complex here. */
986 static unlinked_circuits_t
*
987 unlinked_get_or_create(const uint8_t *nonce
, bool is_client
)
989 unlinked_circuits_t
*unlinked
;
993 unlinked
= unlinked_pool_get(nonce
, is_client
);
995 unlinked
= unlinked_new(nonce
, is_client
);
997 /* If this is a leg of an existing linked set, use that conflux object
998 * instead so all legs point to the same. It is put in the leg's circuit
999 * once the link is confirmed. */
1000 conflux_t
*cfx
= linked_pool_get(nonce
, is_client
);
1002 conflux_free(unlinked
->cfx
);
1003 unlinked
->cfx
= cfx
;
1004 unlinked
->is_for_linked_set
= true;
1006 /* Add this set to the unlinked pool. */
1007 unlinked_pool_add(unlinked
, is_client
);
1014 * On the client side, we need to determine if there is already
1015 * an exit in use for this set, and if so, use that.
1017 * Otherwise, we return NULL and the exit is decided by the
1018 * circuitbuild.c code.
1020 static extend_info_t
*
1021 get_exit_for_nonce(const uint8_t *nonce
)
1023 extend_info_t
*exit
= NULL
;
1027 // First, check the linked pool for the nonce
1028 const conflux_t
*cfx
= linked_pool_get(nonce
, true);
1030 tor_assert(cfx
->legs
);
1031 /* Get the exit from the first leg */
1032 conflux_leg_t
*leg
= smartlist_get(cfx
->legs
, 0);
1034 tor_assert(leg
->circ
);
1035 tor_assert(TO_ORIGIN_CIRCUIT(leg
->circ
)->cpath
);
1036 exit
= TO_ORIGIN_CIRCUIT(leg
->circ
)->cpath
->prev
->extend_info
;
1039 unlinked_circuits_t
*unlinked
= NULL
;
1040 unlinked
= unlinked_pool_get(nonce
, true);
1043 tor_assert(unlinked
->legs
);
1044 if (smartlist_len(unlinked
->legs
) > 0) {
1045 /* Get the exit from the first leg */
1046 leg_t
*leg
= smartlist_get(unlinked
->legs
, 0);
1048 tor_assert(leg
->circ
);
1049 tor_assert(TO_ORIGIN_CIRCUIT(leg
->circ
)->cpath
);
1050 exit
= TO_ORIGIN_CIRCUIT(leg
->circ
)->cpath
->prev
->extend_info
;
1060 * Return the currently configured client UX.
1065 #ifdef TOR_UNIT_TESTS
1066 return DEFAULT_CLIENT_UX
;
1068 const or_options_t
*opt
= get_options();
1070 (void)DEFAULT_CLIENT_UX
;
1073 return opt
->ConfluxClientUX
;
1077 /** Return true iff the given conflux object is allowed to launch a new leg. If
1078 * the cfx object is NULL, then it is always allowed to launch a new leg. */
1080 launch_leg_is_allowed(const conflux_t
*cfx
)
1086 /* The maximum number of retry is the minimum number of legs we are allowed
1087 * per set plus the maximum amount of retries we are allowed to do. */
1088 unsigned int max_num_launch
=
1089 conflux_params_get_num_legs_set() +
1090 conflux_params_get_max_unlinked_leg_retry();
1092 /* Only log once per nonce if we've reached the maximum. */
1093 if (cfx
->num_leg_launch
== max_num_launch
) {
1094 log_info(LD_CIRC
, "Maximum number of leg launch reached for nonce %s",
1095 fmt_nonce(cfx
->nonce
));
1098 if (cfx
->num_leg_launch
>= max_num_launch
) {
1110 /** Launch a new conflux leg for the given nonce.
1112 * Return true on success else false which teardowns the entire unlinked set if
1115 conflux_launch_leg(const uint8_t *nonce
)
1117 int flags
= CIRCLAUNCH_NEED_UPTIME
| CIRCLAUNCH_NEED_CAPACITY
|
1118 CIRCLAUNCH_NEED_CONFLUX
;
1119 unlinked_circuits_t
*unlinked
= NULL
;
1120 extend_info_t
*exit
= NULL
;
1124 /* Get or create a new unlinked object for this leg. */
1125 unlinked
= unlinked_get_or_create(nonce
, true);
1126 tor_assert(unlinked
);
1128 /* If we have an existing linked set, validate the number of leg retries
1129 * before attempting the launch. */
1130 if (!launch_leg_is_allowed(unlinked
->cfx
)) {
1134 exit
= get_exit_for_nonce(nonce
);
1137 log_info(LD_CIRC
, "Launching conflux leg for nonce %s.", fmt_nonce(nonce
));
1139 log_info(LD_CIRC
, "Launching new conflux set for nonce %s.",
1143 /* Increase the retry count for this conflux object as in this nonce.
1144 * We must do this now, because some of the maze's early failure paths
1145 * call right back into this function for relaunch. */
1146 unlinked
->cfx
->num_leg_launch
++;
1148 origin_circuit_t
*circ
=
1149 circuit_establish_circuit_conflux(nonce
, CIRCUIT_PURPOSE_CONFLUX_UNLINKED
,
1154 tor_assert(TO_CIRCUIT(circ
)->conflux_pending_nonce
);
1156 /* At this point, the unlinked object has either a new conflux_t or the one
1157 * used by a linked set so it is fine to use the cfx from the unlinked object
1160 /* Get the max_seq_sent and recv from the linked pool, if it exists, and pass
1161 * to new link cell. */
1162 uint64_t last_seq_sent
= conflux_get_max_seq_sent(unlinked
->cfx
);
1163 uint64_t last_seq_recv
= unlinked
->cfx
->last_seq_delivered
;
1165 // TODO-329-ARTI: To support resumption/retransmit, the client should store
1166 // the last_seq_sent now, so that it can know how much data to retransmit to
1167 // the server after link. C-Tor will not be implementing this, but arti and
1168 // arti-relay could (if resumption seems worthwhile; it may not be worth the
1169 // memory storage there, either).
1171 /* We have a circuit, create the new leg and attach it to the set. */
1172 leg_t
*leg
= leg_new(TO_CIRCUIT(circ
),
1173 conflux_cell_new_link(nonce
,
1174 last_seq_sent
, last_seq_recv
,
1177 unlinked_leg_add(unlinked
, leg
);
1185 * Add the identity digest of the guard nodes of all legs of the conflux
1188 * This function checks both pending and linked conflux circuits.
1191 conflux_add_guards_to_exclude_list(const origin_circuit_t
*orig_circ
,
1192 smartlist_t
*excluded
)
1194 tor_assert(orig_circ
);
1195 tor_assert(excluded
);
1197 /* Ease our lives. */
1198 const circuit_t
*circ
= TO_CIRCUIT(orig_circ
);
1200 /* Ignore if this is not conflux related. */
1201 if (!CIRCUIT_IS_CONFLUX(circ
)) {
1205 /* When building a circuit, we should not have a conflux object
1206 * ourselves (though one may exist elsewhere). */
1207 tor_assert(!circ
->conflux
);
1209 /* Getting here without a nonce is a code flow issue. */
1210 if (BUG(!circ
->conflux_pending_nonce
)) {
1214 /* If there is only one bridge, then only issue a warn once that
1215 * at least two bridges are best for conflux. Exempt Snowflake
1217 if (get_options()->UseBridges
&& !conflux_can_exclude_used_bridges()) {
1218 /* Do not build any exclude lists; not enough bridges */
1222 /* A linked set exists, use it. */
1223 const conflux_t
*cfx
= linked_pool_get(circ
->conflux_pending_nonce
, true);
1225 CONFLUX_FOR_EACH_LEG_BEGIN(cfx
, leg
) {
1226 const origin_circuit_t
*ocirc
= CONST_TO_ORIGIN_CIRCUIT(leg
->circ
);
1227 smartlist_add(excluded
,
1228 tor_memdup(ocirc
->cpath
->extend_info
->identity_digest
,
1230 } CONFLUX_FOR_EACH_LEG_END(leg
);
1233 /* An unlinked set might exist for this nonce, if so, add the second hop of
1234 * the existing legs to the exclusion list. */
1235 unlinked_circuits_t
*unlinked
=
1236 unlinked_pool_get(circ
->conflux_pending_nonce
, true);
1238 tor_assert(unlinked
->is_client
);
1239 SMARTLIST_FOREACH_BEGIN(unlinked
->legs
, leg_t
*, leg
) {
1240 /* Convert to origin circ and get cpath */
1241 const origin_circuit_t
*ocirc
= CONST_TO_ORIGIN_CIRCUIT(leg
->circ
);
1242 smartlist_add(excluded
,
1243 tor_memdup(ocirc
->cpath
->extend_info
->identity_digest
,
1245 } SMARTLIST_FOREACH_END(leg
);
1250 * Add the identity digest of the middle nodes of all legs of the conflux
1253 * This function checks both pending and linked conflux circuits.
1255 * XXX: The add guard and middle could be merged since it is the exact same
1256 * code except for the cpath position and the identity digest vs node_t in
1257 * the list. We could use an extra param indicating guard or middle. */
1259 conflux_add_middles_to_exclude_list(const origin_circuit_t
*orig_circ
,
1260 smartlist_t
*excluded
)
1262 tor_assert(orig_circ
);
1263 tor_assert(excluded
);
1265 /* Ease our lives. */
1266 const circuit_t
*circ
= TO_CIRCUIT(orig_circ
);
1268 /* Ignore if this is not conflux related. */
1269 if (!CIRCUIT_IS_CONFLUX(circ
)) {
1273 /* When building a circuit, we should not have a conflux object
1274 * ourselves (though one may exist elsewhere). */
1275 tor_assert(!circ
->conflux
);
1277 /* Getting here without a nonce is a code flow issue. */
1278 if (BUG(!circ
->conflux_pending_nonce
)) {
1282 /* A linked set exists, use it. */
1283 const conflux_t
*cfx
= linked_pool_get(circ
->conflux_pending_nonce
, true);
1285 CONFLUX_FOR_EACH_LEG_BEGIN(cfx
, leg
) {
1286 const origin_circuit_t
*ocirc
= CONST_TO_ORIGIN_CIRCUIT(leg
->circ
);
1287 node_t
*node
= node_get_mutable_by_id(
1288 ocirc
->cpath
->next
->extend_info
->identity_digest
);
1290 smartlist_add(excluded
, node
);
1292 } CONFLUX_FOR_EACH_LEG_END(leg
);
1295 /* An unlinked set might exist for this nonce, if so, add the second hop of
1296 * the existing legs to the exclusion list. */
1297 unlinked_circuits_t
*unlinked
=
1298 unlinked_pool_get(circ
->conflux_pending_nonce
, true);
1300 tor_assert(unlinked
->is_client
);
1301 SMARTLIST_FOREACH_BEGIN(unlinked
->legs
, leg_t
*, leg
) {
1302 /* Convert to origin circ and get cpath */
1303 const origin_circuit_t
*ocirc
= CONST_TO_ORIGIN_CIRCUIT(leg
->circ
);
1304 node_t
*node
= node_get_mutable_by_id(
1305 ocirc
->cpath
->next
->extend_info
->identity_digest
);
1307 smartlist_add(excluded
, node
);
1309 } SMARTLIST_FOREACH_END(leg
);
1313 /** Return the number of unused client linked set. */
1315 count_client_usable_sets(void)
1319 DIGEST256MAP_FOREACH(client_linked_pool
, key
, conflux_t
*, cfx
) {
1320 conflux_leg_t
*leg
= smartlist_get(cfx
->legs
, 0);
1321 if (BUG(!leg
->circ
)) {
1322 log_warn(LD_BUG
, "Client conflux linked set leg without a circuit");
1325 if (!CONST_TO_ORIGIN_CIRCUIT(leg
->circ
)->unusable_for_new_conns
) {
1328 } DIGEST256MAP_FOREACH_END
;
1333 /** Determine if we need to launch new conflux circuits for our preemptive
1336 * This is called once a second from the mainloop from
1337 * circuit_predict_and_launch_new(). */
1339 conflux_predict_new(time_t now
)
1343 /* If conflux is disabled, or we have insufficient consensus exits,
1344 * don't prebuild. */
1345 if (!conflux_is_enabled(NULL
) ||
1346 router_have_consensus_path() != CONSENSUS_PATH_EXIT
) {
1350 /* Don't attempt to build a new set if we are above our allowed maximum of
1352 if (digest256map_size(client_linked_pool
) >=
1353 conflux_params_get_max_linked_set()) {
1357 /* Count the linked and unlinked to get the total number of sets we have
1359 int num_linked
= count_client_usable_sets();
1360 int num_unlinked
= digest256map_size(client_unlinked_pool
);
1361 int num_set
= num_unlinked
+ num_linked
;
1362 int max_prebuilt
= conflux_params_get_max_prebuilt();
1364 if (num_set
>= max_prebuilt
) {
1368 log_info(LD_CIRC
, "Preemptively launching new conflux circuit set(s). "
1369 "We have %d linked and %d unlinked.",
1370 num_linked
, num_unlinked
);
1372 for (int i
= 0; i
< (max_prebuilt
- num_set
); i
++) {
1373 if (!launch_new_set(conflux_params_get_num_legs_set())) {
1374 /* Failing once likely means we'll fail next attempt so stop for now and
1375 * we'll try later. */
1381 /** Return the first circuit from the linked pool that will work with the conn.
1382 * If no such circuit exists, return NULL. */
1384 conflux_get_circ_for_conn(const entry_connection_t
*conn
, time_t now
)
1386 /* Use conn to check the exit policy of the first circuit
1387 * of each set in the linked pool. */
1390 DIGEST256MAP_FOREACH(client_linked_pool
, key
, conflux_t
*, cfx
) {
1391 /* Get the first circuit of the set. */
1392 conflux_leg_t
*leg
= smartlist_get(cfx
->legs
, 0);
1394 tor_assert(leg
->circ
);
1396 /* Bug on these but we can recover. */
1397 if (BUG(leg
->circ
->purpose
!= CIRCUIT_PURPOSE_CONFLUX_LINKED
)) {
1400 if (BUG(!CIRCUIT_IS_ORIGIN(leg
->circ
))) {
1403 origin_circuit_t
*ocirc
= TO_ORIGIN_CIRCUIT(leg
->circ
);
1405 /* Make sure the connection conforms with the exit policy and the isolation
1406 * flags also allows it. */
1407 if (!circuit_is_acceptable(ocirc
, conn
, 1 /* Must be open */,
1408 CIRCUIT_PURPOSE_CONFLUX_LINKED
,
1409 1 /* Need uptime */,
1410 0 /* No need for internal */, now
)) {
1414 /* Found a circuit that works. */
1416 } DIGEST256MAP_FOREACH_END
;
1421 /** The given circuit is conflux pending and has closed. This deletes the leg
1422 * from the set, attempt to finalize it and relaunch a new leg. If the set is
1423 * empty after removing this leg, it is deleted. */
1425 unlinked_circuit_closed(circuit_t
*circ
)
1427 uint8_t nonce
[DIGEST256_LEN
];
1428 unlinked_circuits_t
*unlinked
= NULL
;
1429 bool is_client
= false;
1432 tor_assert(circ
->conflux_pending_nonce
);
1434 if (CIRCUIT_IS_ORIGIN(circ
)) {
1435 tor_assert_nonfatal(circ
->purpose
== CIRCUIT_PURPOSE_CONFLUX_UNLINKED
);
1439 unlinked
= unlinked_pool_get(circ
->conflux_pending_nonce
, is_client
);
1441 /* This circuit is part of set that has already been removed previously freed
1442 * by another leg closing. */
1447 /* We keep the nonce here because we will try to recover if we can and the
1448 * pending nonce will get nullified early. */
1449 memcpy(nonce
, circ
->conflux_pending_nonce
, sizeof(nonce
));
1451 log_info(LD_CIRC
, "Conflux unlinked circuit with nonce %s has closed",
1454 /* Remove leg from set. */
1455 unlinked_leg_del_and_free(unlinked
, circ
);
1456 /* The circuit pending nonce has been nullified at this point. */
1458 /* If no more legs, opportunistically free the unlinked set. */
1459 if (smartlist_len(unlinked
->legs
) == 0) {
1460 unlinked_pool_del_and_free(unlinked
, is_client
);
1461 } else if (!shutting_down
) {
1462 /* Launch a new leg for this set to recover. */
1463 if (CIRCUIT_IS_ORIGIN(circ
)) {
1464 conflux_launch_leg(nonce
);
1467 /* After this, it might have been freed. */
1470 /* Unlinked circuits should not have attached streams, but check
1471 * anyway, because The Maze. */
1472 validate_circ_has_no_streams(circ
);
1475 /** Update all stream pointers to point to this circuit.
1476 * This is used when a linked circuit is closed and we need to update the
1477 * streams to point to the remaining circuit
1480 linked_update_stream_backpointers(circuit_t
*circ
)
1483 tor_assert_nonfatal(circ
->conflux
);
1485 if (CIRCUIT_IS_ORIGIN(circ
)) {
1486 origin_circuit_t
*ocirc
= TO_ORIGIN_CIRCUIT(circ
);
1487 tor_assert_nonfatal(circ
->purpose
== CIRCUIT_PURPOSE_CONFLUX_LINKED
);
1488 /* Iterate over stream list using next_stream pointer, until null */
1489 for (edge_connection_t
*stream
= ocirc
->p_streams
; stream
;
1490 stream
= stream
->next_stream
) {
1491 /* Update the circuit pointer of each stream */
1492 stream
->on_circuit
= circ
;
1493 stream
->cpath_layer
= ocirc
->cpath
->prev
;
1496 or_circuit_t
*orcirc
= TO_OR_CIRCUIT(circ
);
1497 /* Iterate over stream list using next_stream pointer, until null */
1498 for (edge_connection_t
*stream
= orcirc
->n_streams
; stream
;
1499 stream
= stream
->next_stream
) {
1500 /* Update the circuit pointer of each stream */
1501 stream
->on_circuit
= circ
;
1503 /* Iterate over stream list using next_stream pointer, until null */
1504 for (edge_connection_t
*stream
= orcirc
->resolving_streams
; stream
;
1505 stream
= stream
->next_stream
) {
1506 /* Update the circuit pointer of each stream */
1507 stream
->on_circuit
= circ
;
1512 /** Nullify all streams of the given circuit. */
1514 linked_nullify_streams(circuit_t
*circ
)
1518 if (CIRCUIT_IS_ORIGIN(circ
)) {
1519 origin_circuit_t
*ocirc
= TO_ORIGIN_CIRCUIT(circ
);
1520 ocirc
->p_streams
= NULL
;
1521 ocirc
->half_streams
= NULL
;
1523 or_circuit_t
*orcirc
= TO_OR_CIRCUIT(circ
);
1524 orcirc
->n_streams
= NULL
;
1525 orcirc
->resolving_streams
= NULL
;
1529 /** The given circuit is already linked to a set and has been closed. Remove it
1530 * from the set and free the pool if no more legs. */
1532 linked_circuit_closed(circuit_t
*circ
)
1534 bool is_client
= false;
1535 bool full_teardown
= false;
1536 uint8_t nonce
[DIGEST256_LEN
] = {0};
1539 tor_assert(circ
->conflux
);
1541 if (CIRCUIT_IS_ORIGIN(circ
)) {
1542 tor_assert_nonfatal(circ
->purpose
== CIRCUIT_PURPOSE_CONFLUX_LINKED
);
1546 /* Unlink circuit from its conflux object. */
1547 full_teardown
= cfx_del_leg(circ
->conflux
, circ
);
1549 if (CONFLUX_NUM_LEGS(circ
->conflux
) == 0) {
1550 /* Last leg, remove conflux object from linked set. */
1551 linked_pool_del(circ
->conflux
->nonce
, is_client
);
1553 /* If there are other circuits, update streams backpointers and
1554 * nullify the stream lists. We do not free those streams in circuit_free_.
1555 * (They only get freed when the last circuit is freed). */
1556 conflux_leg_t
*leg
= smartlist_get(circ
->conflux
->legs
, 0);
1557 linked_update_stream_backpointers(leg
->circ
);
1558 linked_nullify_streams(circ
);
1561 /* Keep the nonce so we can use it through out the rest of the function in
1562 * case we nullify the conflux object before. Reason is that in the case of a
1563 * full teardown, this function becomes basically recursive and so we must
1564 * nullify the conflux object of this circuit now before the recursiveness
1565 * starts leading to all legs being removed and thus not noticing if we are
1566 * the last or the first.
1568 * Not the prettiest but that is the price to pay to live in the C-tor maze
1569 * and protected by ballrogs. */
1570 memcpy(nonce
, circ
->conflux
->nonce
, sizeof(nonce
));
1572 /* Nullify the conflux object from the circuit being closed iff we have more
1573 * legs. Reason being that the last leg needs to have the conflux object
1574 * attached to the circuit so it can be freed in conflux_circuit_free(). */
1575 if (CONFLUX_NUM_LEGS(circ
->conflux
) > 0) {
1576 circ
->conflux
= NULL
;
1579 /* If this was a teardown condition, we need to mark other circuits,
1580 * including any potential unlinked circuits, for close.
1582 * This call is recursive in the sense that linked_circuit_closed() will end
1583 * up being called for all legs and so by the time we come back here, the
1584 * linked is likely entirely gone. Thus why this is done last. */
1585 if (full_teardown
) {
1586 conflux_mark_all_for_close(nonce
, is_client
, END_CIRC_REASON_FINISHED
);
1590 /** The given circuit is being freed and it is a linked leg. Clean up and free
1591 * anything that has to do with this circuit.
1593 * After this call, the circuit should NOT be referenced anymore anywhere. */
1595 linked_circuit_free(circuit_t
*circ
, bool is_client
)
1598 tor_assert(circ
->conflux
);
1600 tor_assert(circ
->purpose
== CIRCUIT_PURPOSE_CONFLUX_LINKED
);
1603 /* Circuit can be freed without being closed and so we try to delete this leg
1604 * so we can learn if this circuit is the last leg or not. */
1605 cfx_del_leg(circ
->conflux
, circ
);
1607 if (CONFLUX_NUM_LEGS(circ
->conflux
) > 0) {
1608 /* The last leg will free the streams but until then, we nullify to avoid
1609 * use-after-free. */
1610 linked_nullify_streams(circ
);
1612 /* We are the last leg. */
1614 /* Remove from pool in case it is still lingering there else we'll end up
1615 * in a double free situation. */
1616 linked_pool_del(circ
->conflux
->nonce
, is_client
);
1618 /* If there is an unlinked circuit that was also created for this set, we
1619 * need to look for it, and tell it is no longer part of a linked set
1620 * anymore, so it can be freed properly, or can complete the link if it is
1621 * able to. Effectively, the conflux_t object lifetime is longer than
1622 * either the linked or unlinked sets by themselves. This is a situation we
1623 * could cover with handles, but so far, it is not clear they are an
1624 * obvious benefit for other cases than this one. */
1625 unlinked_circuits_t
*unlinked
=
1626 unlinked_pool_get(circ
->conflux
->nonce
, is_client
);
1628 tor_assert(unlinked
->is_for_linked_set
);
1629 unlinked
->is_for_linked_set
= false;
1631 /* We are the last one, clear the conflux object. If an unlinked object
1632 * has a reference to it, it won't get freed due to is_for_linked_set
1634 conflux_free(circ
->conflux
);
1639 /** The given circuit is being freed and it is an unlinked leg. Clean up and
1640 * free anything that has to do with this circuit.
1642 * After this call, the circuit should NOT be referenced anymore anywhere. */
1644 unlinked_circuit_free(circuit_t
*circ
, bool is_client
)
1647 tor_assert(circ
->conflux_pending_nonce
);
1649 tor_assert(circ
->purpose
== CIRCUIT_PURPOSE_CONFLUX_UNLINKED
);
1652 /* Cleanup circuit reference if a leg exists. This is possible if the circuit
1653 * was not marked for close before being freed. */
1654 leg_t
*leg
= unlinked_leg_find(circ
, is_client
);
1659 /* Null pointers are safe here. */
1660 tor_free(circ
->conflux_pending_nonce
);
1663 /** Circuit has been marked for close. */
1665 conflux_circuit_has_closed(circuit_t
*circ
)
1667 /* The unlinked case. If an unlinked set exists, we delete the leg and then
1668 * attempt to finalize it. After that, we'll launch a new leg to recover. */
1669 if (circ
->conflux_pending_nonce
) {
1670 unlinked_circuit_closed(circ
);
1671 } else if (circ
->conflux
) {
1672 linked_circuit_closed(circ
);
1676 /** Circuit with conflux purpose just opened. */
1678 conflux_circuit_has_opened(origin_circuit_t
*orig_circ
)
1680 circuit_t
*circ
= NULL
;
1683 tor_assert(orig_circ
);
1685 circ
= TO_CIRCUIT(orig_circ
);
1687 /* Extra safety layer so we never let a circuit opens if conflux is not
1689 if (!conflux_is_enabled(circ
)) {
1690 circuit_mark_for_close(circ
, END_CIRC_REASON_TORPROTOCOL
);
1691 static ratelim_t conflux_ratelim
= RATELIM_INIT(600);
1692 log_fn_ratelim(&conflux_ratelim
, LOG_NOTICE
, LD_CIRC
,
1693 "Conflux circuit opened without negotiating "
1694 "congestion control");
1698 /* Unrelated to conflux. */
1699 if (circ
->conflux_pending_nonce
== NULL
) {
1703 log_info(LD_CIRC
, "Conflux circuit has opened with nonce %s",
1704 fmt_nonce(circ
->conflux_pending_nonce
));
1706 leg
= unlinked_leg_find(circ
, true);
1708 log_warn(LD_CIRC
, "Unable to find conflux leg in unlinked set.");
1712 /* On failure here, the circuit is closed and thus the leg and unlinked set
1713 * will be cleaned up. */
1714 if (!conflux_cell_send_link(leg
->link
, orig_circ
)) {
1718 /* Mark the leg on when the LINK cell is sent. Used to timeout the circuit
1719 * for a minimum RTT when getting the LINKED. */
1720 leg
->link_sent_usec
= monotime_absolute_usec();
1723 validate_circ_has_no_streams(circ
);
1727 /** Process a CONFLUX_LINK cell which arrived on the given circuit. */
1729 conflux_process_link(circuit_t
*circ
, const cell_t
*cell
,
1730 const uint16_t cell_len
)
1732 unlinked_circuits_t
*unlinked
= NULL
;
1733 conflux_cell_link_t
*link
= NULL
;
1738 if (!conflux_is_enabled(circ
)) {
1739 circuit_mark_for_close(circ
, END_CIRC_REASON_TORPROTOCOL
);
1743 /* This cell can't be received on an origin circuit because only the endpoint
1744 * creating the circuit sends it. */
1745 if (CIRCUIT_IS_ORIGIN(circ
)) {
1746 log_fn(LOG_PROTOCOL_WARN
, LD_CIRC
,
1747 "Got a CONFLUX_LINK cell on an origin circuit. Closing circuit.");
1748 circuit_mark_for_close(circ
, END_CIRC_REASON_TORPROTOCOL
);
1752 if (!conflux_validate_source_hop(circ
, NULL
)) {
1753 log_fn(LOG_PROTOCOL_WARN
, LD_CIRC
,
1754 "Got a CONFLUX_LINK with further hops. Closing circuit.");
1755 circuit_mark_for_close(circ
, END_CIRC_REASON_TORPROTOCOL
);
1759 if (circ
->conflux_pending_nonce
) {
1760 log_fn(LOG_PROTOCOL_WARN
, LD_CIRC
,
1761 "Got a CONFLUX_LINK on a circuit with a pending nonce. "
1762 "Closing circuit.");
1763 circuit_mark_for_close(circ
, END_CIRC_REASON_TORPROTOCOL
);
1767 if (circ
->conflux
) {
1768 log_fn(LOG_PROTOCOL_WARN
, LD_CIRC
,
1769 "Got a CONFLUX_LINK on an already linked circuit "
1770 "Closing circuit.");
1771 circuit_mark_for_close(circ
, END_CIRC_REASON_TORPROTOCOL
);
1775 /* On errors, logging is emitted in this parsing function. */
1776 link
= conflux_cell_parse_link(cell
, cell_len
);
1778 log_fn(LOG_PROTOCOL_WARN
, LD_CIRC
, "Unable to parse "
1779 "CONFLUX_LINK cell. Closing circuit.");
1780 circuit_mark_for_close(circ
, END_CIRC_REASON_TORPROTOCOL
);
1784 log_info(LD_CIRC
, "Processing a CONFLUX_LINK for set %s",
1785 fmt_nonce(link
->nonce
));
1787 /* Consider this circuit a new leg. We'll now attempt to attach it to an
1788 * existing set or unlinked one. */
1789 leg_t
*leg
= leg_new(circ
, link
);
1790 unlinked
= unlinked_get_or_create(link
->nonce
, false);
1791 tor_assert(unlinked
);
1793 /* Attach leg to the unlinked set. */
1794 unlinked_leg_add(unlinked
, leg
);
1796 /* Set the circuit in a pending conflux state for the LINKED_ACK. */
1797 circ
->conflux_pending_nonce
= tor_memdup(leg
->link
->nonce
,
1798 sizeof(leg
->link
->nonce
));
1800 /* Mark when we send the LINKED. */
1801 leg
->link_sent_usec
= monotime_absolute_usec();
1804 uint64_t last_seq_sent
= conflux_get_max_seq_sent(unlinked
->cfx
);
1805 uint64_t last_seq_recv
= unlinked
->cfx
->last_seq_delivered
;
1807 // TODO-329-ARTI: To support resumption/retransmit, the server should
1808 // store the last_seq_sent now, so that it can know how much data
1809 // to retransmit to the server after link. C-Tor will not be implementing
1810 // this, but arti and arti-relay could (if resumption seems worthwhile;
1811 // it may not be worth the memory storage there, either).
1813 uint8_t nonce
[DIGEST256_LEN
];
1814 memcpy(nonce
, circ
->conflux_pending_nonce
, sizeof(nonce
));
1816 /* Link the circuit to the a conflux set immediately before the LINKED is
1817 * sent. Reason is that once the client sends the LINKED_ACK, there is a race
1818 * with the BEGIN cell that can be sent immediately after and arrive first.
1819 * And so, we need to sync the streams before that happens that is before we
1820 * receive the LINKED_ACK. */
1821 if (link_circuit(circ
) != ERR_LINK_CIRC_OK
) {
1822 circuit_mark_for_close(circ
, END_CIRC_REASON_TORPROTOCOL
);
1826 /* Exits should always request min latency from clients */
1827 conflux_cell_link_t
*linked
= conflux_cell_new_link(nonce
, last_seq_sent
,
1831 conflux_cell_send_linked(linked
, TO_OR_CIRCUIT(circ
));
1838 /** Process a CONFLUX_LINKED cell which arrived on the given circuit. */
1840 conflux_process_linked(circuit_t
*circ
, crypt_path_t
*layer_hint
,
1842 const uint16_t cell_len
)
1844 conflux_cell_link_t
*link
= NULL
;
1849 * There several ways a malicious exit could create problems when sending
1850 * back this LINKED cell.
1852 * 1. Using a different nonce that it knows about from another set. Accepting
1853 * it would mean a confirmation attack of linking sets to the same client.
1854 * To address that, the cell nonce MUST be matched with the circuit nonce.
1856 * 2. Re-Sending a LINKED cell on an already linked circuit could create side
1857 * channel attacks or unpredictable issues. Circuit is closed.
1859 * 3. Receiving a LINKED cell on a circuit that was not expecting it. Again,
1860 * as (2), can create side channel(s). Circuit is closed.
1862 * 4. Receiving a LINKED cell from the another hop other than the last one
1863 * (exit). Same as (2) and (3) in terms of issues. Circuit is closed.
1866 if (!conflux_is_enabled(circ
)) {
1867 circuit_mark_for_close(circ
, END_CIRC_REASON_TORPROTOCOL
);
1871 /* LINKED cell are in response to a LINK cell which are only sent on an
1872 * origin circuit and thus received on such.*/
1873 if (!CIRCUIT_IS_ORIGIN(circ
)) {
1874 log_fn(LOG_PROTOCOL_WARN
, LD_CIRC
,
1875 "Received CONFLUX_LINKED cell on a non origin circuit.");
1879 if (!circ
->conflux_pending_nonce
) {
1880 log_fn(LOG_PROTOCOL_WARN
, LD_CIRC
,
1881 "Received a CONFLUX_LINKED cell without having sent a "
1882 "CONFLUX_LINK cell. Closing circuit.");
1886 if (circ
->conflux
) {
1887 log_fn(LOG_PROTOCOL_WARN
, LD_CIRC
,
1888 "Received a CONFLUX_LINKED cell on a circuit that is already "
1889 "linked. Closing circuit.");
1893 if (!conflux_validate_source_hop(circ
, layer_hint
)) {
1894 log_fn(LOG_PROTOCOL_WARN
, LD_CIRC
,
1895 "Got a CONFLUX_LINKED from wrong hop on circuit. Closing circuit.");
1899 tor_assert_nonfatal(circ
->purpose
== CIRCUIT_PURPOSE_CONFLUX_UNLINKED
);
1901 /* On errors, logging is emitted in this parsing function. */
1902 link
= conflux_cell_parse_link(cell
, cell_len
);
1907 log_info(LD_CIRC
, "Processing a CONFLUX_LINKED for set %s",
1908 fmt_nonce(link
->nonce
));
1910 /* Make sure the cell nonce matches the one on the circuit that was
1911 * previously set by the CONFLUX_LINK cell. */
1912 if (tor_memneq(link
->nonce
, circ
->conflux_pending_nonce
,
1913 sizeof(*link
->nonce
))) {
1914 log_fn(LOG_PROTOCOL_WARN
, LD_CIRC
,
1915 "Received CONFLUX_LINKED but circuit nonce doesn't match "
1916 "cell nonce. Closing circuit.");
1920 /* Find the leg from the associated unlinked set. */
1921 leg_t
*leg
= unlinked_leg_find(circ
, true);
1923 log_warn(LD_CIRC
, "Received CONFLUX_LINKED but can't find "
1924 "associated leg. Closing circuit.");
1928 log_info(LD_CIRC
, "Successfully processed a CONFLUX_LINKED cell.");
1930 /* Free the old link, and store the new one. We need to validate
1931 * the one we get during finalize, not the one we sent. */
1932 tor_free(leg
->link
);
1935 /* Record the RTT for this circuit. On failure, it means the RTT was too
1936 * high, we relaunch to recover. */
1937 if (!record_rtt(circ
, true)) {
1941 /* The following will link the circuit with its set and attempt to finalize
1942 * the set if all expected legs have linked. On error, we close the circuit
1943 * because it means the unlinked set needs to be teardowned. */
1944 link_circ_err_t err
= link_circuit(circ
);
1946 case ERR_LINK_CIRC_OK
:
1947 /* Successfully linked. */
1949 case ERR_LINK_CIRC_INVALID_LEG
:
1950 case ERR_LINK_CIRC_MISSING_SET
:
1951 /* No relaunch if the leg is invalid or the set is not found as in the
1952 * nonce is unknown. */
1954 case ERR_LINK_CIRC_BAD_RTT
:
1955 case ERR_LINK_CIRC_MISSING_LEG
:
1959 /* We can send the ack only if we finalize. This will not cause issues,
1960 * because LINKED_ACK is exempted from multiplexing in
1961 * conflux_should_multiplex(). */
1962 if (!conflux_cell_send_linked_ack(TO_ORIGIN_CIRCUIT(circ
))) {
1963 /* On failure, the circuit is closed by the underlying function(s). */
1967 /* If this set is ready to use with a valid conflux set, try any pending
1969 if (circ
->conflux
) {
1970 connection_ap_attach_pending(1);
1976 circuit_mark_for_close(circ
, END_CIRC_REASON_TORPROTOCOL
);
1982 /** Process a CONFLUX_LINKED_ACK cell which arrived on the given circuit. */
1984 conflux_process_linked_ack(circuit_t
*circ
)
1988 if (!conflux_is_enabled(circ
)) {
1992 if (CIRCUIT_IS_ORIGIN(circ
)) {
1993 log_fn(LOG_PROTOCOL_WARN
, LD_CIRC
,
1994 "Received CONFLUX_LINKED_ACK cell on an origin circuit. Closing.");
1998 if (!conflux_validate_source_hop(circ
, NULL
)) {
1999 log_fn(LOG_PROTOCOL_WARN
, LD_CIRC
,
2000 "Got a CONFLUX_LINKED_ACK with further hops. Closing circuit.");
2004 if (BUG(!circ
->conflux
)) {
2005 log_fn(LOG_PROTOCOL_WARN
, LD_CIRC
,
2006 "Received a CONFLUX_LINKED_ACK cell on a circuit that is not"
2007 "linked. Closing circuit.");
2011 log_info(LD_CIRC
, "Processing a CONFLUX_LINKED_ACK for set %s",
2012 fmt_nonce(circ
->conflux
->nonce
));
2014 /* Record the RTT for this circuit. This should not fail */
2015 if (BUG(!record_rtt(circ
, false))) {
2022 circuit_mark_for_close(circ
, END_CIRC_REASON_TORPROTOCOL
);
2025 /** Called when a circuit is freed.
2027 * It is possible a conflux circuit gets freed without being closed (for
2028 * instance SIGTERM) and so this callback is needed in order to finalize the
2031 conflux_circuit_about_to_free(circuit_t
*circ
)
2035 bool is_client
= CIRCUIT_IS_ORIGIN(circ
);
2037 if (circ
->conflux
) {
2038 linked_circuit_free(circ
, is_client
);
2039 } else if (circ
->conflux_pending_nonce
) {
2040 unlinked_circuit_free(circ
, is_client
);
2043 /* Whatever happens, nullify all conflux related pointers. */
2044 circ
->conflux
= NULL
;
2045 circ
->conflux_pending_nonce
= NULL
;
2048 /** Initialize the conflux pool subsystem. This is called by the subsys
2051 conflux_pool_init(void)
2053 if (!client_linked_pool
) {
2054 client_linked_pool
= digest256map_new();
2056 if (!client_unlinked_pool
) {
2057 client_unlinked_pool
= digest256map_new();
2059 if (!server_linked_pool
) {
2060 server_linked_pool
= digest256map_new();
2062 if (!server_unlinked_pool
) {
2063 server_unlinked_pool
= digest256map_new();
2068 * Return a description of all linked and unlinked circuits associated
2069 * with a conflux set.
2071 * For use in rare bug cases that are hard to diagnose.
2074 conflux_log_set(int loglevel
, const conflux_t
*cfx
, bool is_client
)
2080 "Conflux %s: %d linked, %d launched. Delivered: %"PRIu64
"; "
2081 "teardown: %d; Current: %p, Previous: %p",
2082 fmt_nonce(cfx
->nonce
), smartlist_len(cfx
->legs
),
2083 cfx
->num_leg_launch
,
2084 cfx
->last_seq_delivered
, cfx
->in_full_teardown
,
2085 cfx
->curr_leg
, cfx
->prev_leg
);
2087 // Log all linked legs
2089 CONFLUX_FOR_EACH_LEG_BEGIN(cfx
, leg
) {
2090 const struct congestion_control_t
*cc
= circuit_ccontrol(leg
->circ
);
2091 log_fn(loglevel
, LD_BUG
,
2092 " - Linked Leg %d purpose=%d; RTT %"PRIu64
", sent: %"PRIu64
2093 "; sent: %"PRIu64
", recv: %"PRIu64
", infl: %"PRIu64
", "
2094 "ptr: %p, idx: %d, marked: %d",
2095 legs
, leg
->circ
->purpose
, leg
->circ_rtts_usec
,
2096 leg
->linked_sent_usec
, leg
->last_seq_recv
,
2097 leg
->last_seq_sent
, cc
->inflight
, leg
->circ
,
2098 leg
->circ
->global_circuitlist_idx
,
2099 leg
->circ
->marked_for_close
);
2101 } CONFLUX_FOR_EACH_LEG_END(leg
);
2103 // Look up the nonce to see if we have any unlinked circuits.
2104 unlinked_circuits_t
*unlinked
= unlinked_pool_get(cfx
->nonce
, is_client
);
2106 // Log the number of legs and the is_for_linked_set status
2107 log_fn(loglevel
, LD_BUG
, " - Unlinked set: %d legs, for link: %d",
2108 smartlist_len(unlinked
->legs
), unlinked
->is_for_linked_set
);
2110 SMARTLIST_FOREACH_BEGIN(unlinked
->legs
, leg_t
*, leg
) {
2111 log_fn(loglevel
, LD_BUG
,
2112 " Unlinked Leg: %d purpose=%d; linked: %d, RTT %"PRIu64
", "
2113 "sent: %"PRIu64
" link ptr %p, circ ptr: %p, idx: %d, marked: %d",
2114 legs
, leg
->circ
->purpose
, leg
->linked
,
2115 leg
->rtt_usec
, leg
->link_sent_usec
,
2116 leg
->link
, leg
->circ
,
2117 leg
->circ
->global_circuitlist_idx
,
2118 leg
->circ
->marked_for_close
);
2120 } SMARTLIST_FOREACH_END(leg
);
2124 /** Free and clean up the conflux pool subsystem. This is called by the subsys
2125 * manager AFTER all circuits have been freed which implies that all objects in
2126 * the pools aren't referenced anymore. */
2128 conflux_pool_free_all(void)
2130 shutting_down
= true;
2132 digest256map_free(client_linked_pool
, free_conflux_void_
);
2133 digest256map_free(server_linked_pool
, free_conflux_void_
);
2134 digest256map_free(client_unlinked_pool
, free_unlinked_void_
);
2135 digest256map_free(server_unlinked_pool
, free_unlinked_void_
);