2 * Connection oriented routing
3 * Copyright (C) 2007-2021 Michael Blizek
5 * This program is free software; you can redistribute it and/or
6 * modify it under the terms of the GNU General Public License
7 * as published by the Free Software Foundation; either version 2
8 * of the License, or (at your option) any later version.
10 * This program is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
15 * You should have received a copy of the GNU General Public License
16 * along with this program; if not, write to the Free Software
17 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
21 #include <linux/delay.h>
25 static DEFINE_SPINLOCK(cor_neighbor_list_lock
);
26 static LIST_HEAD(cor_nb_list
);
27 static struct kmem_cache
*cor_nb_slab
;
28 atomic_t cor_num_neighs
;
30 static DEFINE_SPINLOCK(cor_connid_gen
);
33 void cor_neighbor_free(struct kref
*ref
)
35 struct cor_neighbor
*nb
= container_of(ref
, struct cor_neighbor
, ref
);
37 WARN_ONCE(list_empty(&(nb
->cmsg_queue_pong
)) == 0,
38 "cor_neighbor_free(): nb->cmsg_queue_pong is not empty");
39 WARN_ONCE(list_empty(&(nb
->cmsg_queue_ack_fast
)) == 0,
40 "cor_neighbor_free(): nb->cmsg_queue_ack_fast is not empty");
41 WARN_ONCE(list_empty(&(nb
->cmsg_queue_ack_slow
)) == 0,
42 "cor_neighbor_free(): nb->cmsg_queue_ack_slow is not empty");
43 WARN_ONCE(list_empty(&(nb
->cmsg_queue_ackconn_urgent
)) == 0,
44 "cor_neighbor_free(): nb->cmsg_queue_ackconn_urgent is not empty");
45 WARN_ONCE(list_empty(&(nb
->cmsg_queue_ackconn_lowlat
)) == 0,
46 "cor_neighbor_free(): nb->cmsg_queue_ackconn_lowlat is not empty");
47 WARN_ONCE(list_empty(&(nb
->cmsg_queue_ackconn_highlat
)) == 0,
48 "cor_neighbor_free(): nb->cmsg_queue_ackconn_highlat is not empty");
49 WARN_ONCE(list_empty(&(nb
->cmsg_queue_conndata_lowlat
)) == 0,
50 "cor_neighbor_free(): nb->cmsg_queue_conndata_lowlat is not empty");
51 WARN_ONCE(list_empty(&(nb
->cmsg_queue_conndata_highlat
)) == 0,
52 "cor_neighbor_free(): nb->cmsg_queue_conndata_highlat is not empty");
53 WARN_ONCE(list_empty(&(nb
->cmsg_queue_other
)) == 0,
54 "cor_neighbor_free(): nb->cmsg_queue_other is not empty");
55 WARN_ONCE(nb
->pending_conn_resets_rb
.rb_node
!= 0,
56 "cor_neighbor_free(): nb->pending_conn_resets_rb is not empty");
57 WARN_ONCE(nb
->rb_kp
.in_queue
!= RB_INQUEUE_FALSE
,
58 "cor_neighbor_free(): nb->rb_kp.in_queue is not RB_INQUEUE_FALSE");
59 WARN_ONCE(nb
->rb_cr
.in_queue
!= RB_INQUEUE_FALSE
,
60 "cor_neighbor_free(): nb->rb_cr.in_queue is not RB_INQUEUE_FALSE");
61 WARN_ONCE(nb
->rb
.in_queue
!= RB_INQUEUE_FALSE
,
62 "cor_neighbor_free(): nb->rb.in_queue is not RB_INQUEUE_FALSE");
63 WARN_ONCE(list_empty(&(nb
->conns_waiting
.lh
)) == 0,
64 "cor_neighbor_free(): nb->conns_waiting.lh is not empty");
65 WARN_ONCE(list_empty(&(nb
->conns_waiting
.lh_nextpass
)) == 0,
66 "cor_neighbor_free(): nb->conns_waiting.lh_nextpass is not empty");
67 WARN_ONCE(nb
->str_timer_pending
!= 0,
68 "cor_neighbor_free(): nb->str_timer_pending is not 0");
69 WARN_ONCE(nb
->connid_rb
.rb_node
!= 0,
70 "cor _neighbor_free(): nb->connid_rb is not empty");
71 WARN_ONCE(nb
->connid_reuse_rb
.rb_node
!= 0,
72 "cor_neighbor_free(): nb->connid_reuse_rb is not empty");
73 WARN_ONCE(list_empty(&(nb
->connid_reuse_list
)) == 0,
74 "cor_neighbor_free(): nb->connid_reuse_list is not empty");
75 WARN_ONCE(nb
->kp_retransmits_rb
.rb_node
!= 0,
76 "cor_neighbor_free(): nb->kp_retransmits_rb is not empty");
77 WARN_ONCE(list_empty(&(nb
->snd_conn_idle_list
)) == 0,
78 "cor_neighbor_free(): nb->snd_conn_idle_list is not empty");
79 WARN_ONCE(list_empty(&(nb
->snd_conn_busy_list
)) == 0,
80 "cor_neighbor_free(): nb->snd_conn_busy_list is not empty");
81 WARN_ONCE(list_empty(&(nb
->retrans_fast_list
)) == 0,
82 "cor_neighbor_free(): nb->retrans_fast_list is not empty");
83 WARN_ONCE(list_empty(&(nb
->retrans_slow_list
)) == 0,
84 "cor_neighbor_free(): nb->retrans_slow_list is not empty");
85 WARN_ONCE(list_empty(&(nb
->retrans_conn_lowlatency_list
)) == 0,
86 "cor_neighbor_free(): nb->retrans_conn_lowlatency_list is not empty");
87 WARN_ONCE(list_empty(&(nb
->retrans_conn_highlatency_list
)) == 0,
88 "cor_neighbor_free(): nb->retrans_conn_highlatency_list is not empty");
90 /* printk(KERN_ERR "neighbor free\n"); */
91 BUG_ON(nb
->nb_list
.next
!= LIST_POISON1
);
92 BUG_ON(nb
->nb_list
.prev
!= LIST_POISON2
);
100 kref_put(&(nb
->queue
->ref
), cor_free_qos
);
102 kmem_cache_free(cor_nb_slab
, nb
);
103 atomic_dec(&cor_num_neighs
);
106 static void cor_stall_timer(struct work_struct
*work
);
108 static void _cor_reset_neighbor(struct work_struct
*work
);
110 static struct cor_neighbor
*cor_alloc_neighbor(gfp_t allocflags
)
112 struct cor_neighbor
*nb
;
115 if (atomic_inc_return(&cor_num_neighs
) >= MAX_NEIGHBORS
) {
116 atomic_dec(&cor_num_neighs
);
120 nb
= kmem_cache_alloc(cor_nb_slab
, allocflags
);
121 if (unlikely(nb
== 0))
124 memset(nb
, 0, sizeof(struct cor_neighbor
));
126 kref_init(&(nb
->ref
));
127 atomic_set(&(nb
->sessionid_rcv_needed
), 1);
128 atomic_set(&(nb
->sessionid_snd_needed
), 1);
129 timer_setup(&(nb
->cmsg_timer
), cor_controlmsg_timerfunc
, 0);
130 spin_lock_init(&(nb
->cmsg_lock
));
131 INIT_LIST_HEAD(&(nb
->cmsg_queue_pong
));
132 INIT_LIST_HEAD(&(nb
->cmsg_queue_ack_fast
));
133 INIT_LIST_HEAD(&(nb
->cmsg_queue_ack_slow
));
134 INIT_LIST_HEAD(&(nb
->cmsg_queue_ackconn_urgent
));
135 INIT_LIST_HEAD(&(nb
->cmsg_queue_ackconn_lowlat
));
136 INIT_LIST_HEAD(&(nb
->cmsg_queue_ackconn_highlat
));
137 INIT_LIST_HEAD(&(nb
->cmsg_queue_conndata_lowlat
));
138 INIT_LIST_HEAD(&(nb
->cmsg_queue_conndata_highlat
));
139 INIT_LIST_HEAD(&(nb
->cmsg_queue_other
));
140 atomic_set(&(nb
->cmsg_pongs_retrans_cnt
), 0);
141 atomic_set(&(nb
->cmsg_othercnt
), 0);
142 atomic_set(&(nb
->cmsg_bulk_readds
), 0);
143 atomic_set(&(nb
->cmsg_delay_conndata
), 0);
144 atomic_set(&(nb
->rcvmtu_sendneeded
), 1);
145 nb
->last_ping_time
= jiffies
;
146 atomic_set(&(nb
->latency_retrans_us
), PING_GUESSLATENCY_MS
*1000);
147 atomic_set(&(nb
->latency_advertised_us
), PING_GUESSLATENCY_MS
*1000);
148 atomic_set(&(nb
->max_remote_ack_fast_delay_us
), 1000000);
149 atomic_set(&(nb
->max_remote_ack_slow_delay_us
), 1000000);
150 atomic_set(&(nb
->max_remote_ackconn_lowlat_delay_us
), 1000000);
151 atomic_set(&(nb
->max_remote_ackconn_highlat_delay_us
), 1000000);
152 atomic_set(&(nb
->max_remote_pong_delay_us
), 1000000);
153 atomic_set(&(nb
->remote_rcvmtu
), 128);
154 spin_lock_init(&(nb
->conns_waiting
.lock
));
155 INIT_LIST_HEAD(&(nb
->conns_waiting
.lh
));
156 INIT_LIST_HEAD(&(nb
->conns_waiting
.lh_nextpass
));
157 spin_lock_init(&(nb
->nbcongwin
.lock
));
158 atomic64_set(&(nb
->nbcongwin
.data_intransit
), 0);
159 atomic64_set(&(nb
->nbcongwin
.cwin
), 0);
160 spin_lock_init(&(nb
->state_lock
));
161 nb
->state
= NEIGHBOR_STATE_INITIAL
;
162 nb
->state_time
.initial_state_since
= jiffies
;
163 INIT_DELAYED_WORK(&(nb
->stalltimeout_timer
), cor_stall_timer
);
164 spin_lock_init(&(nb
->connid_lock
));
165 spin_lock_init(&(nb
->connid_reuse_lock
));
166 INIT_LIST_HEAD(&(nb
->connid_reuse_list
));
167 get_random_bytes((char *) &seqno
, sizeof(seqno
));
168 nb
->kpacket_seqno
= seqno
;
169 atomic64_set(&(nb
->priority_sum
), 0);
170 spin_lock_init(&(nb
->conn_list_lock
));
171 INIT_LIST_HEAD(&(nb
->snd_conn_idle_list
));
172 INIT_LIST_HEAD(&(nb
->snd_conn_busy_list
));
173 spin_lock_init(&(nb
->retrans_lock
));
174 INIT_LIST_HEAD(&(nb
->retrans_fast_list
));
175 INIT_LIST_HEAD(&(nb
->retrans_slow_list
));
176 spin_lock_init(&(nb
->retrans_conn_lock
));
177 INIT_LIST_HEAD(&(nb
->retrans_conn_lowlatency_list
));
178 INIT_LIST_HEAD(&(nb
->retrans_conn_highlatency_list
));
179 INIT_WORK(&(nb
->reset_neigh_work
), _cor_reset_neighbor
);
184 int cor_is_from_nb(struct sk_buff
*skb
, struct cor_neighbor
*nb
)
188 char source_hw
[MAX_ADDR_LEN
];
189 memset(source_hw
, 0, MAX_ADDR_LEN
);
190 if (skb
->dev
->header_ops
!= 0 &&
191 skb
->dev
->header_ops
->parse
!= 0)
192 skb
->dev
->header_ops
->parse(skb
, source_hw
);
194 rc
= (skb
->dev
== nb
->dev
&& memcmp(nb
->mac
, source_hw
,
199 struct cor_neighbor
*_cor_get_neigh_by_mac(struct net_device
*dev
,
202 struct list_head
*currlh
;
203 struct cor_neighbor
*ret
= 0;
205 spin_lock_bh(&cor_neighbor_list_lock
);
207 currlh
= cor_nb_list
.next
;
208 while (currlh
!= &cor_nb_list
) {
209 struct cor_neighbor
*curr
= container_of(currlh
,
210 struct cor_neighbor
, nb_list
);
212 BUG_ON(curr
->in_nb_list
== 0);
214 if (curr
->dev
== dev
&& memcmp(curr
->mac
, source_hw
,
215 MAX_ADDR_LEN
) == 0) {
217 cor_nb_kref_get(ret
, "stack");
221 currlh
= currlh
->next
;
224 spin_unlock_bh(&cor_neighbor_list_lock
);
226 if (ret
!= 0 && unlikely(cor_get_neigh_state(ret
) ==
227 NEIGHBOR_STATE_KILLED
)) {
228 cor_nb_kref_put(ret
, "stack");
235 struct cor_neighbor
*cor_get_neigh_by_mac(struct sk_buff
*skb
)
237 char source_hw
[MAX_ADDR_LEN
];
238 memset(source_hw
, 0, MAX_ADDR_LEN
);
239 if (skb
->dev
->header_ops
!= 0 &&
240 skb
->dev
->header_ops
->parse
!= 0)
241 skb
->dev
->header_ops
->parse(skb
, source_hw
);
243 return _cor_get_neigh_by_mac(skb
->dev
, source_hw
);
246 struct cor_neighbor
*cor_find_neigh(char *addr
, __u16 addrlen
)
248 struct list_head
*currlh
;
249 struct cor_neighbor
*ret
= 0;
251 if (addr
== 0 || addrlen
== 0)
254 spin_lock_bh(&cor_neighbor_list_lock
);
256 currlh
= cor_nb_list
.next
;
257 while (currlh
!= &cor_nb_list
) {
258 struct cor_neighbor
*curr
= container_of(currlh
,
259 struct cor_neighbor
, nb_list
);
261 BUG_ON(curr
->in_nb_list
== 0);
263 if (curr
->addr
!= 0 && curr
->addrlen
!= 0 &&
264 curr
->addrlen
== addrlen
&&
265 memcmp(curr
->addr
, addr
, addrlen
) == 0) {
267 cor_nb_kref_get(ret
, "stack");
272 currlh
= currlh
->next
;
276 spin_unlock_bh(&cor_neighbor_list_lock
);
278 if (ret
!= 0 && unlikely(cor_get_neigh_state(ret
) ==
279 NEIGHBOR_STATE_KILLED
)) {
280 cor_nb_kref_put(ret
, "stack");
287 void cor_resend_rcvmtu(struct net_device
*dev
)
289 struct list_head
*currlh
;
291 spin_lock_bh(&cor_neighbor_list_lock
);
293 currlh
= cor_nb_list
.next
;
294 while (currlh
!= &cor_nb_list
) {
295 struct cor_neighbor
*nb
= container_of(currlh
,
296 struct cor_neighbor
, nb_list
);
298 unsigned long iflags
;
305 spin_lock_irqsave(&(nb
->state_lock
), iflags
);
307 if (nb
->rcvmtu_allowed_countdown
!= 0)
308 nb
->rcvmtu_delayed_send_needed
= 1;
309 nb
->rcvmtu_allowed_countdown
= 3;
311 spin_unlock_irqrestore(&(nb
->state_lock
), iflags
);
314 currlh
= currlh
->next
;
317 spin_unlock_bh(&cor_neighbor_list_lock
);
320 __u32
cor_generate_neigh_list(char *buf
, __u32 buflen
)
322 struct list_head
*currlh
;
326 __u32 buf_offset
= 4;
331 * The variable length header rowcount need to be generated after the
332 * data. This is done by reserving the maximum space they could take. If
333 * they end up being smaller, the data is moved so that there is no gap.
337 BUG_ON(buflen
< buf_offset
);
340 rc
= cor_encode_len(buf
+ buf_offset
, buflen
- buf_offset
, 2);
345 BUG_ON(buflen
< buf_offset
+ 2);
346 cor_put_u16(buf
+ buf_offset
, LIST_NEIGH_FIELD_ADDR
);
349 rc
= cor_encode_len(buf
+ buf_offset
, buflen
- buf_offset
, 0);
354 BUG_ON(buflen
< buf_offset
+ 2);
355 cor_put_u16(buf
+ buf_offset
, LIST_NEIGH_FIELD_LATENCY
);
358 rc
= cor_encode_len(buf
+ buf_offset
, buflen
- buf_offset
, 1);
362 spin_lock_bh(&cor_neighbor_list_lock
);
364 currlh
= cor_nb_list
.next
;
365 while (currlh
!= &cor_nb_list
) {
366 struct cor_neighbor
*curr
= container_of(currlh
,
367 struct cor_neighbor
, nb_list
);
370 BUG_ON(curr
->in_nb_list
== 0);
372 state
= cor_get_neigh_state(curr
);
374 if (state
!= NEIGHBOR_STATE_ACTIVE
)
377 BUG_ON((curr
->addr
== 0) != (curr
->addrlen
== 0));
378 if (curr
->addr
== 0 || curr
->addrlen
== 0)
381 if (unlikely(buflen
< buf_offset
+ 4 + curr
->addrlen
+ 1))
385 rc
= cor_encode_len(buf
+ buf_offset
, buflen
- buf_offset
,
390 BUG_ON(curr
->addrlen
> buflen
- buf_offset
);
391 memcpy(buf
+ buf_offset
, curr
->addr
, curr
->addrlen
); /* addr */
392 buf_offset
+= curr
->addrlen
;
394 buf
[buf_offset
] = cor_enc_log_64_11(atomic_read(
395 &(curr
->latency_advertised_us
)));
398 BUG_ON(buf_offset
> buflen
);
403 currlh
= currlh
->next
;
406 spin_unlock_bh(&cor_neighbor_list_lock
);
408 rc
= cor_encode_len(buf
, 4, cnt
);
413 memmove(buf
+ ((__u32
) rc
), buf
+4, buf_offset
);
415 return buf_offset
- 4 + ((__u32
) rc
);
418 static void cor_reset_all_conns(struct cor_neighbor
*nb
)
421 unsigned long iflags
;
422 struct cor_conn
*trgt_out
;
423 struct cor_conn
*src_in
;
424 struct cor_conn_bidir
*cnb
;
427 spin_lock_irqsave(&(nb
->conn_list_lock
), iflags
);
429 if (!list_empty(&(nb
->snd_conn_busy_list
))) {
430 trgt_out
= container_of(nb
->snd_conn_busy_list
.next
,
431 struct cor_conn
, trgt
.out
.nb_list
);
432 } else if (!list_empty(&(nb
->snd_conn_idle_list
))) {
433 trgt_out
= container_of(nb
->snd_conn_idle_list
.next
,
434 struct cor_conn
, trgt
.out
.nb_list
);
436 spin_unlock_irqrestore(&(nb
->conn_list_lock
), iflags
);
440 cor_conn_kref_get(trgt_out
, "stack");
442 spin_unlock_irqrestore(&(nb
->conn_list_lock
), iflags
);
444 src_in
= cor_get_conn_reversedir(trgt_out
);
445 cnb
= cor_get_conn_bidir(trgt_out
);
447 spin_lock_bh(&(cnb
->cli
.rcv_lock
));
448 spin_lock_bh(&(cnb
->srv
.rcv_lock
));
450 if (unlikely(unlikely(src_in
->sourcetype
!= SOURCE_IN
) ||
451 unlikely(src_in
->src
.in
.nb
!= nb
))) {
456 rc
= cor_send_reset_conn(nb
, trgt_out
->trgt
.out
.conn_id
, 1);
458 if (unlikely(rc
!= 0))
461 if (trgt_out
->isreset
== 0)
462 trgt_out
->isreset
= 1;
465 spin_unlock_bh(&(cnb
->srv
.rcv_lock
));
466 spin_unlock_bh(&(cnb
->cli
.rcv_lock
));
469 cor_reset_conn(src_in
);
470 cor_conn_kref_put(src_in
, "stack");
472 cor_conn_kref_put(src_in
, "stack");
473 cor_nb_kref_get(nb
, "stalltimeout_timer");
474 schedule_delayed_work(&(nb
->stalltimeout_timer
), HZ
);
480 static void cor_delete_connid_reuse_items(struct cor_neighbor
*nb
);
482 static void _cor_reset_neighbor(struct work_struct
*work
)
484 struct cor_neighbor
*nb
= container_of(work
, struct cor_neighbor
,
487 cor_reset_all_conns(nb
);
488 cor_delete_connid_reuse_items(nb
);
490 spin_lock_bh(&cor_neighbor_list_lock
);
491 if (nb
->in_nb_list
!= 0) {
492 list_del(&(nb
->nb_list
));
494 cor_nb_kref_put_bug(nb
, "neigh_list");
496 spin_unlock_bh(&cor_neighbor_list_lock
);
498 cor_nb_kref_put(nb
, "reset_neigh_work");
501 static void cor_reset_neighbor(struct cor_neighbor
*nb
, int use_workqueue
)
503 unsigned long iflags
;
505 spin_lock_irqsave(&(nb
->state_lock
), iflags
);
506 /* if (nb->state != NEIGHBOR_STATE_KILLED) {
507 printk(KERN_ERR "cor_reset_neighbor\n");
510 nb
->state
= NEIGHBOR_STATE_KILLED
;
511 spin_unlock_irqrestore(&(nb
->state_lock
), iflags
);
515 schedule_work(&(nb
->reset_neigh_work
));
516 cor_nb_kref_get(nb
, "reset_neigh_work");
519 cor_reset_all_conns(nb
);
520 cor_delete_connid_reuse_items(nb
);
522 spin_lock_bh(&cor_neighbor_list_lock
);
523 if (nb
->in_nb_list
!= 0) {
524 list_del(&(nb
->nb_list
));
528 spin_unlock_bh(&cor_neighbor_list_lock
);
531 cor_nb_kref_put(nb
, "neigh_list");
535 void cor_reset_neighbors(struct net_device
*dev
)
537 struct list_head
*currlh
;
540 spin_lock_bh(&cor_neighbor_list_lock
);
542 currlh
= cor_nb_list
.next
;
543 while (currlh
!= &cor_nb_list
) {
544 unsigned long iflags
;
545 struct cor_neighbor
*currnb
= container_of(currlh
,
546 struct cor_neighbor
, nb_list
);
549 BUG_ON(currnb
->in_nb_list
== 0);
551 if (dev
!= 0 && currnb
->dev
!= dev
)
554 spin_lock_irqsave(&(currnb
->state_lock
), iflags
);
555 state
= currnb
->state
;
556 spin_unlock_irqrestore(&(currnb
->state_lock
), iflags
);
558 if (state
!= NEIGHBOR_STATE_KILLED
) {
559 spin_unlock_bh(&cor_neighbor_list_lock
);
560 cor_reset_neighbor(currnb
, 0);
565 currlh
= currlh
->next
;
568 spin_unlock_bh(&cor_neighbor_list_lock
);
571 static void cor_stall_timer(struct work_struct
*work
)
573 struct cor_neighbor
*nb
= container_of(to_delayed_work(work
),
574 struct cor_neighbor
, stalltimeout_timer
);
579 unsigned long iflags
;
581 spin_lock_irqsave(&(nb
->state_lock
), iflags
);
583 spin_unlock_irqrestore(&(nb
->state_lock
), iflags
);
585 if (nbstate
== NEIGHBOR_STATE_STALLED
) {
586 stall_time_ms
= jiffies_to_msecs(jiffies
-
587 nb
->state_time
.last_roundtrip
);
589 if (stall_time_ms
< NB_KILL_TIME_MS
) {
590 schedule_delayed_work(&(nb
->stalltimeout_timer
),
591 msecs_to_jiffies(NB_KILL_TIME_MS
-
596 cor_reset_neighbor(nb
, 1);
599 nb
->str_timer_pending
= 0;
600 cor_nb_kref_put(nb
, "stalltimeout_timer");
603 int cor_get_neigh_state(struct cor_neighbor
*nb
)
606 unsigned long iflags
;
611 spin_lock_irqsave(&(nb
->state_lock
), iflags
);
613 stall_time_ms
= jiffies_to_msecs(jiffies
-
614 nb
->state_time
.last_roundtrip
);
616 WARN_ONCE(likely(nb
->state
== NEIGHBOR_STATE_ACTIVE
) && unlikely(
617 jiffies_to_msecs(jiffies
- nb
->last_ping_time
) >
618 PING_FORCETIME_ACTIVEIDLE_MS
* 4) &&
619 nb
->ping_intransit
== 0,
620 "We have stopped sending pings to a neighbor!?");
622 if (likely(nb
->state
== NEIGHBOR_STATE_ACTIVE
) &&
623 unlikely(stall_time_ms
> NB_STALL_TIME_MS
) && (
624 nb
->ping_intransit
>= NB_STALL_MINPINGS
||
625 nb
->ping_intransit
>= PING_COOKIES_PER_NEIGH
)) {
626 nb
->state
= NEIGHBOR_STATE_STALLED
;
627 nb
->ping_success
= 0;
628 if (nb
->str_timer_pending
== 0) {
629 nb
->str_timer_pending
= 1;
630 cor_nb_kref_get(nb
, "stalltimeout_timer");
632 schedule_delayed_work(&(nb
->stalltimeout_timer
),
633 msecs_to_jiffies(NB_KILL_TIME_MS
-
637 /* printk(KERN_ERR "changed to stalled\n"); */
638 BUG_ON(nb
->ping_intransit
> PING_COOKIES_PER_NEIGH
);
639 } else if (unlikely(nb
->state
== NEIGHBOR_STATE_INITIAL
) &&
640 time_after(jiffies
, nb
->state_time
.initial_state_since
+
641 INITIAL_TIME_LIMIT_SEC
* HZ
)) {
642 spin_unlock_irqrestore(&(nb
->state_lock
), iflags
);
643 cor_reset_neighbor(nb
, 1);
644 return NEIGHBOR_STATE_KILLED
;
649 spin_unlock_irqrestore(&(nb
->state_lock
), iflags
);
654 static struct cor_ping_cookie
*cor_find_cookie(struct cor_neighbor
*nb
,
659 for(i
=0;i
<PING_COOKIES_PER_NEIGH
;i
++) {
660 if (nb
->cookies
[i
].cookie
== cookie
)
661 return &(nb
->cookies
[i
]);
666 static void cor_reset_cookie(struct cor_neighbor
*nb
, struct cor_ping_cookie
*c
)
671 if (nb
->cookie_unsent
!= c
->cookie
)
672 nb
->ping_intransit
--;
677 static __u32
sqrt(__u64 x
)
682 if (unlikely(x
<= 1))
686 y
= y
/2 + div64_u64(x
/2, y
);
687 if (unlikely(y
== 0))
691 if (unlikely(y
> U32_MAX
))
697 static __u32
cor_calc_newlatency(struct cor_neighbor
*nb_statelocked
,
698 __u32 oldlatency_us
, __s64 newlatency_ns
)
700 __s64 oldlatency
= oldlatency_us
* 1000LL;
703 if (unlikely(unlikely(nb_statelocked
->state
== NEIGHBOR_STATE_INITIAL
)&&
704 nb_statelocked
->ping_success
< 16))
705 newlatency
= div64_s64(
706 oldlatency
* nb_statelocked
->ping_success
+
708 nb_statelocked
->ping_success
+ 1);
710 newlatency
= (oldlatency
* 15 + newlatency_ns
) / 16;
712 newlatency
= div_s64(newlatency
+ 500, 1000);
714 if (unlikely(newlatency
< 0))
716 if (unlikely(newlatency
> U32_MAX
))
717 newlatency
= U32_MAX
;
719 return (__u32
) newlatency
;
722 static void cor_update_nb_latency(struct cor_neighbor
*nb_statelocked
,
723 struct cor_ping_cookie
*c
, __u32 respdelay
)
725 ktime_t now
= ktime_get();
727 __s64 pinglatency_retrans_ns
= ktime_to_ns(now
) -
728 ktime_to_ns(c
->time_sent
) - respdelay
* 1000LL;
729 __s64 pinglatency_advertised_ns
= ktime_to_ns(now
) -
730 ktime_to_ns(c
->time_created
) - respdelay
* 1000LL;
732 __u32 oldlatency_retrans_us
=
733 atomic_read(&(nb_statelocked
->latency_retrans_us
));
735 __u32 newlatency_retrans_us
= cor_calc_newlatency(nb_statelocked
,
736 oldlatency_retrans_us
, pinglatency_retrans_ns
);
738 atomic_set(&(nb_statelocked
->latency_retrans_us
),
739 newlatency_retrans_us
);
741 if (unlikely(unlikely(nb_statelocked
->state
== NEIGHBOR_STATE_INITIAL
)&&
742 nb_statelocked
->ping_success
< 16)) {
743 nb_statelocked
->latency_variance_retrans_us
=
744 ((__u64
) newlatency_retrans_us
) *
745 newlatency_retrans_us
;
746 atomic_set(&(nb_statelocked
->latency_stddev_retrans_us
), sqrt(
747 nb_statelocked
->latency_variance_retrans_us
));
748 } else if (pinglatency_retrans_ns
> oldlatency_retrans_us
*
750 __s64 newdiff
= div_s64(pinglatency_retrans_ns
-
751 oldlatency_retrans_us
* ((__s64
) 1000), 1000);
752 __u32 newdiff32
= (__u32
) (unlikely(newdiff
>= U32_MAX
) ?
754 __u64 newvar
= ((__u64
) newdiff32
) * newdiff32
;
756 __u64 oldval
= nb_statelocked
->latency_variance_retrans_us
;
758 if (unlikely(unlikely(newvar
> (1LL << 55)) || unlikely(
759 oldval
> (1LL << 55)))) {
760 nb_statelocked
->latency_variance_retrans_us
=
761 (oldval
/ 16) * 15 + newvar
/16;
763 nb_statelocked
->latency_variance_retrans_us
=
764 (oldval
* 15 + newvar
) / 16;
767 atomic_set(&(nb_statelocked
->latency_stddev_retrans_us
), sqrt(
768 nb_statelocked
->latency_variance_retrans_us
));
771 atomic_set(&(nb_statelocked
->latency_advertised_us
),
772 cor_calc_newlatency(nb_statelocked
,
773 atomic_read(&(nb_statelocked
->latency_advertised_us
)),
774 pinglatency_advertised_ns
));
776 nb_statelocked
->last_roundtrip_end
= now
;
779 static void cor_connid_used_pingsuccess(struct cor_neighbor
*nb
);
781 void cor_ping_resp(struct cor_neighbor
*nb
, __u32 cookie
, __u32 respdelay
)
783 unsigned long iflags
;
785 struct cor_ping_cookie
*c
;
787 int stalledresume
= 0;
789 int call_connidreuse
= 0;
790 int call_send_rcvmtu
= 0;
792 if (unlikely(cookie
== 0))
795 spin_lock_irqsave(&(nb
->state_lock
), iflags
);
797 c
= cor_find_cookie(nb
, cookie
);
799 if (unlikely(c
== 0))
802 atomic_set(&(nb
->sessionid_snd_needed
), 0);
804 call_connidreuse
= ktime_before_eq(nb
->last_roundtrip_end
,
807 cor_update_nb_latency(nb
, c
, respdelay
);
811 cor_reset_cookie(nb
, c
);
813 for(i
=0;i
<PING_COOKIES_PER_NEIGH
;i
++) {
814 if (nb
->cookies
[i
].cookie
!= 0 && ktime_before(
815 nb
->cookies
[i
].time_created
, c
->time_created
)) {
816 nb
->cookies
[i
].pongs
++;
817 if (nb
->cookies
[i
].pongs
>= PING_PONGLIMIT
) {
818 cor_reset_cookie(nb
, &(nb
->cookies
[i
]));
823 if (unlikely(nb
->state
== NEIGHBOR_STATE_INITIAL
||
824 nb
->state
== NEIGHBOR_STATE_STALLED
)) {
825 call_connidreuse
= 0;
827 if ((nb
->state
== NEIGHBOR_STATE_INITIAL
&&
828 nb
->ping_success
>= PING_SUCCESS_CNT_INIT
) || (
829 nb
->state
== NEIGHBOR_STATE_STALLED
&&
830 nb
->ping_success
>= PING_SUCCESS_CNT_STALLED
)) {
831 stalledresume
= (nb
->state
== NEIGHBOR_STATE_STALLED
);
832 nb
->state
= NEIGHBOR_STATE_ACTIVE
;
833 /* printk(KERN_ERR "changed to active\n"); */
837 if (likely(nb
->state
== NEIGHBOR_STATE_ACTIVE
) ||
838 nb
->state
== NEIGHBOR_STATE_STALLED
)
839 nb
->state_time
.last_roundtrip
= c
->jiffies_sent
;
841 if (c
== &(nb
->cookies
[0]) &&
842 unlikely(nb
->rcvmtu_allowed_countdown
!= 0)) {
843 nb
->rcvmtu_allowed_countdown
--;
845 if (unlikely(nb
->rcvmtu_allowed_countdown
== 0 &&
846 nb
->rcvmtu_delayed_send_needed
!= 0)) {
847 nb
->rcvmtu_allowed_countdown
= 3;
848 nb
->rcvmtu_delayed_send_needed
= 0;
849 call_send_rcvmtu
= 1;
854 spin_unlock_irqrestore(&(nb
->state_lock
), iflags
);
856 if (call_connidreuse
)
857 cor_connid_used_pingsuccess(nb
);
859 if (unlikely(call_send_rcvmtu
))
862 if (unlikely(stalledresume
)) {
863 spin_lock_bh(&(nb
->retrans_conn_lock
));
864 cor_reschedule_conn_retrans_timer(nb
);
865 spin_unlock_bh(&(nb
->retrans_conn_lock
));
867 cor_qos_enqueue(nb
->queue
, &(nb
->rb
), 0, ns_to_ktime(0),
868 QOS_CALLER_NEIGHBOR
, 1);
872 __u32
cor_add_ping_req(struct cor_neighbor
*nb
, unsigned long *last_ping_time
)
874 unsigned long iflags
;
875 struct cor_ping_cookie
*c
;
880 ktime_t now
= ktime_get();
882 spin_lock_irqsave(&(nb
->state_lock
), iflags
);
884 if (nb
->cookie_unsent
!= 0) {
885 c
= cor_find_cookie(nb
, nb
->cookie_unsent
);
889 nb
->cookie_unsent
= 0;
892 c
= cor_find_cookie(nb
, 0);
896 get_random_bytes((char *) &i
, sizeof(i
));
897 i
= (i
% PING_COOKIES_PER_NEIGH
);
898 c
= &(nb
->cookies
[i
]);
899 cor_reset_cookie(nb
, c
);
903 if (unlikely(nb
->lastcookie
== 0))
905 c
->cookie
= nb
->lastcookie
;
906 c
->time_created
= now
;
911 c
->jiffies_sent
= jiffies
;
914 nb
->ping_intransit
++;
916 *last_ping_time
= nb
->last_ping_time
;
917 nb
->last_ping_time
= c
->jiffies_sent
;
919 spin_unlock_irqrestore(&(nb
->state_lock
), iflags
);
926 void cor_ping_sent(struct cor_neighbor
*nb
, __u32 cookie
)
928 unsigned long iflags
;
932 spin_lock_irqsave(&(nb
->state_lock
), iflags
);
934 if (nb
->cookie_unsent
== cookie
)
935 nb
->cookie_unsent
= 0;
937 spin_unlock_irqrestore(&(nb
->state_lock
), iflags
);
940 void cor_unadd_ping_req(struct cor_neighbor
*nb
, __u32 cookie
,
941 unsigned long last_ping_time
, int congested
)
943 unsigned long iflags
;
945 struct cor_ping_cookie
*c
;
949 spin_lock_irqsave(&(nb
->state_lock
), iflags
);
952 BUG_ON(nb
->cookie_unsent
!= 0 && nb
->cookie_unsent
!= cookie
);
953 nb
->cookie_unsent
= cookie
;
956 c
= cor_find_cookie(nb
, cookie
);
957 if (likely(c
!= 0)) {
960 nb
->ping_intransit
--;
963 nb
->last_ping_time
= last_ping_time
;
965 spin_unlock_irqrestore(&(nb
->state_lock
), iflags
);
968 static int cor_get_ping_forcetime_ms(struct cor_neighbor
*nb
)
970 unsigned long iflags
;
974 if (unlikely(cor_get_neigh_state(nb
) != NEIGHBOR_STATE_ACTIVE
))
975 return PING_FORCETIME_MS
;
977 spin_lock_irqsave(&(nb
->state_lock
), iflags
);
978 fast
= ((nb
->ping_success
< PING_ACTIVE_FASTINITIAL_COUNT
) ||
979 (nb
->ping_intransit
> 0));
980 if (unlikely(nb
->rcvmtu_delayed_send_needed
!= 0)) {
981 BUG_ON(nb
->rcvmtu_allowed_countdown
== 0);
984 spin_unlock_irqrestore(&(nb
->state_lock
), iflags
);
987 return PING_FORCETIME_ACTIVE_FAST_MS
;
989 spin_lock_irqsave(&(nb
->conn_list_lock
), iflags
);
990 idle
= list_empty(&(nb
->snd_conn_idle_list
)) &&
991 list_empty(&(nb
->snd_conn_busy_list
));
992 spin_unlock_irqrestore(&(nb
->conn_list_lock
), iflags
);
995 return PING_FORCETIME_ACTIVEIDLE_MS
;
997 return PING_FORCETIME_ACTIVE_MS
;
1000 static __u32
cor_get_ping_mindelay_ms(struct cor_neighbor
*nb_statelocked
)
1002 __u32 latency_us
= ((__u32
) atomic_read(
1003 &(nb_statelocked
->latency_advertised_us
)));
1004 __u32 max_remote_pong_delay_us
= ((__u32
) atomic_read(
1005 &(nb_statelocked
->max_remote_pong_delay_us
)));
1008 if (latency_us
< PING_GUESSLATENCY_MS
* 1000)
1009 latency_us
= PING_GUESSLATENCY_MS
* 1000;
1011 if (unlikely(nb_statelocked
->state
!= NEIGHBOR_STATE_ACTIVE
))
1012 mindelay_ms
= latency_us
/1000;
1014 mindelay_ms
= ((latency_us
/2 +
1015 max_remote_pong_delay_us
/2)/500);
1017 if (likely(nb_statelocked
->ping_intransit
< PING_COOKIES_THROTTLESTART
))
1020 mindelay_ms
= mindelay_ms
* (1 + 9 * (nb_statelocked
->ping_intransit
*
1021 nb_statelocked
->ping_intransit
/
1022 (PING_COOKIES_PER_NEIGH
* PING_COOKIES_PER_NEIGH
)));
1028 * Check whether we want to send a ping now:
1029 * 0... Do not send ping.
1030 * 1... Send ping now, but only if it can be merged with other messages. This
1031 * can happen way before the time requested by cor_get_next_ping_time().
1032 * 2... Send ping now, even if a packet has to be created just for the ping
1035 int cor_time_to_send_ping(struct cor_neighbor
*nb
)
1037 unsigned long iflags
;
1038 int rc
= TIMETOSENDPING_YES
;
1040 __u32 ms_since_last_ping
;
1042 __u32 forcetime
= cor_get_ping_forcetime_ms(nb
);
1045 spin_lock_irqsave(&(nb
->state_lock
), iflags
);
1047 ms_since_last_ping
= jiffies_to_msecs(jiffies
- nb
->last_ping_time
);
1049 mindelay
= cor_get_ping_mindelay_ms(nb
);
1051 if (forcetime
< (mindelay
* 3))
1052 forcetime
= mindelay
* 3;
1053 else if (forcetime
> (mindelay
* 3))
1054 mindelay
= forcetime
/3;
1056 if (ms_since_last_ping
< mindelay
|| ms_since_last_ping
< (forcetime
/4))
1057 rc
= TIMETOSENDPING_NO
;
1058 else if (ms_since_last_ping
>= forcetime
)
1059 rc
= TIMETOSENDPING_FORCE
;
1061 spin_unlock_irqrestore(&(nb
->state_lock
), iflags
);
1066 unsigned long cor_get_next_ping_time(struct cor_neighbor
*nb
)
1068 unsigned long iflags
;
1070 __u32 forcetime
= cor_get_ping_forcetime_ms(nb
);
1073 spin_lock_irqsave(&(nb
->state_lock
), iflags
);
1074 mindelay
= cor_get_ping_mindelay_ms(nb
);
1075 spin_unlock_irqrestore(&(nb
->state_lock
), iflags
);
1077 if (forcetime
< (mindelay
* 3))
1078 forcetime
= mindelay
* 3;
1080 return nb
->last_ping_time
+ msecs_to_jiffies(forcetime
);
1083 void cor_add_neighbor(struct cor_neighbor_discdata
*nb_dd
)
1085 struct cor_neighbor
*nb
;
1086 struct list_head
*currlh
;
1088 nb
= cor_alloc_neighbor(GFP_KERNEL
);
1089 if (unlikely(nb
== 0))
1092 nb
->queue
= cor_get_queue(nb_dd
->dev
);
1093 if (nb
->queue
== 0) {
1094 kmem_cache_free(cor_nb_slab
, nb
);
1095 atomic_dec(&cor_num_neighs
);
1099 dev_hold(nb_dd
->dev
);
1100 nb
->dev
= nb_dd
->dev
;
1102 memcpy(nb
->mac
, nb_dd
->mac
, MAX_ADDR_LEN
);
1104 nb
->addr
= nb_dd
->addr
;
1105 nb
->addrlen
= nb_dd
->addrlen
;
1107 nb_dd
->nb_allocated
= 1;
1111 spin_lock_bh(&cor_neighbor_list_lock
);
1113 BUG_ON((nb
->addr
== 0) != (nb
->addrlen
== 0));
1115 if (cor_is_clientmode() && (nb
->addr
== 0|| nb
->addrlen
== 0))
1116 goto already_present
;
1118 currlh
= cor_nb_list
.next
;
1119 while (currlh
!= &cor_nb_list
) {
1120 struct cor_neighbor
*curr
= container_of(currlh
,
1121 struct cor_neighbor
, nb_list
);
1123 BUG_ON((curr
->addr
== 0) != (curr
->addrlen
== 0));
1125 if (curr
->dev
== nb
->dev
&&
1126 memcmp(curr
->mac
, nb
->mac
, MAX_ADDR_LEN
) == 0)
1127 goto already_present
;
1129 if (curr
->addr
!= 0 && curr
->addrlen
!= 0 &&
1130 nb
->addr
!= 0 && nb
->addrlen
!= 0 &&
1131 curr
->addrlen
== nb
->addrlen
&&
1132 memcmp(curr
->addr
, nb
->addr
, curr
->addrlen
) ==0)
1133 goto already_present
;
1135 currlh
= currlh
->next
;
1138 /* printk(KERN_ERR "add_neigh\n"); */
1140 spin_lock_bh(&cor_local_addr_lock
);
1141 nb
->sessionid
= cor_local_addr_sessionid
^ nb_dd
->sessionid
;
1142 spin_unlock_bh(&cor_local_addr_lock
);
1144 timer_setup(&(nb
->retrans_timer
), cor_retransmit_timerfunc
, 0);
1146 timer_setup(&(nb
->retrans_conn_timer
), cor_retransmit_conn_timerfunc
, 0);
1148 spin_lock_bh(&(nb
->cmsg_lock
));
1149 nb
->last_ping_time
= jiffies
;
1150 cor_schedule_controlmsg_timer(nb
);
1151 spin_unlock_bh(&(nb
->cmsg_lock
));
1153 list_add_tail(&(nb
->nb_list
), &cor_nb_list
);
1155 cor_nb_kref_get(nb
, "neigh_list");
1156 cor_nb_kref_put_bug(nb
, "alloc");
1160 kmem_cache_free(cor_nb_slab
, nb
);
1161 atomic_dec(&cor_num_neighs
);
1164 spin_unlock_bh(&cor_neighbor_list_lock
);
1167 struct cor_conn
*cor_get_conn(struct cor_neighbor
*nb
, __u32 conn_id
)
1169 unsigned long iflags
;
1171 struct rb_node
* n
= 0;
1172 struct cor_conn
*ret
= 0;
1174 spin_lock_irqsave(&(nb
->connid_lock
), iflags
);
1176 n
= nb
->connid_rb
.rb_node
;
1178 while (likely(n
!= 0) && ret
== 0) {
1179 struct cor_conn
*src_in_o
= container_of(n
, struct cor_conn
,
1182 BUG_ON(src_in_o
->sourcetype
!= SOURCE_IN
);
1184 if (conn_id
< src_in_o
->src
.in
.conn_id
)
1186 else if (conn_id
> src_in_o
->src
.in
.conn_id
)
1193 cor_conn_kref_get(ret
, "stack");
1195 spin_unlock_irqrestore(&(nb
->connid_lock
), iflags
);
1200 int cor_insert_connid(struct cor_neighbor
*nb
, struct cor_conn
*src_in_ll
)
1204 unsigned long iflags
;
1206 __u32 conn_id
= src_in_ll
->src
.in
.conn_id
;
1208 struct rb_root
*root
;
1210 struct rb_node
*parent
= 0;
1212 BUG_ON(src_in_ll
->sourcetype
!= SOURCE_IN
);
1214 spin_lock_irqsave(&(nb
->connid_lock
), iflags
);
1216 root
= &(nb
->connid_rb
);
1217 p
= &(root
->rb_node
);
1220 struct cor_conn
*src_in_o
= container_of(*p
, struct cor_conn
,
1223 BUG_ON(src_in_o
->sourcetype
!= SOURCE_IN
);
1226 if (unlikely(conn_id
== src_in_o
->src
.in
.conn_id
)) {
1228 } else if (conn_id
< src_in_o
->src
.in
.conn_id
) {
1230 } else if (conn_id
> src_in_o
->src
.in
.conn_id
) {
1231 p
= &(*p
)->rb_right
;
1237 cor_conn_kref_get(src_in_ll
, "connid table");
1238 rb_link_node(&(src_in_ll
->src
.in
.rbn
), parent
, p
);
1239 rb_insert_color(&(src_in_ll
->src
.in
.rbn
), root
);
1246 spin_unlock_irqrestore(&(nb
->connid_lock
), iflags
);
1251 static struct cor_connid_reuse_item
*cor_get_connid_reuseitem(
1252 struct cor_neighbor
*nb
, __u32 conn_id
)
1254 unsigned long iflags
;
1256 struct rb_node
*n
= 0;
1257 struct cor_connid_reuse_item
*ret
= 0;
1259 spin_lock_irqsave(&(nb
->connid_reuse_lock
), iflags
);
1261 n
= nb
->connid_reuse_rb
.rb_node
;
1263 while (likely(n
!= 0) && ret
== 0) {
1264 struct cor_connid_reuse_item
*cir
= container_of(n
,
1265 struct cor_connid_reuse_item
, rbn
);
1267 BUG_ON(cir
->conn_id
== 0);
1269 if (conn_id
< cir
->conn_id
)
1271 else if (conn_id
> cir
->conn_id
)
1278 kref_get(&(ret
->ref
));
1280 spin_unlock_irqrestore(&(nb
->connid_reuse_lock
), iflags
);
1285 /* nb->connid_reuse_lock must be held by the caller */
1286 static void _cor_insert_connid_reuse_insertrb(struct cor_neighbor
*nb
,
1287 struct cor_connid_reuse_item
*ins
)
1289 struct rb_root
*root
;
1291 struct rb_node
*parent
= 0;
1293 BUG_ON(ins
->conn_id
== 0);
1295 root
= &(nb
->connid_reuse_rb
);
1296 p
= &(root
->rb_node
);
1299 struct cor_connid_reuse_item
*curr
= container_of(*p
,
1300 struct cor_connid_reuse_item
, rbn
);
1302 BUG_ON(curr
->conn_id
== 0);
1305 if (unlikely(ins
->conn_id
== curr
->conn_id
)) {
1307 } else if (ins
->conn_id
< curr
->conn_id
) {
1309 } else if (ins
->conn_id
> curr
->conn_id
) {
1310 p
= &(*p
)->rb_right
;
1316 kref_get(&(ins
->ref
));
1317 rb_link_node(&(ins
->rbn
), parent
, p
);
1318 rb_insert_color(&(ins
->rbn
), root
);
1321 void cor_insert_connid_reuse(struct cor_neighbor
*nb
, __u32 conn_id
)
1323 unsigned long iflags
;
1325 struct cor_connid_reuse_item
*cir
= kmem_cache_alloc(
1326 cor_connid_reuse_slab
, GFP_ATOMIC
);
1328 if (unlikely(cir
== 0)) {
1329 BUILD_BUG_ON(CONNID_REUSE_RTTS
> 255);
1331 spin_lock_irqsave(&(nb
->connid_reuse_lock
), iflags
);
1332 nb
->connid_reuse_oom_countdown
= CONNID_REUSE_RTTS
;
1333 spin_unlock_irqrestore(&(nb
->connid_reuse_lock
), iflags
);
1338 memset(cir
, 0, sizeof(struct cor_connid_reuse_item
));
1340 kref_init(&(cir
->ref
));
1341 cir
->conn_id
= conn_id
;
1343 spin_lock_irqsave(&(nb
->connid_reuse_lock
), iflags
);
1345 cir
->pingcnt
= nb
->connid_reuse_pingcnt
;
1347 _cor_insert_connid_reuse_insertrb(nb
, cir
);
1348 list_add_tail(&(cir
->lh
), &(nb
->connid_reuse_list
));
1350 spin_unlock_irqrestore(&(nb
->connid_reuse_lock
), iflags
);
1353 static void cor_free_connid_reuse(struct kref
*ref
)
1355 struct cor_connid_reuse_item
*cir
= container_of(ref
,
1356 struct cor_connid_reuse_item
, ref
);
1358 kmem_cache_free(cor_connid_reuse_slab
, cir
);
1361 static void cor_delete_connid_reuse_items(struct cor_neighbor
*nb
)
1363 unsigned long iflags
;
1364 struct cor_connid_reuse_item
*cri
;
1366 spin_lock_irqsave(&(nb
->connid_reuse_lock
), iflags
);
1368 while (list_empty(&(nb
->connid_reuse_list
)) == 0) {
1369 cri
= container_of(nb
->connid_reuse_list
.next
,
1370 struct cor_connid_reuse_item
, lh
);
1372 rb_erase(&(cri
->rbn
), &(nb
->connid_reuse_rb
));
1373 kref_put(&(cri
->ref
), cor_kreffree_bug
);
1375 list_del(&(cri
->lh
));
1376 kref_put(&(cri
->ref
), cor_free_connid_reuse
);
1379 spin_unlock_irqrestore(&(nb
->connid_reuse_lock
), iflags
);
1382 static void cor_connid_used_pingsuccess(struct cor_neighbor
*nb
)
1384 unsigned long iflags
;
1385 struct cor_connid_reuse_item
*cri
;
1387 spin_lock_irqsave(&(nb
->connid_reuse_lock
), iflags
);
1389 nb
->connid_reuse_pingcnt
++;
1390 while (list_empty(&(nb
->connid_reuse_list
)) == 0) {
1391 cri
= container_of(nb
->connid_reuse_list
.next
,
1392 struct cor_connid_reuse_item
, lh
);
1393 if ((cri
->pingcnt
+ CONNID_REUSE_RTTS
-
1394 nb
->connid_reuse_pingcnt
) < 32768)
1397 rb_erase(&(cri
->rbn
), &(nb
->connid_reuse_rb
));
1398 kref_put(&(cri
->ref
), cor_kreffree_bug
);
1400 list_del(&(cri
->lh
));
1401 kref_put(&(cri
->ref
), cor_free_connid_reuse
);
1404 if (unlikely(nb
->connid_reuse_oom_countdown
!= 0))
1405 nb
->connid_reuse_oom_countdown
--;
1408 spin_unlock_irqrestore(&(nb
->connid_reuse_lock
), iflags
);
1411 static int cor_connid_used(struct cor_neighbor
*nb
, __u32 conn_id
)
1413 struct cor_conn
*cn
;
1414 struct cor_connid_reuse_item
*cir
;
1416 cn
= cor_get_conn(nb
, conn_id
);
1417 if (unlikely(cn
!= 0)) {
1418 cor_conn_kref_put(cn
, "stack");
1422 cir
= cor_get_connid_reuseitem(nb
, conn_id
);
1423 if (unlikely(cir
!= 0)) {
1424 kref_put(&(cir
->ref
), cor_free_connid_reuse
);
1431 int cor_connid_alloc(struct cor_neighbor
*nb
, struct cor_conn
*src_in_ll
)
1433 unsigned long iflags
;
1434 struct cor_conn
*trgt_out_ll
= cor_get_conn_reversedir(src_in_ll
);
1438 BUG_ON(src_in_ll
->sourcetype
!= SOURCE_IN
);
1439 BUG_ON(trgt_out_ll
->targettype
!= TARGET_OUT
);
1441 spin_lock_irqsave(&cor_connid_gen
, iflags
);
1442 for (i
=0;i
<16;i
++) {
1444 get_random_bytes((char *) &conn_id
, sizeof(conn_id
));
1445 conn_id
= (conn_id
& ~(1 << 31));
1447 if (unlikely(conn_id
== 0))
1450 if (unlikely(cor_connid_used(nb
, conn_id
)))
1455 spin_unlock_irqrestore(&cor_connid_gen
, iflags
);
1460 spin_lock_irqsave(&(nb
->connid_reuse_lock
), iflags
);
1461 if (unlikely(nb
->connid_reuse_oom_countdown
!= 0)) {
1462 spin_unlock_irqrestore(&(nb
->connid_reuse_lock
), iflags
);
1465 spin_unlock_irqrestore(&(nb
->connid_reuse_lock
), iflags
);
1468 src_in_ll
->src
.in
.conn_id
= conn_id
;
1469 trgt_out_ll
->trgt
.out
.conn_id
= cor_get_connid_reverse(conn_id
);
1470 if (unlikely(cor_insert_connid(nb
, src_in_ll
) != 0)) {
1473 spin_unlock_irqrestore(&cor_connid_gen
, iflags
);
1477 int __init
cor_neighbor_init(void)
1479 cor_nb_slab
= kmem_cache_create("cor_neighbor",
1480 sizeof(struct cor_neighbor
), 8, 0, 0);
1481 if (unlikely(cor_nb_slab
== 0))
1484 atomic_set(&cor_num_neighs
, 0);
1489 void __exit
cor_neighbor_exit2(void)
1491 BUG_ON(atomic_read(&cor_num_neighs
) != 0);
1493 kmem_cache_destroy(cor_nb_slab
);
1497 MODULE_LICENSE("GPL");