?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
do not preallocate cor_connid_reuse_item
[cor.git] / net / cor / neigh_snd.c
blob83d69b41839732d3768a3758ec2e01a9db3304dd
1 /**
2 * Connection oriented routing
3 * Copyright (C) 2007-2021 Michael Blizek
4 *
5 * This program is free software; you can redistribute it and/or
6 * modify it under the terms of the GNU General Public License
7 * as published by the Free Software Foundation; either version 2
8 * of the License, or (at your option) any later version.
9 *
10 * This program is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
14 *
15 * You should have received a copy of the GNU General Public License
16 * along with this program; if not, write to the Free Software
17 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
18 * 02110-1301, USA.
19 */
20
21 #include <asm/byteorder.h>
22
23 #include "cor.h"
24
25 /* not sent over the network - internal meaning only */
26 #define MSGTYPE_PONG 1
27 #define MSGTYPE_ACK 2
28 #define MSGTYPE_ACK_CONN 3
29 #define MSGTYPE_CONNECT 4
30 #define MSGTYPE_CONNECT_SUCCESS 5
31 #define MSGTYPE_RESET_CONN 6
32 #define MSGTYPE_CONNDATA 7
33 #define MSGTYPE_SET_MAX_CMSG_DELAY 8
34 #define MSGTYPE_SET_RCVMTU 9
35
36 #define MSGTYPE_PONG_TIMEENQUEUED 1
37 #define MSGTYPE_PONG_RESPDELAY 2
38
39 struct cor_control_msg_out{
40 __u8 type;
41 __u32 length;
42 struct kref ref;
43 struct cor_neighbor *nb;
44
45 /* either queue or control_retrans_packet */
46 struct list_head lh;
47
48 unsigned long time_added;
49
50 union{
51 struct{
52 __u32 cookie;
53 __u8 type;
54
55 ktime_t ping_rcvtime;
56 ktime_t time_enqueued;
57 }pong;
58
59 struct{
60 __u64 seqno;
61 __u8 fast;
62 }ack;
63
64 struct{
65 struct cor_conn *src_in;
66 struct list_head conn_acks;
67 __u32 conn_id;
68 __u64 seqno;
69 __u64 seqno_ooo;
70 __u32 length;
71
72 __u8 flags;
73
74 __u8 bufsize_changerate;
75
76 __u16 priority;
77 __u8 priority_seqno;
78
79 __u8 is_highlatency;
80 __u8 queue;
81
82 __u32 ack_seqno;
83 }ack_conn;
84
85 struct{
86 __u32 conn_id;
87 __u64 seqno1;
88 __u64 seqno2;
89 struct cor_conn *src_in;
90 }connect;
91
92 struct{
93 __u32 conn_id;
94 struct cor_conn *src_in;
95 }connect_success;
96
97 struct{
98 struct rb_node rbn;
99 __u8 in_pending_conn_resets;
100 __u32 conn_id;
101 }reset_conn;
102
103 struct{
104 __u32 conn_id;
105 __u64 seqno;
106 __u32 datalen;
107 __u8 windowused;
108 __u8 flush;
109 __u8 highlatency;
110 char *data_orig;
111 char *data;
112 struct cor_conn_retrans *cr;
113 }conn_data;
114
115 struct{
116 __u32 ack_fast_delay;
117 __u32 ack_slow_delay;
118 __u32 ackconn_lowlatency_delay;
119 __u32 ackconn_highlatency_delay;
120 __u32 pong_delay;
121 }set_max_cmsg_delay;
122
123 struct{
124 __u32 rcvmtu;
125 }set_rcvmtu;
126 }msg;
127 };
128
129 struct cor_control_retrans {
130 struct kref ref;
131
132 struct cor_neighbor *nb;
133 __u64 seqno;
134
135 unsigned long timeout;
136
137 struct list_head msgs;
138
139 struct rb_node rbn;
140 struct list_head timeout_list;
141 };
142
143
144 static struct kmem_cache *cor_controlmsg_slab;
145 static struct kmem_cache *cor_controlretrans_slab;
146
147 static atomic_t cor_cmsg_othercnt = ATOMIC_INIT(0);
148
149 #define ADDCMSG_SRC_NEW 1
150 #define ADDCMSG_SRC_SPLITCONNDATA 2
151 #define ADDCMSG_SRC_READD 3
152 #define ADDCMSG_SRC_RETRANS 4
153
154 static void cor_enqueue_control_msg(struct cor_control_msg_out *msg, int src);
155
156 static void cor_try_merge_ackconns(struct cor_conn *src_in_l,
157 struct cor_control_msg_out *cm);
158
159 static void cor_merge_or_enqueue_ackconn(struct cor_conn *src_in_l,
160 struct cor_control_msg_out *cm, int src);
161
162 static struct cor_control_msg_out *_cor_alloc_control_msg(
163 struct cor_neighbor *nb)
164 {
165 struct cor_control_msg_out *cm;
166
167 BUG_ON(nb == 0);
168
169 cm = kmem_cache_alloc(cor_controlmsg_slab, GFP_ATOMIC);
170 if (unlikely(cm == 0))
171 return 0;
172 memset(cm, 0, sizeof(struct cor_control_msg_out));
173 kref_init(&(cm->ref));
174 cm->nb = nb;
175 return cm;
176 }
177
178 static int cor_calc_limit(int limit, int priority)
179 {
180 if (priority == ACM_PRIORITY_LOW)
181 return (limit+1)/2;
182 else if (priority == ACM_PRIORITY_MED)
183 return (limit * 3 + 1)/4;
184 else if (priority == ACM_PRIORITY_HIGH)
185 return limit;
186 else
187 BUG();
188 }
189
190 struct cor_control_msg_out *cor_alloc_control_msg(struct cor_neighbor *nb,
191 int priority)
192 {
193 struct cor_control_msg_out *cm = 0;
194
195 long packets1;
196 long packets2;
197
198 BUG_ON(nb == 0);
199
200 packets1 = atomic_inc_return(&(nb->cmsg_othercnt));
201 packets2 = atomic_inc_return(&(cor_cmsg_othercnt));
202
203 BUG_ON(packets1 <= 0);
204 BUG_ON(packets2 <= 0);
205
206 if (packets1 <= cor_calc_limit(GUARANTEED_CMSGS_PER_NEIGH, priority))
207 goto alloc;
208
209 if (unlikely(unlikely(packets1 > cor_calc_limit(MAX_CMSGS_PER_NEIGH,
210 priority)) ||
211 unlikely(packets2 > cor_calc_limit(MAX_CMSGS,
212 priority))))
213 goto full;
214
215 alloc:
216 cm = _cor_alloc_control_msg(nb);
217 if (unlikely(cm == 0)) {
218 full:
219
220 /* printk(KERN_ERR "cor_alloc_control_msg failed %ld %ld\n",
221 packets1, packets2); */
222 atomic_dec(&(nb->cmsg_othercnt));
223 atomic_dec(&(cor_cmsg_othercnt));
224 }
225 return cm;
226 }
227
228 static void cor_cmsg_kref_free(struct kref *ref)
229 {
230 struct cor_control_msg_out *cm = container_of(ref,
231 struct cor_control_msg_out, ref);
232 kmem_cache_free(cor_controlmsg_slab, cm);
233 }
234
235 void cor_free_control_msg(struct cor_control_msg_out *cm)
236 {
237 if (likely(cm->type != MSGTYPE_PONG)) {
238 atomic_dec(&(cm->nb->cmsg_othercnt));
239 atomic_dec(&(cor_cmsg_othercnt));
240 }
241
242 if (cm->type == MSGTYPE_ACK_CONN) {
243 BUG_ON(cm->msg.ack_conn.src_in == 0);
244 if ((cm->msg.ack_conn.flags & KP_ACK_CONN_FLAGS_PRIORITY) != 0){
245 struct cor_conn *trgt_out = cor_get_conn_reversedir(
246 cm->msg.ack_conn.src_in);
247 spin_lock_bh(&(trgt_out->rcv_lock));
248 BUG_ON(trgt_out->targettype != TARGET_OUT);
249 if (trgt_out->trgt.out.priority_send_allowed != 0) {
250 trgt_out->trgt.out.priority_send_allowed = 1;
251 spin_unlock_bh(&(trgt_out->rcv_lock));
252 cor_conn_refresh_priority(trgt_out, 0);
253 } else {
254 spin_unlock_bh(&(trgt_out->rcv_lock));
255 }
256 }
257 cor_conn_kref_put(cm->msg.ack_conn.src_in,
258 "cor_control_msg_out ack_conn");
259 cm->msg.ack_conn.src_in = 0;
260 } else if (cm->type == MSGTYPE_CONNECT) {
261 BUG_ON(cm->msg.connect.src_in == 0);
262 cor_conn_kref_put(cm->msg.connect.src_in,
263 "cor_control_msg_out connect");
264 cm->msg.connect.src_in = 0;
265 } else if (cm->type == MSGTYPE_CONNECT_SUCCESS) {
266 BUG_ON(cm->msg.connect_success.src_in == 0);
267 cor_conn_kref_put(cm->msg.connect_success.src_in,
268 "cor_control_msg_out connect_success");
269 cm->msg.connect_success.src_in = 0;
270 } else if (cm->type == MSGTYPE_RESET_CONN) {
271 spin_lock_bh(&(cm->nb->cmsg_lock));
272 if (cm->msg.reset_conn.in_pending_conn_resets != 0) {
273 rb_erase(&(cm->msg.reset_conn.rbn),
274 &(cm->nb->pending_conn_resets_rb));
275 cm->msg.reset_conn.in_pending_conn_resets = 0;
276
277 kref_put(&(cm->ref), cor_kreffree_bug);
278 }
279 spin_unlock_bh(&(cm->nb->cmsg_lock));
280 }
281
282 kref_put(&(cm->ref), cor_cmsg_kref_free);
283 }
284
285 static void cor_free_control_retrans(struct kref *ref)
286 {
287 struct cor_control_retrans *cr = container_of(ref,
288 struct cor_control_retrans, ref);
289
290 while (list_empty(&(cr->msgs)) == 0) {
291 struct cor_control_msg_out *cm = container_of(cr->msgs.next,
292 struct cor_control_msg_out, lh);
293
294 if (cm->type == MSGTYPE_PONG)
295 atomic_dec(&(cm->nb->cmsg_pongs_retrans_cnt));
296
297 list_del(&(cm->lh));
298 cor_free_control_msg(cm);
299 }
300
301 kmem_cache_free(cor_controlretrans_slab, cr);
302 }
303
304 struct cor_control_retrans *cor_get_control_retrans(
305 struct cor_neighbor *nb_retranslocked, __u64 seqno)
306 {
307 struct rb_node *n = 0;
308 struct cor_control_retrans *ret = 0;
309
310 n = nb_retranslocked->kp_retransmits_rb.rb_node;
311
312 while (likely(n != 0) && ret == 0) {
313 struct cor_control_retrans *cr = container_of(n,
314 struct cor_control_retrans, rbn);
315
316 BUG_ON(cr->nb != nb_retranslocked);
317
318 if (cor_seqno_before(seqno, cr->seqno))
319 n = n->rb_left;
320 else if (cor_seqno_after(seqno, cr->seqno))
321 n = n->rb_right;
322 else
323 ret = cr;
324 }
325
326 if (ret != 0)
327 kref_get(&(ret->ref));
328
329 return ret;
330 }
331
332 /* nb->retrans_lock must be held */
333 void cor_insert_control_retrans(struct cor_control_retrans *ins)
334 {
335 struct cor_neighbor *nb = ins->nb;
336 __u64 seqno = ins->seqno;
337
338 struct rb_root *root;
339 struct rb_node **p;
340 struct rb_node *parent = 0;
341
342 BUG_ON(nb == 0);
343
344 root = &(nb->kp_retransmits_rb);
345 p = &(root->rb_node);
346
347 while ((*p) != 0) {
348 struct cor_control_retrans *cr = container_of(*p,
349 struct cor_control_retrans, rbn);
350
351 BUG_ON(cr->nb != nb);
352
353 parent = *p;
354 if (unlikely(cor_seqno_eq(seqno, cr->seqno))) {
355 BUG();
356 } else if (cor_seqno_before(seqno, cr->seqno)) {
357 p = &(*p)->rb_left;
358 } else if (cor_seqno_after(seqno, cr->seqno)) {
359 p = &(*p)->rb_right;
360 } else {
361 BUG();
362 }
363 }
364
365 kref_get(&(ins->ref));
366 rb_link_node(&(ins->rbn), parent, p);
367 rb_insert_color(&(ins->rbn), root);
368 }
369
370 static void cor_remove_connack_oooflag_ifold(struct cor_conn *src_in_l,
371 struct cor_control_msg_out *cm)
372 {
373 if (cor_ooolen(cm->msg.ack_conn.flags) != 0 && cor_seqno_before_eq(
374 cm->msg.ack_conn.seqno_ooo +
375 cm->msg.ack_conn.length,
376 src_in_l->src.in.next_seqno)) {
377 cm->msg.ack_conn.length = 0;
378 cm->msg.ack_conn.flags = (cm->msg.ack_conn.flags &
379 (~KP_ACK_CONN_FLAGS_OOO));
380 }
381 }
382
383 static int cor_ackconn_prepare_requeue(struct cor_conn *cn_l,
384 struct cor_control_msg_out *cm)
385 {
386 if (unlikely(unlikely(cn_l->sourcetype != SOURCE_IN) ||
387 unlikely(cn_l->src.in.nb != cm->nb) ||
388 unlikely(
389 cor_get_connid_reverse(cn_l->src.in.conn_id) !=
390 cm->msg.ack_conn.conn_id) ||
391 unlikely(cn_l->isreset != 0)))
392 return 0;
393
394 cor_remove_connack_oooflag_ifold(cn_l, cm);
395
396 if (!cor_seqno_eq(cm->msg.ack_conn.ack_seqno, cn_l->src.in.ack_seqno))
397 cm->msg.ack_conn.flags = (cm->msg.ack_conn.flags &
398 (~KP_ACK_CONN_FLAGS_SEQNO) &
399 (~KP_ACK_CONN_FLAGS_WINDOW));
400
401 if (cm->msg.ack_conn.flags == 0)
402 return 0;
403
404 cm->length = 5 + cor_ack_conn_len(cm->msg.ack_conn.flags);
405
406 return 1;
407 }
408
409 static void cor_requeue_control_retrans(struct cor_control_retrans *cr)
410 {
411 atomic_inc(&(cr->nb->cmsg_bulk_readds));
412
413 while (list_empty(&(cr->msgs)) == 0) {
414 struct cor_control_msg_out *cm = container_of(cr->msgs.prev,
415 struct cor_control_msg_out, lh);
416 list_del(&(cm->lh));
417
418 BUG_ON(cm->nb != cr->nb);
419
420 if (cm->type == MSGTYPE_ACK_CONN) {
421 struct cor_conn *cn_l = cm->msg.ack_conn.src_in;
422 spin_lock_bh(&(cn_l->rcv_lock));
423 if (unlikely(cor_ackconn_prepare_requeue(cn_l,
424 cm) == 0)) {
425 cor_free_control_msg(cm);
426 } else {
427 cor_merge_or_enqueue_ackconn(cn_l, cm,
428 ADDCMSG_SRC_RETRANS);
429 }
430
431 spin_unlock_bh(&(cn_l->rcv_lock));
432 } else {
433 if (cm->type == MSGTYPE_PONG)
434 atomic_dec(&(cm->nb->cmsg_pongs_retrans_cnt));
435 cor_enqueue_control_msg(cm, ADDCMSG_SRC_RETRANS);
436 }
437 }
438
439 atomic_dec(&(cr->nb->cmsg_bulk_readds));
440
441 spin_lock_bh(&(cr->nb->cmsg_lock));
442 cor_schedule_controlmsg_timer(cr->nb);
443 spin_unlock_bh(&(cr->nb->cmsg_lock));
444 }
445
446 static void _cor_empty_retrans_queue(struct cor_neighbor *nb_retranslocked,
447 struct list_head *retrans_list)
448 {
449 while (!list_empty(retrans_list)) {
450 struct cor_control_retrans *cr = container_of(
451 retrans_list->next, struct cor_control_retrans,
452 timeout_list);
453
454 BUG_ON(cr->nb != nb_retranslocked);
455
456 list_del(&(cr->timeout_list));
457 rb_erase(&(cr->rbn), &(nb_retranslocked->kp_retransmits_rb));
458
459 kref_put(&(cr->ref), cor_kreffree_bug); /* rb */
460 kref_put(&(cr->ref), cor_free_control_retrans); /* list */
461 }
462 }
463
464 static void cor_empty_retrans_queue(struct cor_neighbor *nb_retranslocked)
465 {
466 _cor_empty_retrans_queue(nb_retranslocked,
467 &(nb_retranslocked->retrans_fast_list));
468 _cor_empty_retrans_queue(nb_retranslocked,
469 &(nb_retranslocked->retrans_slow_list));
470 }
471
472 static unsigned long cor_get_retransmit_timeout(
473 struct cor_neighbor *nb_retranslocked)
474 {
475 struct cor_control_retrans *cr1 = 0;
476 struct cor_control_retrans *cr2 = 0;
477 struct cor_control_retrans *cr = 0;
478
479 if (list_empty(&(nb_retranslocked->retrans_fast_list)) == 0) {
480 cr1 = container_of(nb_retranslocked->retrans_fast_list.next,
481 struct cor_control_retrans, timeout_list);
482 BUG_ON(cr1->nb != nb_retranslocked);
483 }
484
485 if (list_empty(&(nb_retranslocked->retrans_slow_list)) == 0) {
486 cr2 = container_of(nb_retranslocked->retrans_slow_list.next,
487 struct cor_control_retrans, timeout_list);
488 BUG_ON(cr2->nb != nb_retranslocked);
489 }
490
491 if (cr1 == 0)
492 cr = cr2;
493 else if (cr2 == 0)
494 cr = cr1;
495 else
496 cr = (time_after(cr1->timeout, cr2->timeout) ? cr2 : cr1);
497
498 BUG_ON(cr == 0);
499
500 return cr->timeout;
501 }
502
503 void cor_retransmit_timerfunc(struct timer_list *retrans_timer)
504 {
505 struct cor_neighbor *nb = container_of(retrans_timer,
506 struct cor_neighbor, retrans_timer);
507 int nbstate = cor_get_neigh_state(nb);
508 unsigned long timeout;
509
510 spin_lock_bh(&(nb->retrans_lock));
511
512 if (list_empty(&(nb->retrans_fast_list)) &&
513 list_empty(&(nb->retrans_slow_list))) {
514 spin_unlock_bh(&(nb->retrans_lock));
515 cor_nb_kref_put(nb, "retransmit_timer");
516 return;
517 }
518
519 if (unlikely(nbstate == NEIGHBOR_STATE_KILLED)) {
520 cor_empty_retrans_queue(nb);
521 spin_unlock_bh(&(nb->retrans_lock));
522 cor_nb_kref_put(nb, "retransmit_timer");
523 return;
524 }
525
526 timeout = cor_get_retransmit_timeout(nb);
527
528 if (time_after(timeout, jiffies)) {
529 int rc = mod_timer(&(nb->retrans_timer), timeout);
530 spin_unlock_bh(&(nb->retrans_lock));
531 if (rc != 0)
532 cor_nb_kref_put(nb, "retransmit_timer");
533 return;
534 }
535
536 spin_unlock_bh(&(nb->retrans_lock));
537
538 spin_lock_bh(&(nb->cmsg_lock));
539 nb->add_retrans_needed = 1;
540 cor_schedule_controlmsg_timer(nb);
541 spin_unlock_bh(&(nb->cmsg_lock));
542
543 cor_nb_kref_put(nb, "retransmit_timer");
544 }
545
546 static void cor_schedule_retransmit(struct cor_control_retrans *cr,
547 struct cor_neighbor *nb, int fastack)
548 {
549 int first;
550
551 cr->timeout = cor_calc_timeout(atomic_read(&(nb->latency_retrans_us)),
552 atomic_read(&(nb->latency_stddev_retrans_us)),
553 fastack ?
554 atomic_read(&(nb->max_remote_ack_fast_delay_us)) :
555 atomic_read(&(nb->max_remote_ack_slow_delay_us)));
556
557 spin_lock_bh(&(nb->retrans_lock));
558
559 cor_insert_control_retrans(cr);
560 if (fastack) {
561 first = list_empty(&(nb->retrans_fast_list));
562 list_add_tail(&(cr->timeout_list), &(nb->retrans_fast_list));
563 } else {
564 first = list_empty(&(nb->retrans_slow_list));
565 list_add_tail(&(cr->timeout_list), &(nb->retrans_slow_list));
566 }
567
568 if (first) {
569 if (mod_timer(&(nb->retrans_timer),
570 cor_get_retransmit_timeout(nb)) == 0) {
571 cor_nb_kref_get(nb, "retransmit_timer");
572 }
573 }
574
575 spin_unlock_bh(&(nb->retrans_lock));
576 }
577
578 void cor_kern_ack_rcvd(struct cor_neighbor *nb, __u64 seqno)
579 {
580 struct cor_control_retrans *cr = 0;
581
582 spin_lock_bh(&(nb->retrans_lock));
583
584 cr = cor_get_control_retrans(nb, seqno);
585
586 if (cr == 0) {
587 /* char *seqno_p = (char *) &seqno;
588 seqno = cpu_to_be32(seqno);
589 printk(KERN_ERR "bogus/duplicate ack received %d %d %d %d\n",
590 seqno_p[0], seqno_p[1], seqno_p[2], seqno_p[3]);
591 */
592 goto out;
593 }
594
595 rb_erase(&(cr->rbn), &(nb->kp_retransmits_rb));
596
597 BUG_ON(cr->nb != nb);
598
599 list_del(&(cr->timeout_list));
600
601 out:
602 spin_unlock_bh(&(nb->retrans_lock));
603
604 if (cr != 0) {
605 /* cor_get_control_retrans */
606 kref_put(&(cr->ref), cor_kreffree_bug);
607
608 kref_put(&(cr->ref), cor_kreffree_bug); /* rb_erase */
609 kref_put(&(cr->ref), cor_free_control_retrans); /* list */
610 }
611 }
612
613 static __u16 cor_get_window(struct cor_conn *cn,
614 struct cor_neighbor *expectedsender, __u32 expected_connid)
615 {
616 __u16 window = 0;
617
618 BUG_ON(expectedsender == 0);
619
620 spin_lock_bh(&(cn->rcv_lock));
621
622 if (cor_is_conn_in(cn, expectedsender, expected_connid) == 0)
623 goto out;
624
625 window = cor_enc_window(cor_seqno_clean(
626 cn->src.in.window_seqnolimit -
627 cn->src.in.next_seqno));
628
629 cn->src.in.window_seqnolimit_remote = cn->src.in.next_seqno +
630 cor_dec_window(window);
631
632 out:
633 spin_unlock_bh(&(cn->rcv_lock));
634
635 return window;
636 }
637
638 /* static void padding(struct sk_buff *skb, __u32 length)
639 {
640 char *dst;
641 if (length <= 0)
642 return;
643 dst = skb_put(skb, length);
644 BUG_ON(dst == 0);
645 memset(dst, KP_PADDING, length);
646 } */
647
648
649 static __u32 cor_add_init_session(struct sk_buff *skb, __be32 sessionid,
650 __u32 spaceleft)
651 {
652 char *dst;
653
654 BUG_ON(KP_MISC_INIT_SESSION_CMDLEN != 5);
655
656 if (unlikely(spaceleft < 5))
657 return 0;
658
659 dst = skb_put(skb, 5);
660 BUG_ON(dst == 0);
661
662 dst[0] = get_kp_code(KP_MISC, KP_MISC_INIT_SESSION);
663 cor_put_be32(dst + 1, sessionid);
664
665 return 5;
666 }
667
668 static __u32 cor_add_ack(struct sk_buff *skb, struct cor_control_retrans *cr,
669 struct cor_control_msg_out *cm, __u32 spaceleft)
670 {
671 char *dst;
672
673 BUG_ON(cm->length != 7);
674
675 if (unlikely(spaceleft < 7))
676 return 0;
677
678 dst = skb_put(skb, 7);
679 BUG_ON(dst == 0);
680
681 dst[0] = get_kp_code(KP_MISC, KP_MISC_ACK);
682 cor_put_u48(dst + 1, cm->msg.ack.seqno);
683
684 list_add_tail(&(cm->lh), &(cr->msgs));
685
686 return 7;
687 }
688
689 static inline __u8 cor_add_ack_conn_get_delayremaining(
690 struct cor_control_msg_out *cm, unsigned long cmsg_send_start_j)
691 {
692 __u32 maxdelay_ms = 0;
693 unsigned long jiffies_timeout;
694 if (cm->msg.ack_conn.is_highlatency) {
695 maxdelay_ms = CMSG_MAXDELAY_ACKCONN_HIGHLATENCY_MS;
696 } else {
697 maxdelay_ms = CMSG_MAXDELAY_ACKCONN_LOWLATENCY_MS;
698 }
699
700 jiffies_timeout = cm->time_added + msecs_to_jiffies(maxdelay_ms);
701
702 if (time_before_eq(cmsg_send_start_j, cm->time_added)) {
703 return 255;
704 } else if (time_after_eq(cmsg_send_start_j, jiffies_timeout)) {
705 return 0;
706 } else {
707 __u64 delay_remaining = jiffies_timeout - cmsg_send_start_j;
708
709 BUG_ON(delay_remaining > U32_MAX);
710 BUG_ON(delay_remaining > msecs_to_jiffies(maxdelay_ms));
711
712 return (__u8) div64_u64(255 * delay_remaining +
713 msecs_to_jiffies(maxdelay_ms)/2,
714 msecs_to_jiffies(maxdelay_ms));
715 }
716 }
717
718 static __u32 cor_add_ack_conn(struct sk_buff *skb,
719 struct cor_control_retrans *cr, struct cor_control_msg_out *cm,
720 __u32 spaceleft, unsigned long cmsg_send_start_j,
721 int *ackneeded)
722 {
723 char *dst;
724 __u32 offset = 0;
725
726 if (unlikely(spaceleft < cm->length))
727 return 0;
728
729 dst = skb_put(skb, cm->length);
730 BUG_ON(dst == 0);
731
732 dst[offset] = get_kp_code(KP_ACK_CONN, cm->msg.ack_conn.flags);
733 offset++;
734 cor_put_u32(dst + offset, cm->msg.ack_conn.conn_id);
735 offset += 4;
736
737 if (likely((cm->msg.ack_conn.flags & KP_ACK_CONN_FLAGS_SEQNO) != 0 ||
738 cor_ooolen(cm->msg.ack_conn.flags) != 0)) {
739 dst[offset] = cor_add_ack_conn_get_delayremaining(cm,
740 cmsg_send_start_j);
741 offset++;
742 }
743
744 if ((cm->msg.ack_conn.flags & KP_ACK_CONN_FLAGS_SEQNO) != 0) {
745 cor_put_u48(dst + offset, cm->msg.ack_conn.seqno);
746 offset += 6;
747
748 if ((cm->msg.ack_conn.flags & KP_ACK_CONN_FLAGS_WINDOW) != 0) {
749 BUG_ON(cm->msg.ack_conn.src_in == 0);
750 cor_put_u16(dst + offset, cor_get_window(
751 cm->msg.ack_conn.src_in,
752 cm->nb, cor_get_connid_reverse(
753 cm->msg.ack_conn.conn_id)));
754 offset += 2;
755 dst[offset] = cm->msg.ack_conn.bufsize_changerate;
756 offset++;
757 }
758 }
759
760 if (cor_ooolen(cm->msg.ack_conn.flags) != 0) {
761 cor_put_u48(dst + offset, cm->msg.ack_conn.seqno_ooo);
762 offset += 6;
763 if (cor_ooolen(cm->msg.ack_conn.flags) == 1) {
764 BUG_ON(cm->msg.ack_conn.length > 255);
765 dst[offset] = cm->msg.ack_conn.length;
766 offset += 1;
767 } else if (cor_ooolen(cm->msg.ack_conn.flags) == 2) {
768 BUG_ON(cm->msg.ack_conn.length <= 255);
769 BUG_ON(cm->msg.ack_conn.length > 65535);
770 cor_put_u16(dst + offset, cm->msg.ack_conn.length);
771 offset += 2;
772 } else if (cor_ooolen(cm->msg.ack_conn.flags) == 4) {
773 BUG_ON(cm->msg.ack_conn.length <= 65535);
774 cor_put_u32(dst + offset, cm->msg.ack_conn.length);
775 offset += 4;
776 } else {
777 BUG();
778 }
779 }
780
781 if (unlikely((cm->msg.ack_conn.flags &
782 KP_ACK_CONN_FLAGS_PRIORITY) != 0)) {
783 __u16 priority = (cm->msg.ack_conn.priority_seqno << 12) &
784 cm->msg.ack_conn.priority;
785 BUG_ON(cm->msg.ack_conn.priority_seqno > 15);
786 BUG_ON(cm->msg.ack_conn.priority > 4095);
787
788 cor_put_u16(dst + offset, priority);
789 offset+=2;
790 }
791
792 list_add_tail(&(cm->lh), &(cr->msgs));
793 if (likely((cm->msg.ack_conn.flags & KP_ACK_CONN_FLAGS_SEQNO) != 0 ||
794 cor_ooolen(cm->msg.ack_conn.flags) != 0) &&
795 cm->msg.ack_conn.is_highlatency == 0) {
796 *ackneeded = ACK_NEEDED_FAST;
797 } else if (*ackneeded != ACK_NEEDED_FAST) {
798 *ackneeded = ACK_NEEDED_SLOW;
799 }
800
801 BUG_ON(offset != cm->length);
802 return offset;
803 }
804
805 static __u32 cor_add_ping(struct sk_buff *skb, __u32 cookie, __u32 spaceleft)
806 {
807 char *dst;
808
809 BUG_ON(KP_MISC_PING_CMDLEN != 5);
810
811 if (unlikely(spaceleft < 5))
812 return 0;
813
814 dst = skb_put(skb, 5);
815 BUG_ON(dst == 0);
816
817 dst[0] = get_kp_code(KP_MISC, KP_MISC_PING);
818 cor_put_u32(dst + 1, cookie);
819
820 return 5;
821 }
822
823 static __u32 cor_calc_respdelay(ktime_t time_pong_enqueued, ktime_t time_end)
824 {
825 if (unlikely(ktime_before(time_end, time_pong_enqueued))) {
826 return 0;
827 } else {
828 __s64 respdelay = div_u64(ktime_to_ns(time_end) -
829 ktime_to_ns(time_pong_enqueued) + 500,
830 1000);
831
832 if (unlikely(respdelay > U32_MAX))
833 return U32_MAX;
834 else if (unlikely(respdelay < 0))
835 return 0;
836 else
837 return (__u32) respdelay;
838 }
839 }
840
841 static __u32 cor_add_pong(struct sk_buff *skb, struct cor_control_retrans *cr,
842 struct cor_control_msg_out *cm, __u32 spaceleft, int nbstate,
843 ktime_t cmsg_send_start, int *ackneeded)
844 {
845 __u32 respdelay_full;
846 __u32 respdelay_netonly;
847 char *dst;
848
849 BUG_ON(cm->length != 13);
850
851 if (unlikely(spaceleft < 13))
852 return 0;
853
854 respdelay_full = cor_calc_respdelay(cm->msg.pong.time_enqueued,
855 cmsg_send_start);
856 respdelay_netonly = cor_calc_respdelay(cm->msg.pong.ping_rcvtime,
857 ktime_get());
858
859 dst = skb_put(skb, 13);
860 BUG_ON(dst == 0);
861
862 dst[0] = get_kp_code(KP_MISC, KP_MISC_PONG);
863 cor_put_u32(dst + 1, cm->msg.pong.cookie);
864 cor_put_u32(dst + 5, (__u32) respdelay_full);
865 cor_put_u32(dst + 9, (__u32) respdelay_netonly);
866
867 list_add_tail(&(cm->lh), &(cr->msgs));
868 if (likely(nbstate == NEIGHBOR_STATE_ACTIVE) &&
869 *ackneeded != ACK_NEEDED_FAST)
870 *ackneeded = ACK_NEEDED_SLOW;
871
872 return 13;
873 }
874
875 static __u32 cor_add_connect(struct sk_buff *skb,
876 struct cor_control_retrans *cr, struct cor_control_msg_out *cm,
877 __u32 spaceleft, int *ackneeded)
878 {
879 char *dst;
880 struct cor_conn *src_in = cm->msg.connect.src_in;
881 struct cor_conn *trgt_out = cor_get_conn_reversedir(src_in);
882 __u16 priority;
883
884 BUG_ON(cm->length != 22);
885
886 if (unlikely(spaceleft < 22))
887 return 0;
888
889 dst = skb_put(skb, 22);
890 BUG_ON(dst == 0);
891
892 dst[0] = get_kp_code(KP_MISC, KP_MISC_CONNECT);
893 cor_put_u32(dst + 1, cm->msg.connect.conn_id);
894 cor_put_u48(dst + 5, cm->msg.connect.seqno1);
895 cor_put_u48(dst + 11, cm->msg.connect.seqno2);
896 BUG_ON(cm->msg.connect.src_in == 0);
897 cor_put_u16(dst + 17, cor_get_window(cm->msg.connect.src_in, cm->nb,
898 cor_get_connid_reverse(cm->msg.connect.conn_id)));
899
900 spin_lock_bh(&(trgt_out->rcv_lock));
901 BUG_ON(trgt_out->targettype != TARGET_OUT);
902
903 priority = (trgt_out->trgt.out.priority_seqno << 12) &
904 trgt_out->trgt.out.priority_last;
905 BUG_ON(trgt_out->trgt.out.priority_seqno > 15);
906 BUG_ON(trgt_out->trgt.out.priority_last > 4095);
907 cor_put_u16(dst + 19, priority);
908
909 if (src_in->is_highlatency == 0)
910 dst[21] = 0;
911 else
912 dst[21] = 1;
913
914 spin_unlock_bh(&(trgt_out->rcv_lock));
915
916 list_add_tail(&(cm->lh), &(cr->msgs));
917 if (*ackneeded != ACK_NEEDED_FAST)
918 *ackneeded = ACK_NEEDED_SLOW;
919
920 return 22;
921 }
922
923 static __u32 cor_add_connect_success(struct sk_buff *skb,
924 struct cor_control_retrans *cr, struct cor_control_msg_out *cm,
925 __u32 spaceleft, int *ackneeded)
926 {
927 char *dst;
928
929 BUG_ON(cm->length != 7);
930
931 if (unlikely(spaceleft < 7))
932 return 0;
933
934 dst = skb_put(skb, 7);
935 BUG_ON(dst == 0);
936
937 dst[0] = get_kp_code(KP_MISC, KP_MISC_CONNECT_SUCCESS);
938 cor_put_u32(dst + 1, cm->msg.connect_success.conn_id);
939 BUG_ON(cm->msg.connect_success.src_in == 0);
940 cor_put_u16(dst + 5, cor_get_window(
941 cm->msg.connect_success.src_in, cm->nb,
942 cor_get_connid_reverse(
943 cm->msg.connect_success.conn_id)));
944
945 list_add_tail(&(cm->lh), &(cr->msgs));
946 if (*ackneeded != ACK_NEEDED_FAST)
947 *ackneeded = ACK_NEEDED_SLOW;
948
949 return 7;
950 }
951
952 static __u32 cor_add_reset_conn(struct sk_buff *skb,
953 struct cor_control_retrans *cr, struct cor_control_msg_out *cm,
954 __u32 spaceleft, int *ackneeded)
955 {
956 char *dst;
957
958 BUG_ON(cm->length != 5);
959
960 if (unlikely(spaceleft < 5))
961 return 0;
962
963 dst = skb_put(skb, 5);
964 BUG_ON(dst == 0);
965
966 dst[0] = get_kp_code(KP_MISC, KP_MISC_RESET_CONN);
967 cor_put_u32(dst + 1, cm->msg.reset_conn.conn_id);
968
969 list_add_tail(&(cm->lh), &(cr->msgs));
970 if (*ackneeded != ACK_NEEDED_FAST)
971 *ackneeded = ACK_NEEDED_SLOW;
972
973 return 5;
974 }
975
976 static __u32 cor_add_conndata(struct sk_buff *skb,
977 struct cor_control_retrans *cr, struct cor_control_msg_out *cm,
978 __u32 spaceleft, struct cor_control_msg_out **split_conndata,
979 __u32 *sc_sendlen)
980 {
981 char *dst;
982 __u32 offset = 0;
983
984 __u32 totallen = get_kp_conn_data_length(cm->msg.conn_data.datalen);
985 __u32 putlen = totallen;
986 __u32 dataputlen = cm->msg.conn_data.datalen;
987 __u8 code_min = 0;
988
989 BUILD_BUG_ON(KP_CONN_DATA_MAXLEN != 128+32767);
990 BUG_ON(cm->msg.conn_data.datalen > KP_CONN_DATA_MAXLEN);
991
992 BUG_ON(cm->length != totallen);
993
994 BUG_ON(putlen > 1024*1024*1024);
995
996 BUG_ON(split_conndata == 0);
997 BUG_ON(*split_conndata != 0);
998 BUG_ON(sc_sendlen == 0);
999 BUG_ON(*sc_sendlen != 0);
1000
1001 if (putlen > spaceleft) {
1002 if (spaceleft < get_kp_conn_data_length(1))
1003 return 0;
1004
1005 BUG_ON(spaceleft < 13);
1006
1007 if (spaceleft <= 127 + 12) {
1008 dataputlen = spaceleft - 12;
1009 putlen = spaceleft;
1010 } else if (spaceleft == 127 - 12 + 1) {
1011 dataputlen = spaceleft - 12 - 1;
1012 putlen = spaceleft - 1;
1013 } else {
1014 dataputlen = spaceleft - 13;
1015 putlen = spaceleft;
1016 }
1017
1018 BUG_ON(putlen != get_kp_conn_data_length(dataputlen));
1019 }
1020
1021 dst = skb_put(skb, putlen);
1022 BUG_ON(dst == 0);
1023
1024 BUG_ON((cm->msg.conn_data.windowused &
1025 (~KP_CONN_DATA_FLAGS_WINDOWUSED)) != 0);
1026 code_min = 0;
1027 if (cm->msg.conn_data.flush != 0)
1028 code_min |= KP_CONN_DATA_FLAGS_FLUSH;
1029 code_min |= cm->msg.conn_data.windowused;
1030
1031 dst[0] = get_kp_code(KP_CONN_DATA, code_min);
1032 offset++;
1033 cor_put_u32(dst + offset, cm->msg.conn_data.conn_id);
1034 offset += 4;
1035 cor_put_u48(dst + offset, cm->msg.conn_data.seqno);
1036 offset += 6;
1037
1038 if (dataputlen < 128) {
1039 dst[offset] = (__u8) dataputlen;
1040 offset++;
1041 } else {
1042 __u8 high = (__u8) (128 + ((dataputlen - 128) / 256));
1043 __u8 low = (__u8) ((dataputlen - 128) % 256);
1044 BUG_ON(((dataputlen - 128) / 256) > 127);
1045 dst[offset] = high;
1046 dst[offset+1] = low;
1047 offset += 2;
1048 }
1049
1050 BUG_ON(offset > putlen);
1051 BUG_ON(putlen - offset != dataputlen);
1052 memcpy(dst + offset, cm->msg.conn_data.data, dataputlen);
1053 offset += dataputlen;
1054
1055 if (cm->msg.conn_data.datalen == dataputlen) {
1056 BUG_ON(cm->length != putlen);
1057 list_add_tail(&(cm->lh), &(cr->msgs));
1058 } else {
1059 *split_conndata = cm;
1060 *sc_sendlen = dataputlen;
1061 }
1062
1063 return putlen;
1064 }
1065
1066 static __u32 cor_add_set_max_cmsg_dly(struct sk_buff *skb,
1067 struct cor_control_retrans *cr, struct cor_control_msg_out *cm,
1068 __u32 spaceleft, int *ackneeded)
1069 {
1070 char *dst;
1071
1072 BUG_ON(KP_MISC_SET_MAX_CMSG_DELAY_CMDLEN != 21);
1073 BUG_ON(cm->length != KP_MISC_SET_MAX_CMSG_DELAY_CMDLEN);
1074
1075 if (unlikely(spaceleft < 21))
1076 return 0;
1077
1078 dst = skb_put(skb, 21);
1079 BUG_ON(dst == 0);
1080
1081 dst[0] = get_kp_code(KP_MISC, KP_MISC_SET_MAX_CMSG_DELAY);
1082 cor_put_u32(dst + 1, cm->msg.set_max_cmsg_delay.ack_fast_delay);
1083 cor_put_u32(dst + 5, cm->msg.set_max_cmsg_delay.ack_slow_delay);
1084 cor_put_u32(dst + 9,
1085 cm->msg.set_max_cmsg_delay.ackconn_lowlatency_delay);
1086 cor_put_u32(dst + 13,
1087 cm->msg.set_max_cmsg_delay.ackconn_highlatency_delay);
1088 cor_put_u32(dst + 17, cm->msg.set_max_cmsg_delay.pong_delay);
1089
1090 list_add_tail(&(cm->lh), &(cr->msgs));
1091 if (*ackneeded != ACK_NEEDED_FAST)
1092 *ackneeded = ACK_NEEDED_SLOW;
1093
1094 return 21;
1095 }
1096
1097 static __u32 cor_add_set_rcvmtu(struct sk_buff *skb,
1098 struct cor_control_retrans *cr, struct cor_control_msg_out *cm,
1099 __u32 spaceleft, int *ackneeded)
1100 {
1101 char *dst;
1102
1103 BUG_ON(KP_MISC_SET_RECEIVE_MTU_CMDLEN != 5);
1104 BUG_ON(cm->length != KP_MISC_SET_RECEIVE_MTU_CMDLEN);
1105
1106 if (unlikely(spaceleft < 5))
1107 return 0;
1108
1109 dst = skb_put(skb, 5);
1110 BUG_ON(dst == 0);
1111
1112 dst[0] = get_kp_code(KP_MISC, KP_MISC_SET_RECEIVE_MTU);
1113 cor_put_u32(dst + 1, cm->msg.set_rcvmtu.rcvmtu);
1114
1115 list_add_tail(&(cm->lh), &(cr->msgs));
1116 if (*ackneeded != ACK_NEEDED_FAST)
1117 *ackneeded = ACK_NEEDED_SLOW;
1118
1119 return 5;
1120 }
1121
1122 static __u32 cor_add_message(struct sk_buff *skb,
1123 struct cor_control_retrans *cr, struct cor_control_msg_out *cm,
1124 __u32 spaceleft, int nbstate, unsigned long cmsg_send_start_j,
1125 ktime_t cmsg_send_start_kt,
1126 struct cor_control_msg_out **split_conndata, __u32 *sc_sendlen,
1127 int *ackneeded)
1128 {
1129 BUG_ON(split_conndata != 0 && *split_conndata != 0);
1130 BUG_ON(sc_sendlen != 0 && *sc_sendlen != 0);
1131
1132 switch (cm->type) {
1133 case MSGTYPE_ACK:
1134 return cor_add_ack(skb, cr, cm, spaceleft);
1135 case MSGTYPE_ACK_CONN:
1136 return cor_add_ack_conn(skb, cr, cm, spaceleft,
1137 cmsg_send_start_j, ackneeded);
1138 case MSGTYPE_PONG:
1139 return cor_add_pong(skb, cr, cm, spaceleft, nbstate,
1140 cmsg_send_start_kt, ackneeded);
1141 case MSGTYPE_CONNECT:
1142 return cor_add_connect(skb, cr, cm, spaceleft, ackneeded);
1143 case MSGTYPE_CONNECT_SUCCESS:
1144 return cor_add_connect_success(skb, cr, cm, spaceleft,
1145 ackneeded);
1146 case MSGTYPE_RESET_CONN:
1147 return cor_add_reset_conn(skb, cr, cm, spaceleft, ackneeded);
1148 case MSGTYPE_CONNDATA:
1149 return cor_add_conndata(skb, cr, cm, spaceleft, split_conndata,
1150 sc_sendlen);
1151 case MSGTYPE_SET_MAX_CMSG_DELAY:
1152 return cor_add_set_max_cmsg_dly(skb, cr, cm, spaceleft,
1153 ackneeded);
1154 case MSGTYPE_SET_RCVMTU:
1155 return cor_add_set_rcvmtu(skb, cr, cm, spaceleft,
1156 ackneeded);
1157 default:
1158 BUG();
1159 }
1160 BUG();
1161 return 0;
1162 }
1163
1164 static __u32 ___cor_send_messages(struct cor_neighbor *nb, struct sk_buff *skb,
1165 struct cor_control_retrans *cr, struct list_head *cmsgs,
1166 __u32 spaceleft, int nbstate, unsigned long cmsg_send_start_j,
1167 ktime_t cmsg_send_start_kt,
1168 struct cor_control_msg_out **split_conndata, __u32 *sc_sendlen,
1169 int *ackneeded)
1170 {
1171 __u32 length = 0;
1172 while (!list_empty(cmsgs)) {
1173 __u32 rc;
1174 struct cor_control_msg_out *cm = container_of(cmsgs->next,
1175 struct cor_control_msg_out, lh);
1176
1177 list_del(&(cm->lh));
1178
1179 rc = cor_add_message(skb, cr, cm, spaceleft - length, nbstate,
1180 cmsg_send_start_j, cmsg_send_start_kt,
1181 split_conndata, sc_sendlen, ackneeded);
1182 if (rc == 0) {
1183 BUG();
1184 list_add(&(cm->lh), cmsgs);
1185 break;
1186 }
1187
1188 BUG_ON(rc != cm->length && cm->type != MSGTYPE_CONNDATA);
1189
1190 length += rc;
1191 }
1192
1193 return length;
1194 }
1195
1196 static __u32 ___cor_send_messages_smcd(struct cor_neighbor *nb,
1197 struct sk_buff *skb, struct cor_control_retrans *cr,
1198 __u32 spaceleft, int nbstate, unsigned long cmsg_send_start_j,
1199 ktime_t cmsg_send_start_kt, int *ackneeded)
1200 {
1201 struct cor_control_msg_out *cm;
1202 __u32 rc;
1203
1204 cm = cor_alloc_control_msg(nb, ACM_PRIORITY_MED);
1205
1206 if (unlikely(cm == 0))
1207 return 0;
1208
1209 cm->type = MSGTYPE_SET_MAX_CMSG_DELAY;
1210 cm->msg.set_max_cmsg_delay.ack_fast_delay =
1211 CMSG_MAXDELAY_ACK_FAST_MS * 1000;
1212 cm->msg.set_max_cmsg_delay.ack_slow_delay =
1213 CMSG_MAXDELAY_ACK_SLOW_MS * 1000;
1214 cm->msg.set_max_cmsg_delay.ackconn_lowlatency_delay =
1215 CMSG_MAXDELAY_ACKCONN_LOWLATENCY_MS * 1000;
1216 cm->msg.set_max_cmsg_delay.ackconn_highlatency_delay =
1217 CMSG_MAXDELAY_ACKCONN_HIGHLATENCY_MS * 1000;
1218 cm->msg.set_max_cmsg_delay.pong_delay =
1219 CMSG_MAXDELAY_OTHER_MS * 1000;
1220 cm->length = KP_MISC_SET_MAX_CMSG_DELAY_CMDLEN;
1221
1222 rc = cor_add_message(skb, cr, cm, spaceleft, nbstate, cmsg_send_start_j,
1223 cmsg_send_start_kt, 0, 0, ackneeded);
1224
1225 nb->max_cmsg_delay_sent = 1;
1226
1227 return rc;
1228 }
1229
1230 static __u32 ___cor_send_messages_rcvmtu(struct cor_neighbor *nb,
1231 struct sk_buff *skb, struct cor_control_retrans *cr,
1232 __u32 spaceleft, int nbstate, unsigned long cmsg_send_start_j,
1233 ktime_t cmsg_send_start_kt, int *ackneeded)
1234 {
1235 struct cor_control_msg_out *cm;
1236 __u32 rc;
1237
1238 cm = cor_alloc_control_msg(nb, ACM_PRIORITY_MED);
1239
1240 if (unlikely(cm == 0))
1241 return 0;
1242
1243 cm->type = MSGTYPE_SET_RCVMTU;
1244 cm->msg.set_rcvmtu.rcvmtu = cor_rcv_mtu(nb);
1245 cm->length = KP_MISC_SET_RECEIVE_MTU_CMDLEN;
1246
1247 rc = cor_add_message(skb, cr, cm, spaceleft, nbstate, cmsg_send_start_j,
1248 cmsg_send_start_kt, 0, 0, ackneeded);
1249
1250 atomic_set(&(nb->rcvmtu_sendneeded), 0);
1251
1252 return rc;
1253 }
1254
1255 #define CMSGQUEUE_PONG 1
1256 #define CMSGQUEUE_ACK_FAST 2
1257 #define CMSGQUEUE_ACK_SLOW 3
1258 #define CMSGQUEUE_ACK_CONN_URGENT 4
1259 #define CMSGQUEUE_ACK_CONN_LOWLAT 5
1260 #define CMSGQUEUE_ACK_CONN_HIGHLAT 6
1261 #define CMSGQUEUE_CONNDATA_LOWLAT 7
1262 #define CMSGQUEUE_CONNDATA_HIGHLAT 8
1263 #define CMSGQUEUE_OTHER 9
1264
1265 static void cor_requeue_message(struct cor_control_msg_out *cm)
1266 {
1267 if (cm->type == MSGTYPE_ACK_CONN) {
1268 struct cor_conn *cn_l = cm->msg.ack_conn.src_in;
1269
1270 spin_lock_bh(&(cn_l->rcv_lock));
1271 if (unlikely(cor_ackconn_prepare_requeue(cn_l, cm) == 0)) {
1272 cor_free_control_msg(cm);
1273 } else {
1274 spin_lock_bh(&(cm->nb->cmsg_lock));
1275
1276 if (unlikely(cm->msg.ack_conn.queue ==
1277 CMSGQUEUE_ACK_CONN_URGENT)) {
1278 list_add(&(cm->lh), &(cm->nb->
1279 cmsg_queue_ackconn_urgent));
1280 } else if (cm->msg.ack_conn.queue ==
1281 CMSGQUEUE_ACK_CONN_LOWLAT) {
1282 list_add(&(cm->lh), &(cm->nb->
1283 cmsg_queue_ackconn_lowlat));
1284 } else if (cm->msg.ack_conn.queue ==
1285 CMSGQUEUE_ACK_CONN_HIGHLAT) {
1286 list_add(&(cm->lh), &(cm->nb->
1287 cmsg_queue_ackconn_highlat));
1288 } else {
1289 BUG();
1290 }
1291
1292 cm->nb->cmsg_otherlength += cm->length;
1293
1294 list_add(&(cm->msg.ack_conn.conn_acks),
1295 &(cn_l->src.in.acks_pending));
1296 cor_try_merge_ackconns(cn_l, cm);
1297
1298 spin_unlock_bh(&(cm->nb->cmsg_lock));
1299 }
1300 spin_unlock_bh(&(cn_l->rcv_lock));
1301 return;
1302 }
1303
1304 cor_enqueue_control_msg(cm, ADDCMSG_SRC_READD);
1305 }
1306
1307 static void cor_requeue_messages(struct list_head *lh)
1308 {
1309 while (list_empty(lh) == 0) {
1310 struct cor_control_msg_out *cm = container_of(lh->prev,
1311 struct cor_control_msg_out, lh);
1312 list_del(&(cm->lh));
1313 cor_requeue_message(cm);
1314 }
1315 }
1316
1317 static int __cor_send_messages_send(struct cor_neighbor *nb,
1318 struct sk_buff *skb, char *packet_type, int ping,
1319 int initsession, struct cor_control_retrans *cr,
1320 struct list_head *cmsgs, __u32 spaceleft, int nbstate,
1321 unsigned long cmsg_send_start_j, ktime_t cmsg_send_start_kt,
1322 int *sent)
1323 {
1324 int rc;
1325 int ackneeded = ACK_NEEDED_NO;
1326 __u32 length = 0;
1327 __u32 pinglen = 0;
1328 __u32 pingcookie = 0;
1329 unsigned long last_ping_time;
1330 struct cor_control_msg_out *split_conndata = 0;
1331 __u32 sc_sendlen = 0;
1332
1333 if (ping != TIMETOSENDPING_NO) {
1334 __u32 rc;
1335
1336 if (unlikely(initsession)) {
1337 rc = cor_add_init_session(skb, nb->sessionid,
1338 spaceleft - length);
1339 BUG_ON(rc <= 0);
1340 pinglen = rc;
1341 length += rc;
1342 }
1343
1344 pingcookie = cor_add_ping_req(nb, &last_ping_time);
1345 rc = cor_add_ping(skb, pingcookie, spaceleft - length);
1346 BUG_ON(rc <= 0);
1347 pinglen += rc;
1348 length += rc;
1349 }
1350
1351 if (likely(nbstate == NEIGHBOR_STATE_ACTIVE) &&
1352 unlikely(nb->max_cmsg_delay_sent == 0))
1353 length += ___cor_send_messages_smcd(nb, skb, cr,
1354 spaceleft - length, nbstate, cmsg_send_start_j,
1355 cmsg_send_start_kt, &ackneeded);
1356
1357 if (unlikely(atomic_read(&(nb->rcvmtu_sendneeded)) != 0)) {
1358 length += ___cor_send_messages_rcvmtu(nb, skb, cr,
1359 spaceleft - length, nbstate, cmsg_send_start_j,
1360 cmsg_send_start_kt, &ackneeded);
1361 }
1362
1363 length += ___cor_send_messages(nb, skb, cr, cmsgs, spaceleft - length,
1364 nbstate, cmsg_send_start_j, cmsg_send_start_kt,
1365 &split_conndata, &sc_sendlen, &ackneeded);
1366
1367 BUG_ON(length > spaceleft);
1368
1369 if (likely(ping != TIMETOSENDPING_FORCE) &&
1370 pinglen != 0 && unlikely(length == pinglen)) {
1371 cor_unadd_ping_req(nb, pingcookie, last_ping_time, 0);
1372 goto drop;
1373 }
1374
1375 if (unlikely(length == 0)) {
1376 drop:
1377 kfree_skb(skb);
1378
1379 BUG_ON(list_empty(&(cr->msgs)) == 0);
1380 kref_put(&(cr->ref), cor_free_control_retrans);
1381
1382 nb->kpacket_seqno--;
1383 return QOS_RESUME_DONE;
1384 }
1385
1386 //padding(skb, spaceleft - length);
1387 BUG_ON(spaceleft - length != 0 &&
1388 (split_conndata == 0 || spaceleft - length != 1));
1389
1390 if (ackneeded == ACK_NEEDED_NO) {
1391 *packet_type = PACKET_TYPE_CMSG_NOACK;
1392 } else if (ackneeded == ACK_NEEDED_SLOW) {
1393 *packet_type = PACKET_TYPE_CMSG_ACKSLOW;
1394 } else if (ackneeded == ACK_NEEDED_FAST) {
1395 *packet_type = PACKET_TYPE_CMSG_ACKFAST;
1396 } else {
1397 BUG();
1398 }
1399
1400 rc = cor_dev_queue_xmit(skb, nb->queue, QOS_CALLER_KPACKET);
1401 if (rc == NET_XMIT_SUCCESS)
1402 *sent = 1;
1403
1404 if (rc == NET_XMIT_DROP) {
1405 if (ping != 0)
1406 cor_unadd_ping_req(nb, pingcookie, last_ping_time, 1);
1407
1408 atomic_inc(&(nb->cmsg_bulk_readds));
1409 if (split_conndata != 0)
1410 cor_requeue_message(split_conndata);
1411
1412 cor_requeue_messages(&(cr->msgs));
1413
1414 kref_put(&(cr->ref), cor_free_control_retrans);
1415
1416 atomic_dec(&(nb->cmsg_bulk_readds));
1417
1418 spin_lock_bh(&(nb->cmsg_lock));
1419 cor_schedule_controlmsg_timer(nb);
1420 spin_unlock_bh(&(nb->cmsg_lock));
1421 } else {
1422 struct list_head *curr = cr->msgs.next;
1423
1424 if (pingcookie != 0)
1425 cor_ping_sent(nb, pingcookie);
1426
1427 while (curr != &(cr->msgs)) {
1428 struct cor_control_msg_out *cm = container_of(curr,
1429 struct cor_control_msg_out, lh);
1430
1431 curr = curr->next;
1432
1433 if (cm->type == MSGTYPE_ACK || unlikely(
1434 cm->type == MSGTYPE_PONG &&
1435 (nbstate != NEIGHBOR_STATE_ACTIVE))) {
1436 list_del(&(cm->lh));
1437 cor_free_control_msg(cm);
1438 } else if (unlikely(cm->type == MSGTYPE_PONG &&
1439 atomic_inc_return(
1440 &(nb->cmsg_pongs_retrans_cnt)) >
1441 MAX_PONG_CMSGS_RETRANS_PER_NEIGH)) {
1442 atomic_dec(&(nb->cmsg_pongs_retrans_cnt));
1443 list_del(&(cm->lh));
1444 cor_free_control_msg(cm);
1445 } else if (cm->type == MSGTYPE_CONNDATA) {
1446 cor_schedule_retransmit_conn(
1447 cm->msg.conn_data.cr, 0, 0);
1448 kref_put(&(cm->msg.conn_data.cr->ref),
1449 cor_free_connretrans);
1450 cm->msg.conn_data.cr = 0;
1451 kfree(cm->msg.conn_data.data_orig);
1452 list_del(&(cm->lh));
1453 cor_free_control_msg(cm);
1454 }
1455 }
1456
1457 if (split_conndata != 0) {
1458 BUG_ON(sc_sendlen == 0);
1459 BUG_ON(sc_sendlen >=
1460 split_conndata->msg.conn_data.datalen);
1461
1462 split_conndata->msg.conn_data.seqno += sc_sendlen;
1463 split_conndata->msg.conn_data.data += sc_sendlen;
1464 split_conndata->msg.conn_data.datalen -= sc_sendlen;
1465 split_conndata->length = get_kp_conn_data_length(
1466 split_conndata->msg.conn_data.datalen);
1467 cor_enqueue_control_msg(split_conndata,
1468 ADDCMSG_SRC_SPLITCONNDATA);
1469 }
1470
1471
1472 if (list_empty(&(cr->msgs))) {
1473 kref_put(&(cr->ref), cor_free_control_retrans);
1474 } else {
1475 int fastack = (ackneeded == ACK_NEEDED_FAST);
1476 BUG_ON(ackneeded != ACK_NEEDED_FAST &&
1477 ackneeded != ACK_NEEDED_SLOW);
1478 cor_schedule_retransmit(cr, nb, fastack);
1479 }
1480 }
1481
1482 return (rc == NET_XMIT_SUCCESS) ? QOS_RESUME_DONE : QOS_RESUME_CONG;
1483 }
1484
1485 static int _cor_send_messages_send(struct cor_neighbor *nb, int ping,
1486 int initsession, struct list_head *cmsgs, int nbstate,
1487 __u32 length, __u64 seqno, unsigned long cmsg_send_start_j,
1488 ktime_t cmsg_send_start_kt, int *sent)
1489 {
1490 struct sk_buff *skb;
1491 struct cor_control_retrans *cr;
1492 char *dst;
1493 int rc;
1494
1495 BUG_ON(length > cor_mss_cmsg(nb));
1496 skb = cor_create_packet(nb, length + 7, GFP_ATOMIC);
1497 if (unlikely(skb == 0)) {
1498 printk(KERN_ERR "cor_send_messages(): cannot allocate skb (out of memory?)\n");
1499
1500 cor_requeue_messages(cmsgs);
1501 return QOS_RESUME_CONG;
1502 }
1503
1504 cr = kmem_cache_alloc(cor_controlretrans_slab, GFP_ATOMIC);
1505 if (unlikely(cr == 0)) {
1506 printk(KERN_ERR "cor_send_messages(): cannot allocate cor_control_retrans (out of memory?)\n");
1507 kfree_skb(skb);
1508
1509 cor_requeue_messages(cmsgs);
1510 return QOS_RESUME_CONG;
1511 }
1512
1513 memset(cr, 0, sizeof(struct cor_control_retrans));
1514 kref_init(&(cr->ref));
1515 cr->nb = nb;
1516 cr->seqno = seqno;
1517 INIT_LIST_HEAD(&(cr->msgs));
1518
1519
1520 dst = skb_put(skb, 7);
1521 BUG_ON(dst == 0);
1522
1523 dst[0] = PACKET_TYPE_NONE;
1524 cor_put_u48(dst + 1, seqno);
1525
1526 rc = __cor_send_messages_send(nb, skb, &(dst[0]), ping, initsession, cr,
1527 cmsgs, length, nbstate, cmsg_send_start_j,
1528 cmsg_send_start_kt, sent);
1529
1530 BUG_ON(!list_empty(cmsgs));
1531
1532 return rc;
1533 }
1534
1535 static unsigned long cor_get_cmsg_timeout(struct cor_control_msg_out *cm,
1536 int queue)
1537 {
1538 if (cm->type == MSGTYPE_ACK) {
1539 if (cm->msg.ack.fast != 0) {
1540 BUG_ON(queue != CMSGQUEUE_ACK_FAST);
1541 return cm->time_added + msecs_to_jiffies(
1542 CMSG_MAXDELAY_ACK_FAST_MS);
1543 } else {
1544 BUG_ON(queue != CMSGQUEUE_ACK_SLOW);
1545 return cm->time_added + msecs_to_jiffies(
1546 CMSG_MAXDELAY_ACK_SLOW_MS);
1547 }
1548 } else if (cm->type == MSGTYPE_ACK_CONN) {
1549 __u32 maxdelay_ms = 0;
1550 if (unlikely(queue == CMSGQUEUE_ACK_CONN_URGENT)) {
1551 maxdelay_ms = CMSG_MAXDELAY_ACKCONN_URGENT_MS;
1552 } else if (queue == CMSGQUEUE_ACK_CONN_LOWLAT) {
1553 maxdelay_ms = CMSG_MAXDELAY_ACKCONN_LOWLATENCY_MS;
1554 } else if (queue == CMSGQUEUE_ACK_CONN_HIGHLAT) {
1555 maxdelay_ms = CMSG_MAXDELAY_ACKCONN_HIGHLATENCY_MS;
1556 } else {
1557 BUG();
1558 }
1559 return cm->time_added + msecs_to_jiffies(maxdelay_ms);
1560 } else if (cm->type == MSGTYPE_CONNDATA) {
1561 if (cm->msg.conn_data.highlatency != 0) {
1562 BUG_ON(queue != CMSGQUEUE_CONNDATA_HIGHLAT);
1563 return cm->time_added +
1564 msecs_to_jiffies(
1565 CMSG_MAXDELAY_CONNDATA_MS);
1566 } else {
1567 BUG_ON(queue != CMSGQUEUE_CONNDATA_LOWLAT);
1568 return cm->time_added;
1569 }
1570 } else {
1571 BUG_ON(cm->type == MSGTYPE_PONG && queue != CMSGQUEUE_PONG);
1572 BUG_ON(cm->type != MSGTYPE_PONG && queue != CMSGQUEUE_OTHER);
1573
1574 return cm->time_added +
1575 msecs_to_jiffies(CMSG_MAXDELAY_OTHER_MS);
1576 }
1577 }
1578
1579 static void _cor_peek_message(struct cor_neighbor *nb_cmsglocked, int queue,
1580 struct cor_control_msg_out **currcm, unsigned long *currtimeout,
1581 __u32 **currlen)
1582 {
1583 struct cor_control_msg_out *cm;
1584 unsigned long cmtimeout;
1585
1586 struct list_head *queuelh;
1587 if (queue == CMSGQUEUE_PONG) {
1588 queuelh = &(nb_cmsglocked->cmsg_queue_pong);
1589 } else if (queue == CMSGQUEUE_ACK_FAST) {
1590 queuelh = &(nb_cmsglocked->cmsg_queue_ack_fast);
1591 } else if (queue == CMSGQUEUE_ACK_SLOW) {
1592 queuelh = &(nb_cmsglocked->cmsg_queue_ack_slow);
1593 } else if (queue == CMSGQUEUE_ACK_CONN_URGENT) {
1594 queuelh = &(nb_cmsglocked->cmsg_queue_ackconn_urgent);
1595 } else if (queue == CMSGQUEUE_ACK_CONN_LOWLAT) {
1596 queuelh = &(nb_cmsglocked->cmsg_queue_ackconn_lowlat);
1597 } else if (queue == CMSGQUEUE_ACK_CONN_HIGHLAT) {
1598 queuelh = &(nb_cmsglocked->cmsg_queue_ackconn_highlat);
1599 } else if (queue == CMSGQUEUE_CONNDATA_LOWLAT) {
1600 queuelh = &(nb_cmsglocked->cmsg_queue_conndata_lowlat);
1601 } else if (queue == CMSGQUEUE_CONNDATA_HIGHLAT) {
1602 queuelh = &(nb_cmsglocked->cmsg_queue_conndata_highlat);
1603 } else if (queue == CMSGQUEUE_OTHER) {
1604 queuelh = &(nb_cmsglocked->cmsg_queue_other);
1605 } else {
1606 BUG();
1607 }
1608
1609 if (list_empty(queuelh))
1610 return;
1611
1612 cm = container_of(queuelh->next, struct cor_control_msg_out, lh);
1613 cmtimeout = cor_get_cmsg_timeout(cm, queue);
1614
1615 BUG_ON(cm->nb != nb_cmsglocked);
1616
1617 if (*currcm == 0 || (time_before(cmtimeout, *currtimeout) &&
1618 time_before(jiffies, *currtimeout))) {
1619 *currcm = cm;
1620 *currtimeout = cmtimeout;
1621
1622 if (queue == CMSGQUEUE_PONG) {
1623 *currlen = &(nb_cmsglocked->cmsg_pongslength);
1624 } else {
1625 *currlen = &(nb_cmsglocked->cmsg_otherlength);
1626 }
1627 }
1628 }
1629
1630 static void cor_peek_message(struct cor_neighbor *nb_cmsglocked, int nbstate,
1631 struct cor_control_msg_out **cm, unsigned long *cmtimeout,
1632 __u32 **len, int for_timeout)
1633 {
1634 _cor_peek_message(nb_cmsglocked, CMSGQUEUE_PONG, cm, cmtimeout, len);
1635 if (likely(nbstate == NEIGHBOR_STATE_ACTIVE)) {
1636 _cor_peek_message(nb_cmsglocked, CMSGQUEUE_ACK_FAST, cm,
1637 cmtimeout, len);
1638 _cor_peek_message(nb_cmsglocked, CMSGQUEUE_ACK_SLOW, cm,
1639 cmtimeout, len);
1640 _cor_peek_message(nb_cmsglocked, CMSGQUEUE_ACK_CONN_URGENT, cm,
1641 cmtimeout, len);
1642 _cor_peek_message(nb_cmsglocked, CMSGQUEUE_ACK_CONN_LOWLAT, cm,
1643 cmtimeout, len);
1644 _cor_peek_message(nb_cmsglocked, CMSGQUEUE_ACK_CONN_HIGHLAT, cm,
1645 cmtimeout, len);
1646 if (!for_timeout || atomic_read(
1647 &(nb_cmsglocked->cmsg_delay_conndata)) == 0) {
1648 _cor_peek_message(nb_cmsglocked,
1649 CMSGQUEUE_CONNDATA_LOWLAT,
1650 cm, cmtimeout, len);
1651 _cor_peek_message(nb_cmsglocked,
1652 CMSGQUEUE_CONNDATA_HIGHLAT,
1653 cm, cmtimeout, len);
1654 }
1655 _cor_peek_message(nb_cmsglocked, CMSGQUEUE_OTHER, cm, cmtimeout,
1656 len);
1657 }
1658 }
1659
1660 static unsigned long cor_get_cmsg_timer_timeout(
1661 struct cor_neighbor *nb_cmsglocked, int nbstate)
1662 {
1663 unsigned long pingtimeout = cor_get_next_ping_time(nb_cmsglocked);
1664
1665 struct cor_control_msg_out *cm = 0;
1666 unsigned long cmtimeout;
1667 __u32 *len;
1668
1669 cor_peek_message(nb_cmsglocked, nbstate, &cm, &cmtimeout, &len, 1);
1670
1671 if (cm != 0) {
1672 unsigned long jiffies_tmp = jiffies;
1673
1674 if (time_before(cmtimeout, jiffies_tmp))
1675 return jiffies_tmp;
1676 if (time_before(cmtimeout, pingtimeout))
1677 return cmtimeout;
1678 }
1679
1680 return pingtimeout;
1681 }
1682
1683 static void _cor_dequeue_messages(struct cor_neighbor *nb_cmsglocked,
1684 int nbstate, __u32 targetmss, __u32 *length,
1685 struct list_head *cmsgs)
1686 {
1687 while (1) {
1688 __u32 spaceleft = targetmss - *length;
1689 struct cor_control_msg_out *cm = 0;
1690 unsigned long cmtimeout;
1691 __u32 *len;
1692
1693 cor_peek_message(nb_cmsglocked, nbstate, &cm, &cmtimeout, &len,
1694 0);
1695
1696 if (unlikely(cm == 0))
1697 break;
1698
1699 BUG_ON(len == 0);
1700
1701 if (cm->length > spaceleft) {
1702 if (cm->type == MSGTYPE_CONNDATA) {
1703 BUG_ON(*length == 0 && spaceleft <
1704 get_kp_conn_data_length(1));
1705
1706 if (spaceleft < get_kp_conn_data_length(1) ||
1707 *length > (targetmss/4)*3)
1708 break;
1709 } else {
1710 BUG_ON(*length == 0);
1711 break;
1712 }
1713 }
1714
1715 list_del(&(cm->lh));
1716 *len -= cm->length;
1717
1718 if (cm->type == MSGTYPE_ACK_CONN)
1719 list_del(&(cm->msg.ack_conn.conn_acks));
1720 if (unlikely(cm->type == MSGTYPE_PONG)) {
1721 BUG_ON(cm->nb->cmsg_pongscnt == 0);
1722 cm->nb->cmsg_pongscnt--;
1723 }
1724
1725 if (unlikely(cm->type == MSGTYPE_RESET_CONN)) {
1726 BUG_ON(cm->msg.reset_conn.in_pending_conn_resets == 0);
1727 rb_erase(&(cm->msg.reset_conn.rbn),
1728 &(cm->nb->pending_conn_resets_rb));
1729 cm->msg.reset_conn.in_pending_conn_resets = 0;
1730 kref_put(&(cm->ref), cor_kreffree_bug);
1731 }
1732
1733 BUG_ON(*length + cm->length < *length);
1734 if (cm->length > targetmss - *length) {
1735 BUG_ON(*length >= targetmss);
1736 BUG_ON(cm->type != MSGTYPE_CONNDATA);
1737 *length = targetmss;
1738 } else {
1739 *length += cm->length;
1740 }
1741
1742 list_add_tail(&(cm->lh), cmsgs);
1743 }
1744 }
1745
1746 static __u32 cor_get_total_messages_length(struct cor_neighbor *nb, int ping,
1747 int initsession, int nbstate, int *extralength)
1748 {
1749 __u32 length = nb->cmsg_pongslength;
1750
1751 if (likely(nbstate == NEIGHBOR_STATE_ACTIVE)) {
1752 length += nb->cmsg_otherlength;
1753
1754 if (unlikely(nb->max_cmsg_delay_sent == 0)) {
1755 length += KP_MISC_SET_MAX_CMSG_DELAY_CMDLEN;
1756 *extralength += KP_MISC_SET_MAX_CMSG_DELAY_CMDLEN;
1757 }
1758 }
1759
1760 if (unlikely(atomic_read(&(nb->rcvmtu_sendneeded)) != 0)) {
1761 length += KP_MISC_SET_RECEIVE_MTU_CMDLEN;
1762 *extralength += KP_MISC_SET_RECEIVE_MTU_CMDLEN;
1763 }
1764
1765 if (ping == TIMETOSENDPING_FORCE ||
1766 (length > 0 && ping != TIMETOSENDPING_NO)) {
1767 length += KP_MISC_PING_CMDLEN;
1768 *extralength += KP_MISC_PING_CMDLEN;
1769
1770 if (unlikely(initsession)) {
1771 length += KP_MISC_INIT_SESSION_CMDLEN;
1772 *extralength += KP_MISC_INIT_SESSION_CMDLEN;
1773 }
1774 }
1775
1776 return length;
1777 }
1778
1779 static int cor_dequeue_messages(struct cor_neighbor *nb_cmsglocked, int ping,
1780 int initsession, int nbstate, __u32 targetmss,
1781 __u32 *length, struct list_head *cmsgs)
1782 {
1783 __u32 extralength = 0;
1784 __u32 totallength;
1785
1786 int cmsgqueue_nonpong_empty = (
1787 list_empty(&(nb_cmsglocked->cmsg_queue_ack_fast)) &&
1788 list_empty(&(nb_cmsglocked->cmsg_queue_ack_slow)) &&
1789 list_empty(&(nb_cmsglocked->cmsg_queue_ackconn_urgent)) &&
1790 list_empty(&(nb_cmsglocked->cmsg_queue_ackconn_lowlat)) &&
1791 list_empty(&(nb_cmsglocked->cmsg_queue_ackconn_highlat)) &&
1792 list_empty(&(nb_cmsglocked->cmsg_queue_conndata_lowlat)) &&
1793 list_empty(&(nb_cmsglocked->cmsg_queue_conndata_highlat)) &&
1794 list_empty(&(nb_cmsglocked->cmsg_queue_other)));
1795
1796 BUG_ON(list_empty(&(nb_cmsglocked->cmsg_queue_pong)) &&
1797 nb_cmsglocked->cmsg_pongslength != 0);
1798 BUG_ON(!list_empty(&(nb_cmsglocked->cmsg_queue_pong)) &&
1799 nb_cmsglocked->cmsg_pongslength == 0);
1800 BUG_ON(cmsgqueue_nonpong_empty &&
1801 nb_cmsglocked->cmsg_otherlength != 0);
1802 BUG_ON(!cmsgqueue_nonpong_empty &&
1803 nb_cmsglocked->cmsg_otherlength == 0);
1804
1805 totallength = cor_get_total_messages_length(nb_cmsglocked, ping,
1806 initsession, nbstate, &extralength);
1807
1808 if (totallength == 0)
1809 return 1;
1810
1811 if (totallength < targetmss && ping != TIMETOSENDPING_FORCE &&
1812 time_after(cor_get_cmsg_timer_timeout(nb_cmsglocked,
1813 nbstate), jiffies))
1814 return 1;
1815
1816 *length = extralength;
1817
1818 _cor_dequeue_messages(nb_cmsglocked, nbstate, targetmss, length, cmsgs);
1819
1820 BUG_ON(*length == 0);
1821 BUG_ON(*length > targetmss);
1822
1823 return 0;
1824 }
1825
1826 static struct cor_control_retrans *cor_get_next_timeouted_retrans(
1827 struct cor_neighbor *nb_retranslocked)
1828 {
1829 if (list_empty(&(nb_retranslocked->retrans_fast_list)) == 0) {
1830 struct cor_control_retrans *cr = container_of(
1831 nb_retranslocked->retrans_fast_list.next,
1832 struct cor_control_retrans, timeout_list);
1833 BUG_ON(cr->nb != nb_retranslocked);
1834
1835 if (time_before_eq(cr->timeout, jiffies)) {
1836 return cr;
1837 }
1838 }
1839
1840 if (list_empty(&(nb_retranslocked->retrans_slow_list)) == 0) {
1841 struct cor_control_retrans *cr = container_of(
1842 nb_retranslocked->retrans_slow_list.next,
1843 struct cor_control_retrans, timeout_list);
1844 BUG_ON(cr->nb != nb_retranslocked);
1845
1846 if (time_before_eq(cr->timeout, jiffies)) {
1847 return cr;
1848 }
1849 }
1850
1851 return 0;
1852 }
1853
1854 static void cor_add_timeouted_retrans(struct cor_neighbor *nb)
1855 {
1856 spin_lock_bh(&(nb->retrans_lock));
1857
1858 while (1) {
1859 struct cor_control_retrans *cr =
1860 cor_get_next_timeouted_retrans(nb);
1861
1862 if (cr == 0)
1863 break;
1864
1865 list_del(&(cr->timeout_list));
1866 rb_erase(&(cr->rbn), &(nb->kp_retransmits_rb));
1867
1868 cor_requeue_control_retrans(cr);
1869
1870 kref_put(&(cr->ref), cor_kreffree_bug); /* list_del */
1871 kref_put(&(cr->ref), cor_free_control_retrans); /* rb */
1872 }
1873
1874 if (list_empty(&(nb->retrans_fast_list)) == 0 ||
1875 list_empty(&(nb->retrans_slow_list)) == 0) {
1876 if (mod_timer(&(nb->retrans_timer),
1877 cor_get_retransmit_timeout(nb)) == 0) {
1878 cor_nb_kref_get(nb, "retransmit_timer");
1879 }
1880 }
1881
1882 spin_unlock_bh(&(nb->retrans_lock));
1883 }
1884
1885 static void _cor_delete_all_cmsgs(struct list_head *cmsgs)
1886 {
1887 while (!list_empty(cmsgs)) {
1888 struct cor_control_msg_out *cm = container_of(cmsgs->next,
1889 struct cor_control_msg_out, lh);
1890
1891 list_del(&(cm->lh));
1892
1893 if (cm->type == MSGTYPE_CONNDATA) {
1894 cor_schedule_retransmit_conn(cm->msg.conn_data.cr, 0,
1895 0);
1896 kfree(cm->msg.conn_data.data_orig);
1897 }
1898
1899 cor_free_control_msg(cm);
1900 }
1901 }
1902
1903 static void cor_delete_all_cmsgs(struct cor_neighbor *nb)
1904 {
1905 while (1) {
1906 struct list_head cmsgs;
1907 __u32 length = 0;
1908
1909 INIT_LIST_HEAD(&cmsgs);
1910
1911 spin_lock_bh(&(nb->cmsg_lock));
1912 _cor_dequeue_messages(nb, NEIGHBOR_STATE_ACTIVE, 65536, &length,
1913 &cmsgs);
1914 spin_unlock_bh(&(nb->cmsg_lock));
1915
1916 if (list_empty(&cmsgs))
1917 break;
1918
1919 _cor_delete_all_cmsgs(&cmsgs);
1920 }
1921 }
1922
1923 static int cor_reset_timeouted_conn(struct cor_neighbor *nb,
1924 struct cor_conn *trgt_out)
1925 {
1926 struct cor_conn_bidir *cnb = cor_get_conn_bidir(trgt_out);
1927 struct cor_conn *src_in = cor_get_conn_reversedir(trgt_out);
1928
1929 int resetted = 0;
1930
1931 spin_lock_bh(&(cnb->cli.rcv_lock));
1932 spin_lock_bh(&(cnb->srv.rcv_lock));
1933
1934 BUG_ON(trgt_out->targettype != TARGET_OUT);
1935 BUG_ON(trgt_out->trgt.out.nb != nb);
1936
1937 if (unlikely(trgt_out->isreset != 0))
1938 goto unlock;
1939
1940 if (likely(trgt_out->trgt.out.in_nb_busy_list != 0)) {
1941 if (likely(time_before(jiffies,
1942 trgt_out->trgt.out.jiffies_last_act +
1943 CONN_BUSY_INACTIVITY_TIMEOUT_SEC * HZ))) {
1944 goto unlock;
1945 }
1946 } else {
1947 if (likely(time_before(jiffies,
1948 trgt_out->trgt.out.jiffies_last_act +
1949 CONN_ACTIVITY_UPDATEINTERVAL_SEC * HZ +
1950 CONN_INACTIVITY_TIMEOUT_SEC * HZ))) {
1951 goto unlock;
1952 }
1953 }
1954
1955 resetted = (cor_send_reset_conn(nb, cor_get_connid_reverse(
1956 src_in->src.in.conn_id), 1) == 0);
1957 if (unlikely(resetted == 0))
1958 goto unlock;
1959
1960 BUG_ON(trgt_out->isreset != 0);
1961 trgt_out->isreset = 1;
1962
1963 cor_reset_conn_locked(cnb);
1964
1965 unlock:
1966 spin_unlock_bh(&(cnb->srv.rcv_lock));
1967 spin_unlock_bh(&(cnb->cli.rcv_lock));
1968
1969 return resetted;
1970 }
1971
1972 static void _cor_reset_timeouted_conns(struct cor_neighbor *nb,
1973 struct list_head *nb_snd_conn_list)
1974 {
1975 int i;
1976 for (i=0;i<10000;i++) {
1977 unsigned long iflags;
1978 struct cor_conn *trgt_out;
1979
1980 int resetted;
1981
1982 spin_lock_irqsave(&(nb->conn_list_lock), iflags);
1983
1984 if (list_empty(nb_snd_conn_list)) {
1985 spin_unlock_irqrestore(&(nb->conn_list_lock), iflags);
1986 break;
1987 }
1988
1989 trgt_out = container_of(nb_snd_conn_list->next, struct cor_conn,
1990 trgt.out.nb_list);
1991 cor_conn_kref_get(trgt_out, "stack");
1992
1993 spin_unlock_irqrestore(&(nb->conn_list_lock), iflags);
1994
1995 resetted = cor_reset_timeouted_conn(nb, trgt_out);
1996
1997 cor_conn_kref_put(trgt_out, "stack");
1998
1999 if (likely(resetted == 0))
2000 break;
2001 }
2002 }
2003
2004 static void cor_reset_timeouted_conns(struct cor_neighbor *nb)
2005 {
2006 _cor_reset_timeouted_conns(nb, &(nb->snd_conn_busy_list));
2007 _cor_reset_timeouted_conns(nb, &(nb->snd_conn_idle_list));
2008 }
2009
2010
2011 /**
2012 * may not be called by more than one thread at the same time, because
2013 * 1) readding cor_control_msg_out may reorder them
2014 * 2) multiple pings may be sent
2015 */
2016 int cor_send_messages(struct cor_neighbor *nb, unsigned long cmsg_send_start_j,
2017 ktime_t cmsg_send_start_kt, int *sent)
2018 {
2019 int rc = QOS_RESUME_DONE;
2020 int ping;
2021 int initsession;
2022 __u32 targetmss = cor_mss_cmsg(nb);
2023
2024 int nbstate = cor_get_neigh_state(nb);
2025
2026 if (likely(nbstate == NEIGHBOR_STATE_ACTIVE))
2027 cor_reset_timeouted_conns(nb);
2028
2029 if (unlikely(nbstate == NEIGHBOR_STATE_KILLED)) {
2030 spin_lock_bh(&(nb->retrans_lock));
2031 cor_empty_retrans_queue(nb);
2032 spin_unlock_bh(&(nb->retrans_lock));
2033
2034 cor_delete_all_cmsgs(nb);
2035 return QOS_RESUME_DONE;
2036 }
2037
2038 ping = cor_time_to_send_ping(nb);
2039
2040 spin_lock_bh(&(nb->cmsg_lock));
2041
2042 if (nb->add_retrans_needed != 0) {
2043 nb->add_retrans_needed = 0;
2044 spin_unlock_bh(&(nb->cmsg_lock));
2045 cor_add_timeouted_retrans(nb);
2046 spin_lock_bh(&(nb->cmsg_lock));
2047 }
2048
2049 initsession = unlikely(atomic_read(&(nb->sessionid_snd_needed)) != 0);
2050
2051 while (1) {
2052 struct list_head cmsgs;
2053 __u32 length = 0;
2054 __u64 seqno;
2055
2056 INIT_LIST_HEAD(&cmsgs);
2057
2058 if (cor_dequeue_messages(nb, ping, initsession, nbstate,
2059 targetmss, &length, &cmsgs) != 0) {
2060 cor_schedule_controlmsg_timer(nb);
2061 spin_unlock_bh(&(nb->cmsg_lock));
2062 return QOS_RESUME_DONE;
2063 }
2064
2065 nb->kpacket_seqno++;
2066 seqno = nb->kpacket_seqno;
2067
2068 spin_unlock_bh(&(nb->cmsg_lock));
2069
2070 rc = _cor_send_messages_send(nb, ping, initsession, &cmsgs,
2071 nbstate, length, seqno, cmsg_send_start_j,
2072 cmsg_send_start_kt, sent);
2073
2074 if (rc != QOS_RESUME_DONE)
2075 return rc;
2076
2077 ping = 0;
2078 initsession = 0;
2079
2080 spin_lock_bh(&(nb->cmsg_lock));
2081 }
2082 }
2083
2084 static unsigned long cor_calc_cmsg_send_start_j(unsigned long cmsg_timer_timeout)
2085 {
2086 unsigned long jiffies_tmp = jiffies;
2087 if (unlikely(time_after(cmsg_timer_timeout, jiffies_tmp)))
2088 return jiffies_tmp;
2089 else
2090 return cmsg_timer_timeout;
2091 }
2092
2093 static ktime_t cor_calc_cmsg_send_start_kt(unsigned long cmsg_timer_timeout)
2094 {
2095 ktime_t now = ktime_get();
2096 unsigned long jiffies_tmp = jiffies;
2097
2098 unsigned long jiffies_delayed;
2099 if (unlikely(time_after(cmsg_timer_timeout, jiffies_tmp))) {
2100 jiffies_delayed = 0;
2101 } else {
2102 jiffies_delayed = jiffies_tmp - cmsg_timer_timeout;
2103 if (unlikely(jiffies_delayed > HZ/10)) {
2104 jiffies_delayed = HZ/10;
2105 }
2106 }
2107
2108 return ns_to_ktime(ktime_to_ns(now) -
2109 1000LL * jiffies_to_usecs(jiffies_delayed));
2110 }
2111
2112
2113 void cor_controlmsg_timerfunc(struct timer_list *cmsg_timer)
2114 {
2115 struct cor_neighbor *nb = container_of(cmsg_timer,
2116 struct cor_neighbor, cmsg_timer);
2117 unsigned long cmsg_timer_timeout = (unsigned long)
2118 atomic64_read(&(nb->cmsg_timer_timeout));
2119 unsigned long cmsg_send_start_j = cor_calc_cmsg_send_start_j(
2120 cmsg_timer_timeout);
2121 ktime_t cmsg_send_start_kt = cor_calc_cmsg_send_start_kt(
2122 cmsg_timer_timeout);
2123 cor_qos_enqueue(nb->queue, &(nb->rb_kp), cmsg_send_start_j,
2124 cmsg_send_start_kt, QOS_CALLER_KPACKET, 0);
2125 cor_nb_kref_put(nb, "controlmsg_timer");
2126 }
2127
2128 static int cor_cmsg_full_packet(struct cor_neighbor *nb, int nbstate)
2129 {
2130 __u32 extralength = 0;
2131 int ping = cor_time_to_send_ping(nb);
2132 int initsession = unlikely(atomic_read(&(nb->sessionid_snd_needed)) !=
2133 0);
2134 __u32 len = cor_get_total_messages_length(nb, ping, initsession,
2135 nbstate, &extralength);
2136
2137 if (len == 0)
2138 return 0;
2139 if (len < cor_mss_cmsg(nb))
2140 return 0;
2141
2142 return 1;
2143 }
2144
2145 void cor_schedule_controlmsg_timer(struct cor_neighbor *nb_cmsglocked)
2146 {
2147 unsigned long timeout;
2148 int nbstate = cor_get_neigh_state(nb_cmsglocked);
2149
2150 if (unlikely(nbstate == NEIGHBOR_STATE_KILLED))
2151 goto now;
2152
2153 if (unlikely(atomic_read(&(nb_cmsglocked->rcvmtu_sendneeded)) != 0))
2154 goto now;
2155
2156 if (atomic_read(&(nb_cmsglocked->cmsg_bulk_readds)) != 0)
2157 return;
2158
2159 if (cor_cmsg_full_packet(nb_cmsglocked, nbstate))
2160 goto now;
2161
2162 if (nb_cmsglocked->add_retrans_needed != 0)
2163 goto now;
2164
2165 timeout = cor_get_cmsg_timer_timeout(nb_cmsglocked, nbstate);
2166
2167 if (0) {
2168 now:
2169 cor_qos_enqueue(nb_cmsglocked->queue, &(nb_cmsglocked->rb_kp),
2170 jiffies, ktime_get(), QOS_CALLER_KPACKET, 0);
2171 } else if (time_before_eq(timeout, jiffies)) {
2172 unsigned long cmsg_send_start_j = cor_calc_cmsg_send_start_j(
2173 timeout);
2174 ktime_t cmsg_send_start_kt = cor_calc_cmsg_send_start_kt(
2175 timeout);
2176 cor_qos_enqueue(nb_cmsglocked->queue, &(nb_cmsglocked->rb_kp),
2177 cmsg_send_start_j, cmsg_send_start_kt,
2178 QOS_CALLER_KPACKET, 0);
2179 } else {
2180 atomic64_set(&(nb_cmsglocked->cmsg_timer_timeout), timeout);
2181 barrier();
2182 if (mod_timer(&(nb_cmsglocked->cmsg_timer), timeout) == 0) {
2183 cor_nb_kref_get(nb_cmsglocked, "controlmsg_timer");
2184 }
2185 }
2186 }
2187
2188 static int cor_insert_pending_conn_resets(struct cor_control_msg_out *ins)
2189 {
2190 struct cor_neighbor *nb = ins->nb;
2191 __u32 conn_id = ins->msg.reset_conn.conn_id;
2192
2193 struct rb_root *root;
2194 struct rb_node **p;
2195 struct rb_node *parent = 0;
2196
2197 BUG_ON(nb == 0);
2198 BUG_ON(ins->msg.reset_conn.in_pending_conn_resets != 0);
2199
2200 root = &(nb->pending_conn_resets_rb);
2201 p = &(root->rb_node);
2202
2203 while ((*p) != 0) {
2204 struct cor_control_msg_out *cm = container_of(*p,
2205 struct cor_control_msg_out,
2206 msg.reset_conn.rbn);
2207 __u32 cm_connid = cm->msg.reset_conn.conn_id;
2208
2209 BUG_ON(cm->nb != ins->nb);
2210 BUG_ON(cm->type != MSGTYPE_RESET_CONN);
2211
2212 parent = *p;
2213 if (conn_id == cm_connid) {
2214 return 1;
2215 } else if (conn_id < cm_connid) {
2216 p = &(*p)->rb_left;
2217 } else if (conn_id > cm_connid) {
2218 p = &(*p)->rb_right;
2219 } else {
2220 BUG();
2221 }
2222 }
2223
2224 kref_get(&(ins->ref));
2225 rb_link_node(&(ins->msg.reset_conn.rbn), parent, p);
2226 rb_insert_color(&(ins->msg.reset_conn.rbn), root);
2227 ins->msg.reset_conn.in_pending_conn_resets = 1;
2228
2229 return 0;
2230 }
2231
2232 static void cor_free_oldest_pong(struct cor_neighbor *nb)
2233 {
2234 struct cor_control_msg_out *cm = container_of(nb->cmsg_queue_pong.next,
2235 struct cor_control_msg_out, lh);
2236
2237 BUG_ON(list_empty(&(nb->cmsg_queue_pong)));
2238 BUG_ON(unlikely(cm->type != MSGTYPE_PONG));
2239
2240 list_del(&(cm->lh));
2241 nb->cmsg_pongslength -= cm->length;
2242 BUG_ON(nb->cmsg_pongscnt == 0);
2243 cm->nb->cmsg_pongscnt--;
2244 cor_free_control_msg(cm);
2245 }
2246
2247 static struct list_head * _cor_enqueue_control_msg_getqueue(
2248 struct cor_control_msg_out *cm)
2249 {
2250 if (cm->type == MSGTYPE_ACK) {
2251 if (cm->msg.ack.fast != 0) {
2252 return &(cm->nb->cmsg_queue_ack_fast);
2253 } else {
2254 return &(cm->nb->cmsg_queue_ack_slow);
2255 }
2256 } else if (cm->type == MSGTYPE_ACK_CONN) {
2257 if (unlikely(cm->msg.ack_conn.queue == CMSGQUEUE_ACK_CONN_URGENT)) {
2258 return &(cm->nb->cmsg_queue_ackconn_urgent);
2259 } else if (cm->msg.ack_conn.queue == CMSGQUEUE_ACK_CONN_LOWLAT) {
2260 return &(cm->nb->cmsg_queue_ackconn_lowlat);
2261 } else if (cm->msg.ack_conn.queue == CMSGQUEUE_ACK_CONN_HIGHLAT) {
2262 return &(cm->nb->cmsg_queue_ackconn_highlat);
2263 } else {
2264 BUG();
2265 }
2266 } else if (cm->type == MSGTYPE_CONNDATA) {
2267 if (cm->msg.conn_data.highlatency != 0) {
2268 return &(cm->nb->cmsg_queue_conndata_highlat);
2269 } else {
2270 return &(cm->nb->cmsg_queue_conndata_lowlat);
2271 }
2272 } else {
2273 return &(cm->nb->cmsg_queue_other);
2274 }
2275 }
2276
2277 static int _cor_enqueue_control_msg(struct cor_control_msg_out *cm, int src)
2278 {
2279 if (unlikely(cm->type == MSGTYPE_PONG)) {
2280 BUG_ON(src == ADDCMSG_SRC_SPLITCONNDATA);
2281
2282 if (cm->nb->cmsg_pongscnt >= MAX_PONG_CMSGS_PER_NEIGH) {
2283 if (src != ADDCMSG_SRC_NEW) {
2284 BUG_ON(cm->nb->cmsg_pongscnt == 0);
2285 cm->nb->cmsg_pongscnt--;
2286 cor_free_control_msg(cm);
2287 return 1;
2288 } else {
2289 cor_free_oldest_pong(cm->nb);
2290 }
2291 }
2292
2293 cm->nb->cmsg_pongscnt++;
2294 cm->nb->cmsg_pongslength += cm->length;
2295
2296 if (src != ADDCMSG_SRC_NEW) {
2297 list_add(&(cm->lh), &(cm->nb->cmsg_queue_pong));
2298 } else {
2299 list_add_tail(&(cm->lh), &(cm->nb->cmsg_queue_pong));
2300 }
2301
2302 return 0;
2303 } else if (unlikely(cm->type == MSGTYPE_RESET_CONN)) {
2304 if (cor_insert_pending_conn_resets(cm) != 0) {
2305 cm->type = 0;
2306 cor_free_control_msg(cm);
2307 return 1;
2308 }
2309 }
2310
2311 cm->nb->cmsg_otherlength += cm->length;
2312 if (src == ADDCMSG_SRC_NEW) {
2313 list_add_tail(&(cm->lh), _cor_enqueue_control_msg_getqueue(cm));
2314 } else {
2315 BUG_ON(src == ADDCMSG_SRC_SPLITCONNDATA &&
2316 cm->type != MSGTYPE_CONNDATA);
2317 BUG_ON(src == ADDCMSG_SRC_READD &&
2318 cm->type == MSGTYPE_ACK_CONN);
2319
2320 list_add(&(cm->lh), _cor_enqueue_control_msg_getqueue(cm));
2321 }
2322
2323 return 0;
2324 }
2325
2326 static void cor_enqueue_control_msg(struct cor_control_msg_out *cm, int src)
2327 {
2328 struct cor_neighbor *nb;
2329
2330 BUG_ON(cm == 0);
2331 nb = cm->nb;
2332 BUG_ON(nb == 0);
2333
2334
2335 if (src == ADDCMSG_SRC_NEW)
2336 cm->time_added = jiffies;
2337
2338 spin_lock_bh(&(nb->cmsg_lock));
2339
2340 if (_cor_enqueue_control_msg(cm, src) != 0)
2341 goto out;
2342
2343 if (src != ADDCMSG_SRC_READD && src != ADDCMSG_SRC_RETRANS)
2344 cor_schedule_controlmsg_timer(nb);
2345
2346 out:
2347 spin_unlock_bh(&(nb->cmsg_lock));
2348 }
2349
2350 void cor_send_rcvmtu(struct cor_neighbor *nb)
2351 {
2352 atomic_set(&(nb->rcvmtu_sendneeded), 1);
2353
2354 spin_lock_bh(&(nb->cmsg_lock));
2355 cor_schedule_controlmsg_timer(nb);
2356 spin_unlock_bh(&(nb->cmsg_lock));
2357 }
2358
2359 void cor_send_pong(struct cor_neighbor *nb, __u32 cookie, ktime_t ping_rcvtime)
2360 {
2361 struct cor_control_msg_out *cm = _cor_alloc_control_msg(nb);
2362
2363 if (unlikely(cm == 0))
2364 return;
2365
2366 cm->nb = nb;
2367 cm->type = MSGTYPE_PONG;
2368 cm->msg.pong.cookie = cookie;
2369 cm->msg.pong.type = MSGTYPE_PONG_TIMEENQUEUED;
2370 cm->msg.pong.ping_rcvtime = ping_rcvtime;
2371 cm->msg.pong.time_enqueued = ktime_get();
2372 cm->length = 13;
2373 cor_enqueue_control_msg(cm, ADDCMSG_SRC_NEW);
2374 }
2375
2376 void cor_send_ack(struct cor_neighbor *nb, __u64 seqno, __u8 fast)
2377 {
2378 struct cor_control_msg_out *cm = cor_alloc_control_msg(nb,
2379 ACM_PRIORITY_HIGH);
2380
2381 if (unlikely(cm == 0))
2382 return;
2383
2384 cm->nb = nb;
2385 cm->type = MSGTYPE_ACK;
2386 cm->msg.ack.seqno = seqno;
2387 cm->msg.ack.fast = fast;
2388 cm->length = 7;
2389 cor_enqueue_control_msg(cm, ADDCMSG_SRC_NEW);
2390 }
2391
2392 static __u8 get_queue_for_ackconn(struct cor_conn *src_in_lx)
2393 {
2394 if (src_in_lx->is_highlatency != 0) {
2395 if (unlikely(cor_ackconn_urgent(src_in_lx))) {
2396 return CMSGQUEUE_ACK_CONN_LOWLAT;
2397 } else {
2398 return CMSGQUEUE_ACK_CONN_HIGHLAT;
2399 }
2400 } else {
2401 if (unlikely(cor_ackconn_urgent(src_in_lx))) {
2402 return CMSGQUEUE_ACK_CONN_URGENT;
2403 } else {
2404 return CMSGQUEUE_ACK_CONN_LOWLAT;
2405 }
2406 }
2407 }
2408
2409 static void cor_set_ooolen_flags(struct cor_control_msg_out *cm)
2410 {
2411 cm->msg.ack_conn.flags = (cm->msg.ack_conn.flags &
2412 (~KP_ACK_CONN_FLAGS_OOO));
2413 cm->msg.ack_conn.flags = (cm->msg.ack_conn.flags |
2414 cor_ooolen_to_flags(cm->msg.ack_conn.length));
2415 }
2416
2417 /* cmsg_lock must be held */
2418 static void cor_remove_pending_ackconn(struct cor_control_msg_out *cm)
2419 {
2420 cm->nb->cmsg_otherlength -= cm->length;
2421 list_del(&(cm->lh));
2422
2423 list_del(&(cm->msg.ack_conn.conn_acks));
2424 cor_conn_kref_put(cm->msg.ack_conn.src_in,
2425 "cor_control_msg_out ack_conn");
2426 cm->msg.ack_conn.src_in = 0;
2427
2428 cm->type = 0;
2429 cor_free_control_msg(cm);
2430 }
2431
2432 /* cmsg_lock must be held */
2433 static void cor_recalc_scheduled_ackconn_size(struct cor_control_msg_out *cm)
2434 {
2435 cm->nb->cmsg_otherlength -= cm->length;
2436 cm->length = 5 + cor_ack_conn_len(cm->msg.ack_conn.flags);
2437 cm->nb->cmsg_otherlength += cm->length;
2438 }
2439
2440 /* cmsg_lock must be held */
2441 static int _cor_try_merge_ackconn(struct cor_conn *src_in_l,
2442 struct cor_control_msg_out *fromcm,
2443 struct cor_control_msg_out *tocm, int from_newack)
2444 {
2445 if (cor_ooolen(fromcm->msg.ack_conn.flags) != 0 &&
2446 cor_ooolen(tocm->msg.ack_conn.flags) != 0) {
2447 __u64 tocmseqno = tocm->msg.ack_conn.seqno_ooo;
2448 __u64 tocmlength = tocm->msg.ack_conn.length;
2449 __u64 fromcmseqno = fromcm->msg.ack_conn.seqno_ooo;
2450 __u64 fromcmlength = fromcm->msg.ack_conn.length;
2451
2452 if (cor_seqno_eq(tocmseqno, fromcmseqno)) {
2453 if (fromcmlength > tocmlength)
2454 tocm->msg.ack_conn.length = fromcmlength;
2455 } else if (cor_seqno_after(fromcmseqno, tocmseqno) &&
2456 cor_seqno_before_eq(fromcmseqno, tocmseqno +
2457 tocmlength)) {
2458 __u64 len = cor_seqno_clean(fromcmseqno + fromcmlength -
2459 tocmseqno);
2460 BUG_ON(len > U32_MAX);
2461 tocm->msg.ack_conn.length = (__u32) len;
2462 } else if (cor_seqno_before(fromcmseqno, tocmseqno) &&
2463 cor_seqno_after_eq(fromcmseqno, tocmseqno)) {
2464 __u64 len = cor_seqno_clean(tocmseqno + tocmlength -
2465 fromcmseqno);
2466 BUG_ON(len > U32_MAX);
2467 tocm->msg.ack_conn.seqno_ooo = fromcmseqno;
2468 tocm->msg.ack_conn.length = (__u32) len;
2469 } else {
2470 return 1;
2471 }
2472 cor_set_ooolen_flags(tocm);
2473 }
2474
2475 if ((fromcm->msg.ack_conn.flags &
2476 KP_ACK_CONN_FLAGS_SEQNO) != 0) {
2477 if ((tocm->msg.ack_conn.flags & KP_ACK_CONN_FLAGS_SEQNO) == 0)
2478 goto setseqno;
2479
2480 BUG_ON(cor_seqno_eq(fromcm->msg.ack_conn.ack_seqno,
2481 tocm->msg.ack_conn.ack_seqno));
2482 if (cor_seqno_after_eq(tocm->msg.ack_conn.ack_seqno,
2483 fromcm->msg.ack_conn.ack_seqno)) {
2484 BUG_ON(cor_seqno_after(fromcm->msg.ack_conn.seqno,
2485 tocm->msg.ack_conn.seqno));
2486 goto skipseqno;
2487 }
2488
2489 BUG_ON(cor_seqno_before(fromcm->msg.ack_conn.seqno,
2490 tocm->msg.ack_conn.seqno));
2491
2492 setseqno:
2493 tocm->msg.ack_conn.flags = (tocm->msg.ack_conn.flags |
2494 KP_ACK_CONN_FLAGS_SEQNO);
2495 tocm->msg.ack_conn.seqno = fromcm->msg.ack_conn.seqno;
2496 tocm->msg.ack_conn.ack_seqno = fromcm->msg.ack_conn.ack_seqno;
2497
2498 skipseqno:
2499 if ((fromcm->msg.ack_conn.flags &
2500 KP_ACK_CONN_FLAGS_WINDOW) != 0)
2501 tocm->msg.ack_conn.flags = (tocm->msg.ack_conn.flags |
2502 KP_ACK_CONN_FLAGS_WINDOW);
2503 }
2504
2505
2506 if (cor_ooolen(fromcm->msg.ack_conn.flags) != 0) {
2507 tocm->msg.ack_conn.seqno_ooo = fromcm->msg.ack_conn.seqno_ooo;
2508 tocm->msg.ack_conn.length = fromcm->msg.ack_conn.length;
2509 cor_set_ooolen_flags(tocm);
2510 }
2511
2512 if ((fromcm->msg.ack_conn.flags & KP_ACK_CONN_FLAGS_PRIORITY) != 0) {
2513 BUG_ON((tocm->msg.ack_conn.flags &
2514 KP_ACK_CONN_FLAGS_PRIORITY) != 0);
2515 tocm->msg.ack_conn.priority_seqno =
2516 fromcm->msg.ack_conn.priority_seqno;
2517 tocm->msg.ack_conn.priority = fromcm->msg.ack_conn.priority;
2518 }
2519
2520 cor_recalc_scheduled_ackconn_size(tocm);
2521 if (from_newack == 0)
2522 cor_remove_pending_ackconn(fromcm);
2523
2524 return 0;
2525 }
2526
2527 /* cmsg_lock must be held */
2528 static void cor_try_merge_ackconns(struct cor_conn *src_in_l,
2529 struct cor_control_msg_out *cm)
2530 {
2531 struct list_head *currlh = cm->msg.ack_conn.conn_acks.next;
2532
2533 while (currlh != &(src_in_l->src.in.acks_pending)) {
2534 struct cor_control_msg_out *currcm = container_of(currlh,
2535 struct cor_control_msg_out,
2536 msg.ack_conn.conn_acks);
2537 currlh = currlh->next;
2538 cor_remove_connack_oooflag_ifold(src_in_l, currcm);
2539 _cor_try_merge_ackconn(src_in_l, currcm, cm, 0);
2540 }
2541 }
2542
2543 static void cor_merge_or_enqueue_ackconn(struct cor_conn *src_in_l,
2544 struct cor_control_msg_out *cm, int src)
2545 {
2546 struct list_head *currlh;
2547
2548 BUG_ON(src_in_l->sourcetype != SOURCE_IN);
2549
2550 spin_lock_bh(&(cm->nb->cmsg_lock));
2551
2552 currlh = src_in_l->src.in.acks_pending.next;
2553 while (currlh != &(src_in_l->src.in.acks_pending)) {
2554 struct cor_control_msg_out *currcm = container_of(currlh,
2555 struct cor_control_msg_out,
2556 msg.ack_conn.conn_acks);
2557
2558 BUG_ON(currcm->nb != cm->nb);
2559 BUG_ON(currcm->type != MSGTYPE_ACK_CONN);
2560 BUG_ON(cm->msg.ack_conn.src_in != src_in_l);
2561 BUG_ON(currcm->msg.ack_conn.conn_id !=
2562 cm->msg.ack_conn.conn_id);
2563
2564 if (_cor_try_merge_ackconn(src_in_l, cm, currcm, 1) == 0) {
2565 cor_try_merge_ackconns(src_in_l, currcm);
2566 cor_schedule_controlmsg_timer(currcm->nb);
2567 spin_unlock_bh(&(currcm->nb->cmsg_lock));
2568 /**
2569 * flags:
2570 * when calling cor_free_control_msg here conn may
2571 * already be locked and priority_send_allowed and
2572 * priority_send_allowed should not be reset
2573 */
2574 cm->msg.ack_conn.flags = 0;
2575 cor_free_control_msg(cm);
2576 return;
2577 }
2578
2579 currlh = currlh->next;
2580 }
2581
2582 list_add_tail(&(cm->msg.ack_conn.conn_acks),
2583 &(src_in_l->src.in.acks_pending));
2584
2585 spin_unlock_bh(&(cm->nb->cmsg_lock));
2586
2587 cor_enqueue_control_msg(cm, src);
2588 }
2589
2590 static int cor_try_update_ackconn_seqno(struct cor_conn *src_in_l)
2591 {
2592 int rc = 1;
2593
2594 spin_lock_bh(&(src_in_l->src.in.nb->cmsg_lock));
2595
2596 if (list_empty(&(src_in_l->src.in.acks_pending)) == 0) {
2597 struct cor_control_msg_out *cm = container_of(
2598 src_in_l->src.in.acks_pending.next,
2599 struct cor_control_msg_out,
2600 msg.ack_conn.conn_acks);
2601 BUG_ON(cm->nb != src_in_l->src.in.nb);
2602 BUG_ON(cm->type != MSGTYPE_ACK_CONN);
2603 BUG_ON(cm->msg.ack_conn.src_in != src_in_l);
2604 BUG_ON(cm->msg.ack_conn.conn_id != cor_get_connid_reverse(
2605 src_in_l->src.in.conn_id));
2606
2607 cm->msg.ack_conn.flags = (cm->msg.ack_conn.flags |
2608 KP_ACK_CONN_FLAGS_SEQNO |
2609 KP_ACK_CONN_FLAGS_WINDOW);
2610 cm->msg.ack_conn.seqno = src_in_l->src.in.next_seqno;
2611
2612 src_in_l->src.in.ack_seqno++;
2613 cm->msg.ack_conn.ack_seqno = src_in_l->src.in.ack_seqno;
2614
2615 cor_remove_connack_oooflag_ifold(src_in_l, cm);
2616 cor_recalc_scheduled_ackconn_size(cm);
2617
2618 cor_try_merge_ackconns(src_in_l, cm);
2619
2620 rc = 0;
2621 }
2622
2623 spin_unlock_bh(&(src_in_l->src.in.nb->cmsg_lock));
2624
2625 return rc;
2626 }
2627
2628 void cor_send_ack_conn_ifneeded(struct cor_conn *src_in_l, __u64 seqno_ooo,
2629 __u32 ooo_length)
2630 {
2631 struct cor_control_msg_out *cm;
2632
2633 BUG_ON(src_in_l->sourcetype != SOURCE_IN);
2634
2635 BUG_ON(ooo_length > 0 && cor_seqno_before_eq(seqno_ooo,
2636 src_in_l->src.in.next_seqno));
2637
2638 cor_update_windowlimit(src_in_l);
2639
2640 if (ooo_length != 0) {
2641 cm = cor_alloc_control_msg(src_in_l->src.in.nb,
2642 ACM_PRIORITY_LOW);
2643 if (cm != 0)
2644 goto add;
2645 }
2646
2647 if (src_in_l->src.in.inorder_ack_needed != 0)
2648 goto ack_needed;
2649
2650 if (cor_seqno_clean(src_in_l->src.in.window_seqnolimit -
2651 src_in_l->src.in.next_seqno) < WINDOW_ENCODE_MIN)
2652 return;
2653
2654 if (cor_seqno_clean(src_in_l->src.in.window_seqnolimit_remote -
2655 src_in_l->src.in.next_seqno) >= WINDOW_ENCODE_MIN &&
2656 cor_seqno_clean(src_in_l->src.in.window_seqnolimit -
2657 src_in_l->src.in.next_seqno) * 7 <
2658 cor_seqno_clean(
2659 src_in_l->src.in.window_seqnolimit_remote -
2660 src_in_l->src.in.next_seqno) * 8)
2661 return;
2662
2663 ack_needed:
2664 if (cor_try_update_ackconn_seqno(src_in_l) == 0)
2665 goto out;
2666
2667 cm = cor_alloc_control_msg(src_in_l->src.in.nb, ACM_PRIORITY_MED);
2668 if (cm == 0) {
2669 printk(KERN_ERR "error allocating inorder ack\n");
2670 return;
2671 }
2672
2673 add:
2674 cm->type = MSGTYPE_ACK_CONN;
2675 src_in_l->src.in.ack_seqno++;
2676 cm->msg.ack_conn.ack_seqno = src_in_l->src.in.ack_seqno;
2677 cor_conn_kref_get(src_in_l, "cor_control_msg_out ack_conn");
2678 cm->msg.ack_conn.src_in = src_in_l;
2679 cm->msg.ack_conn.conn_id =
2680 cor_get_connid_reverse(src_in_l->src.in.conn_id);
2681 cm->msg.ack_conn.seqno = src_in_l->src.in.next_seqno;
2682 cm->msg.ack_conn.seqno_ooo = seqno_ooo;
2683 cm->msg.ack_conn.length = ooo_length;
2684 cm->msg.ack_conn.bufsize_changerate =
2685 _cor_bufsize_update_get_changerate(src_in_l);
2686 cm->msg.ack_conn.flags = KP_ACK_CONN_FLAGS_SEQNO |
2687 KP_ACK_CONN_FLAGS_WINDOW;
2688 cor_set_ooolen_flags(cm);
2689 cm->msg.ack_conn.is_highlatency = src_in_l->is_highlatency;
2690 cm->msg.ack_conn.queue = get_queue_for_ackconn(src_in_l);
2691 cm->length = 5 + cor_ack_conn_len(cm->msg.ack_conn.flags);
2692
2693 cor_merge_or_enqueue_ackconn(src_in_l, cm, ADDCMSG_SRC_NEW);
2694
2695 out:
2696 src_in_l->src.in.inorder_ack_needed = 0;
2697 src_in_l->src.in.window_seqnolimit_remote =
2698 src_in_l->src.in.window_seqnolimit;
2699 }
2700
2701 static int cor_try_add_priority(struct cor_conn *trgt_out_ll, __u16 priority)
2702 {
2703 int rc = 1;
2704 struct cor_conn *src_in_ll = cor_get_conn_reversedir(trgt_out_ll);
2705
2706 spin_lock_bh(&(trgt_out_ll->trgt.out.nb->cmsg_lock));
2707
2708 if (list_empty(&(src_in_ll->src.in.acks_pending)) == 0) {
2709 struct cor_control_msg_out *cm = container_of(
2710 src_in_ll->src.in.acks_pending.next,
2711 struct cor_control_msg_out,
2712 msg.ack_conn.conn_acks);
2713 BUG_ON(cm->nb != trgt_out_ll->trgt.out.nb);
2714 BUG_ON(cm->type != MSGTYPE_ACK_CONN);
2715 BUG_ON(cm->msg.ack_conn.src_in != src_in_ll);
2716 BUG_ON(cm->msg.ack_conn.conn_id !=
2717 trgt_out_ll->trgt.out.conn_id);
2718
2719 BUG_ON((cm->msg.ack_conn.flags & KP_ACK_CONN_FLAGS_PRIORITY) !=
2720 0);
2721 cm->msg.ack_conn.flags = (cm->msg.ack_conn.flags |
2722 KP_ACK_CONN_FLAGS_PRIORITY);
2723 cm->msg.ack_conn.priority_seqno =
2724 trgt_out_ll->trgt.out.priority_seqno;
2725 cm->msg.ack_conn.priority = priority;
2726 cor_recalc_scheduled_ackconn_size(cm);
2727
2728 rc = 0;
2729 }
2730
2731 spin_unlock_bh(&(trgt_out_ll->trgt.out.nb->cmsg_lock));
2732
2733 return rc;
2734 }
2735
2736 void cor_send_priority(struct cor_conn *trgt_out_ll, __u16 priority)
2737 {
2738 struct cor_conn *src_in_ll = cor_get_conn_reversedir(trgt_out_ll);
2739 struct cor_control_msg_out *cm;
2740
2741 if (cor_try_add_priority(trgt_out_ll, priority) == 0)
2742 goto out;
2743
2744 cm = cor_alloc_control_msg(trgt_out_ll->trgt.out.nb, ACM_PRIORITY_LOW);
2745 if (cm == 0)
2746 return;
2747
2748 cm->type = MSGTYPE_ACK_CONN;
2749 cm->msg.ack_conn.flags = KP_ACK_CONN_FLAGS_PRIORITY;
2750 cor_conn_kref_get(src_in_ll, "cor_control_msg_out ack_conn");
2751 BUG_ON(trgt_out_ll->targettype != TARGET_OUT);
2752 cm->msg.ack_conn.src_in = src_in_ll;
2753 cm->msg.ack_conn.conn_id = trgt_out_ll->trgt.out.conn_id;
2754 cm->msg.ack_conn.bufsize_changerate =
2755 _cor_bufsize_update_get_changerate(src_in_ll);
2756 cm->msg.ack_conn.priority_seqno = trgt_out_ll->trgt.out.priority_seqno;
2757 cm->msg.ack_conn.priority = priority;
2758 cm->msg.ack_conn.is_highlatency = trgt_out_ll->is_highlatency;
2759 cm->msg.ack_conn.queue = get_queue_for_ackconn(src_in_ll);
2760
2761 cm->length = 5 + cor_ack_conn_len(cm->msg.ack_conn.flags);
2762 cor_merge_or_enqueue_ackconn(src_in_ll, cm, ADDCMSG_SRC_NEW);
2763
2764 out:
2765 trgt_out_ll->trgt.out.priority_last = priority;
2766 trgt_out_ll->trgt.out.priority_seqno =
2767 (trgt_out_ll->trgt.out.priority_seqno + 1) & 15;
2768 trgt_out_ll->trgt.out.priority_send_allowed = 0;
2769 }
2770
2771 void cor_free_ack_conns(struct cor_conn *src_in_lx)
2772 {
2773 int changed = 0;
2774 spin_lock_bh(&(src_in_lx->src.in.nb->cmsg_lock));
2775 while (list_empty(&(src_in_lx->src.in.acks_pending)) == 0) {
2776 struct list_head *currlh =
2777 src_in_lx->src.in.acks_pending.next;
2778 struct cor_control_msg_out *currcm = container_of(currlh,
2779 struct cor_control_msg_out,
2780 msg.ack_conn.conn_acks);
2781
2782 cor_remove_pending_ackconn(currcm);
2783 changed = 1;
2784 }
2785 if (changed)
2786 cor_schedule_controlmsg_timer(src_in_lx->src.in.nb);
2787 spin_unlock_bh(&(src_in_lx->src.in.nb->cmsg_lock));
2788 }
2789
2790 void cor_send_connect_success(struct cor_control_msg_out *cm, __u32 conn_id,
2791 struct cor_conn *src_in)
2792 {
2793 cm->type = MSGTYPE_CONNECT_SUCCESS;
2794 cm->msg.connect_success.conn_id = conn_id;
2795 cor_conn_kref_get(src_in, "cor_control_msg_out connect_success");
2796 cm->msg.connect_success.src_in = src_in;
2797 cm->length = 7;
2798 cor_enqueue_control_msg(cm, ADDCMSG_SRC_NEW);
2799 }
2800
2801 void cor_send_connect_nb(struct cor_control_msg_out *cm, __u32 conn_id,
2802 __u64 seqno1, __u64 seqno2, struct cor_conn *src_in_ll)
2803 {
2804 cm->type = MSGTYPE_CONNECT;
2805 cm->msg.connect.conn_id = conn_id;
2806 cm->msg.connect.seqno1 = seqno1;
2807 cm->msg.connect.seqno2 = seqno2;
2808 cor_conn_kref_get(src_in_ll, "cor_control_msg_out connect");
2809 BUG_ON(src_in_ll->sourcetype != SOURCE_IN);
2810 cm->msg.connect.src_in = src_in_ll;
2811 cm->length = 22;
2812 cor_enqueue_control_msg(cm, ADDCMSG_SRC_NEW);
2813 }
2814
2815 void cor_send_conndata(struct cor_control_msg_out *cm, __u32 conn_id,
2816 __u64 seqno, char *data_orig, char *data, __u32 datalen,
2817 __u8 windowused, __u8 flush, __u8 highlatency,
2818 struct cor_conn_retrans *cr)
2819 {
2820 cm->type = MSGTYPE_CONNDATA;
2821 cm->msg.conn_data.conn_id = conn_id;
2822 cm->msg.conn_data.seqno = seqno;
2823 cm->msg.conn_data.data_orig = data_orig;
2824 cm->msg.conn_data.data = data;
2825 cm->msg.conn_data.datalen = datalen;
2826 cm->msg.conn_data.windowused = windowused;
2827 cm->msg.conn_data.flush = flush;
2828 cm->msg.conn_data.highlatency = highlatency;
2829 cm->msg.conn_data.cr = cr;
2830 kref_get(&(cr->ref));
2831 cm->length = get_kp_conn_data_length(datalen);
2832 cor_enqueue_control_msg(cm, ADDCMSG_SRC_NEW);
2833 }
2834
2835 int cor_send_reset_conn(struct cor_neighbor *nb, __u32 conn_id, int lowprio)
2836 {
2837 struct cor_control_msg_out *cm;
2838
2839 if (unlikely(cor_get_neigh_state(nb) == NEIGHBOR_STATE_KILLED))
2840 return 0;
2841
2842 cm = cor_alloc_control_msg(nb, lowprio ?
2843 ACM_PRIORITY_LOW : ACM_PRIORITY_MED);
2844
2845 if (unlikely(cm == 0))
2846 return 1;
2847
2848 cm->type = MSGTYPE_RESET_CONN;
2849 cm->msg.reset_conn.conn_id = conn_id;
2850 cm->length = 5;
2851
2852 cor_enqueue_control_msg(cm, ADDCMSG_SRC_NEW);
2853
2854 return 0;
2855 }
2856
2857 int __init cor_kgen_init(void)
2858 {
2859 cor_controlmsg_slab = kmem_cache_create("cor_controlmsg",
2860 sizeof(struct cor_control_msg_out), 8, 0, 0);
2861 if (unlikely(cor_controlmsg_slab == 0))
2862 return -ENOMEM;
2863
2864 cor_controlretrans_slab = kmem_cache_create("cor_controlretransmsg",
2865 sizeof(struct cor_control_retrans), 8, 0, 0);
2866 if (unlikely(cor_controlretrans_slab == 0))
2867 return -ENOMEM;
2868
2869 return 0;
2870 }
2871
2872 void __exit cor_kgen_exit2(void)
2873 {
2874 kmem_cache_destroy(cor_controlretrans_slab);
2875 cor_controlretrans_slab = 0;
2876
2877 kmem_cache_destroy(cor_controlmsg_slab);
2878 cor_controlmsg_slab = 0;
2879 }
2880
2881 MODULE_LICENSE("GPL");
connection oriented routing