2 * Connection oriented routing
3 * Copyright (C) 2007-2020 Michael Blizek
5 * This program is free software; you can redistribute it and/or
6 * modify it under the terms of the GNU General Public License
7 * as published by the Free Software Foundation; either version 2
8 * of the License, or (at your option) any later version.
10 * This program is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
15 * You should have received a copy of the GNU General Public License
16 * along with this program; if not, write to the Free Software
17 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
21 #include <linux/gfp.h>
22 #include <linux/jiffies.h>
23 #include <linux/slab.h>
27 static struct kmem_cache
*connretrans_slab
;
29 static DEFINE_SPINLOCK(queues_lock
);
30 static LIST_HEAD(queues
);
32 static int _flush_out(struct conn
*trgt_out_lx
, __u32 maxsend
, __u32
*sent
,
33 int from_qos
, int maxsend_forcedelay
);
35 static void _qos_enqueue(struct qos_queue
*q
, struct resume_block
*rb
,
36 int caller
, int from_nbcongwin_resume
);
39 #ifdef DEBUG_QOS_SLOWSEND
40 static DEFINE_SPINLOCK(slowsend_lock
);
41 static unsigned long last_send
;
44 int _cor_dev_queue_xmit(struct sk_buff
*skb
, int caller
)
47 unsigned long jiffies_tmp
;
48 spin_lock_bh(&slowsend_lock
);
49 jiffies_tmp
= jiffies
;
50 if (last_send
!= jiffies_tmp
) {
51 if (last_send
+ 1 == jiffies_tmp
) {
52 last_send
= jiffies_tmp
;
54 last_send
= jiffies_tmp
- 1;
58 spin_unlock_bh(&slowsend_lock
);
60 /* printk(KERN_ERR "cor_dev_queue_xmit %d, %d", caller, allowsend); */
62 return dev_queue_xmit(skb
);
70 static void free_connretrans(struct kref
*ref
)
72 struct conn_retrans
*cr
= container_of(ref
, struct conn_retrans
, ref
);
73 struct conn
*cn
= cr
->trgt_out_o
;
75 BUG_ON(cr
->state
!= CONN_RETRANS_ACKED
);
77 kmem_cache_free(connretrans_slab
, cr
);
78 kref_put(&(cn
->ref
), free_conn
);
81 void free_qos(struct kref
*ref
)
83 struct qos_queue
*q
= container_of(ref
, struct qos_queue
, ref
);
88 static void qos_queue_set_congstatus(struct qos_queue
*q_locked
);
91 * neighbor congestion window:
92 * increment by 4096 every round trip if more that 2/3 of cwin is used
94 * in case of packet loss decrease by 1/4:
95 * - <= 1/8 immediately and
96 * - <= 1/4 during the next round trip
98 * in case of multiple packet loss events, do not decrement more than once per
104 /*extern __u64 get_bufspace_used(void);
106 static void print_conn_bufstats(struct neighbor *nb)
108 / * not threadsafe, but this is only for debugging... * /
110 __u64 read_remaining = 0;
112 struct list_head *lh;
113 unsigned long iflags;
115 spin_lock_irqsave(&(nb->conns_waiting.lock), iflags);
117 lh = nb->conns_waiting.lh.next;
118 while (lh != &(nb->conns_waiting.lh)) {
119 struct conn *cn = container_of(lh, struct conn,
121 totalsize += cn->data_buf.datasize;
122 read_remaining += cn->data_buf.read_remaining;
126 lh = nb->conns_waiting.lh_nextpass.next;
127 while (lh != &(nb->conns_waiting.lh_nextpass)) {
128 struct conn *cn = container_of(lh, struct conn,
130 totalsize += cn->data_buf.datasize;
131 read_remaining += cn->data_buf.read_remaining;
135 numconns = nb->conns_waiting.cnt;
137 spin_unlock_irqrestore(&(nb->conns_waiting.lock), iflags);
139 printk(KERN_ERR "conn %llu %llu %u", totalsize, read_remaining, numconns);
142 static void nbcongwin_data_retransmitted(struct neighbor
*nb
, __u64 bytes_sent
)
144 __u64 min_cwin
= mss_conndata(nb
, 0)*2 << NBCONGWIN_SHIFT
;
147 unsigned long iflags
;
149 spin_lock_irqsave(&(nb
->nbcongwin
.lock
), iflags
);
151 cwin
= atomic64_read(&(nb
->nbcongwin
.cwin
));
153 /* printk(KERN_ERR "retrans %llu %llu", cwin >> NBCONGWIN_SHIFT,
154 get_bufspace_used());
155 print_conn_bufstats(nb); */
157 BUG_ON(nb
->nbcongwin
.cwin_shrinkto
> cwin
);
158 BUG_ON(cwin
>= U64_MAX
/1024);
160 if (bytes_sent
> 1024)
163 if (nb
->nbcongwin
.cwin_shrinkto
== cwin
) {
164 if (bytes_sent
> 512) {
167 cwin
-= (bytes_sent
* cwin
) / (1024 * 8);
171 atomic64_set(&(nb
->nbcongwin
.cwin
), cwin
);
174 nb
->nbcongwin
.cwin_shrinkto
-=
175 (bytes_sent
* nb
->nbcongwin
.cwin_shrinkto
) / (1024 * 8);
177 nb
->nbcongwin
.cwin_shrinkto
= max(nb
->nbcongwin
.cwin_shrinkto
,
180 if (nb
->nbcongwin
.cwin_shrinkto
< min_cwin
)
181 nb
->nbcongwin
.cwin_shrinkto
= min_cwin
;
183 spin_unlock_irqrestore(&(nb
->nbcongwin
.lock
), iflags
);
186 static __u64
nbcongwin_update_cwin(struct neighbor
*nb_cwlocked
,
187 __u64 data_intransit
, __u64 bytes_acked
)
189 __u64 CWIN_MUL
= (1 << NBCONGWIN_SHIFT
);
190 __u32 INCR_PER_RTT
= 4096;
192 __u64 cwin
= atomic64_read(&(nb_cwlocked
->nbcongwin
.cwin
));
197 if (nb_cwlocked
->nbcongwin
.cwin_shrinkto
< cwin
) {
198 __u64 shrinkby
= (bytes_acked
<< (NBCONGWIN_SHIFT
-2));
199 if (unlikely(shrinkby
> cwin
))
204 if (cwin
< nb_cwlocked
->nbcongwin
.cwin_shrinkto
)
205 cwin
= nb_cwlocked
->nbcongwin
.cwin_shrinkto
;
209 if (cwin
* 2 > data_intransit
* CWIN_MUL
* 3)
212 cwin_tmp
= max(cwin
, bytes_acked
<< NBCONGWIN_SHIFT
);
214 if (unlikely(bytes_acked
>= U64_MAX
/INCR_PER_RTT
/CWIN_MUL
))
215 incrby
= div64_u64(bytes_acked
* INCR_PER_RTT
,
216 cwin_tmp
/ CWIN_MUL
/ CWIN_MUL
);
217 else if (unlikely(bytes_acked
>=
218 U64_MAX
/INCR_PER_RTT
/CWIN_MUL
/CWIN_MUL
))
219 incrby
= div64_u64(bytes_acked
* INCR_PER_RTT
* CWIN_MUL
,
220 cwin_tmp
/ CWIN_MUL
);
222 incrby
= div64_u64(bytes_acked
* INCR_PER_RTT
* CWIN_MUL
*
225 BUG_ON(incrby
> INCR_PER_RTT
* CWIN_MUL
);
227 if (unlikely(cwin
+ incrby
< cwin
))
232 if (unlikely(nb_cwlocked
->nbcongwin
.cwin_shrinkto
+ incrby
<
233 nb_cwlocked
->nbcongwin
.cwin_shrinkto
))
234 nb_cwlocked
->nbcongwin
.cwin_shrinkto
= U64_MAX
;
236 nb_cwlocked
->nbcongwin
.cwin_shrinkto
+= incrby
;
239 atomic64_set(&(nb_cwlocked
->nbcongwin
.cwin
), cwin
);
244 void nbcongwin_data_acked(struct neighbor
*nb
, __u64 bytes_acked
)
246 unsigned long iflags
;
247 struct qos_queue
*q
= nb
->queue
;
248 __u64 data_intransit
;
251 spin_lock_irqsave(&(nb
->nbcongwin
.lock
), iflags
);
253 data_intransit
= atomic64_read(&(nb
->nbcongwin
.data_intransit
));
255 cwin
= nbcongwin_update_cwin(nb
, data_intransit
, bytes_acked
);
257 BUG_ON(bytes_acked
> data_intransit
);
258 atomic64_sub(bytes_acked
, &(nb
->nbcongwin
.data_intransit
));
259 data_intransit
-= bytes_acked
;
261 if (data_intransit
>= cwin
>> NBCONGWIN_SHIFT
)
264 spin_lock(&(q
->qlock
));
265 if (nb
->rb
.in_queue
== RB_INQUEUE_NBCONGWIN
) {
266 if (nb
->conns_waiting
.cnt
== 0) {
267 nb
->rb
.in_queue
= RB_INQUEUE_FALSE
;
269 _qos_enqueue(q
, &(nb
->rb
), QOS_CALLER_NEIGHBOR
, 1);
272 spin_unlock(&(q
->qlock
));
276 spin_unlock_irqrestore(&(nb
->nbcongwin
.lock
), iflags
);
279 static void nbcongwin_data_sent(struct neighbor
*nb
, __u32 bytes_sent
)
281 atomic64_add(bytes_sent
, &(nb
->nbcongwin
.data_intransit
));
284 static int nbcongwin_send_allowed(struct neighbor
*nb
)
286 unsigned long iflags
;
288 struct qos_queue
*q
= nb
->queue
;
289 int krefput_queue
= 0;
291 if (atomic64_read(&(nb
->nbcongwin
.data_intransit
)) <=
292 atomic64_read(&(nb
->nbcongwin
.cwin
)) >> NBCONGWIN_SHIFT
)
295 spin_lock_irqsave(&(nb
->nbcongwin
.lock
), iflags
);
297 if (atomic64_read(&(nb
->nbcongwin
.data_intransit
)) <=
298 atomic64_read(&(nb
->nbcongwin
.cwin
)) >> NBCONGWIN_SHIFT
)
303 spin_lock(&(q
->qlock
));
304 if (nb
->rb
.in_queue
== RB_INQUEUE_FALSE
) {
305 nb
->rb
.in_queue
= RB_INQUEUE_NBCONGWIN
;
306 } else if (nb
->rb
.in_queue
== RB_INQUEUE_TRUE
) {
307 list_del(&(nb
->rb
.lh
));
308 kref_put(&(nb
->ref
), kreffree_bug
);
309 nb
->rb
.in_queue
= RB_INQUEUE_NBCONGWIN
;
310 BUG_ON(q
->numconns
< nb
->conns_waiting
.cnt
);
311 q
->numconns
-= nb
->conns_waiting
.cnt
;
312 q
->priority_sum
-= nb
->conns_waiting
.priority_sum
;
315 qos_queue_set_congstatus(q
);
316 } else if (nb
->rb
.in_queue
== RB_INQUEUE_NBCONGWIN
) {
320 spin_unlock(&(q
->qlock
));
322 if (krefput_queue
!= 0)
323 kref_put(&(q
->ref
), free_qos
);
326 spin_unlock_irqrestore(&(nb
->nbcongwin
.lock
), iflags
);
333 static inline void nbcongwin_data_retransmitted(struct neighbor
*nb
,
338 static inline void nbcongwin_data_acked(struct neighbor
*nb
, __u64 bytes_acked
)
342 static inline void nbcongwin_data_sent(struct neighbor
*nb
, __u32 bytes_sent
)
346 static inline int nbcongwin_send_allowed(struct neighbor
*nb
)
353 static __u64
_resume_conns_maxsend(struct qos_queue
*q
, struct conn
*trgt_out_l
,
354 __u32 newpriority
, int *maxsend_forcedelay
)
356 unsigned long iflags
;
358 struct neighbor
*nb
= trgt_out_l
->target
.out
.nb
;
359 __u32 oldpriority
= trgt_out_l
->target
.out
.rb_priority
;
363 spin_lock_irqsave(&(nb
->conns_waiting
.lock
), iflags
);
364 spin_lock(&(q
->qlock
));
366 BUG_ON(nb
->conns_waiting
.priority_sum
< oldpriority
);
367 BUG_ON(q
->priority_sum
< oldpriority
);
368 nb
->conns_waiting
.priority_sum
-= oldpriority
;
369 q
->priority_sum
-= oldpriority
;
371 BUG_ON(nb
->conns_waiting
.priority_sum
+ newpriority
<
372 nb
->conns_waiting
.priority_sum
);
373 BUG_ON(q
->priority_sum
+ newpriority
< q
->priority_sum
);
374 nb
->conns_waiting
.priority_sum
+= newpriority
;
375 q
->priority_sum
+= newpriority
;
377 priority_sum
= q
->priority_sum
;
378 numconns
= q
->numconns
;
380 spin_unlock(&(q
->qlock
));
381 spin_unlock_irqrestore(&(nb
->conns_waiting
.lock
), iflags
);
383 trgt_out_l
->target
.out
.rb_priority
= newpriority
;
386 *maxsend_forcedelay
= 1;
387 return div_u64(2048LL * ((__u64
) newpriority
) *
388 ((__u64
) numconns
), priority_sum
);
390 *maxsend_forcedelay
= 0;
391 return div_u64(1024LL * ((__u64
) newpriority
) *
392 ((__u64
) numconns
), priority_sum
);
396 static int _resume_neighbors_nextpass(struct neighbor
*nb_waitingconnslocked
)
398 BUG_ON(list_empty(&(nb_waitingconnslocked
->conns_waiting
.lh
)) == 0);
400 if (list_empty(&(nb_waitingconnslocked
->conns_waiting
.lh_nextpass
))) {
401 BUG_ON(nb_waitingconnslocked
->conns_waiting
.cnt
!= 0);
405 BUG_ON(nb_waitingconnslocked
->conns_waiting
.cnt
== 0);
407 nb_waitingconnslocked
->conns_waiting
.lh
.next
=
408 nb_waitingconnslocked
->conns_waiting
.lh_nextpass
.next
;
409 nb_waitingconnslocked
->conns_waiting
.lh
.prev
=
410 nb_waitingconnslocked
->conns_waiting
.lh_nextpass
.prev
;
411 nb_waitingconnslocked
->conns_waiting
.lh
.next
->prev
=
412 &(nb_waitingconnslocked
->conns_waiting
.lh
);
413 nb_waitingconnslocked
->conns_waiting
.lh
.prev
->next
=
414 &(nb_waitingconnslocked
->conns_waiting
.lh
);
415 nb_waitingconnslocked
->conns_waiting
.lh_nextpass
.next
=
416 &(nb_waitingconnslocked
->conns_waiting
.lh_nextpass
);
417 nb_waitingconnslocked
->conns_waiting
.lh_nextpass
.prev
=
418 &(nb_waitingconnslocked
->conns_waiting
.lh_nextpass
);
423 static int _resume_neighbors(struct qos_queue
*q
, struct neighbor
*nb
,
426 unsigned long iflags
;
431 int maxsend_forcedelay
;
437 spin_lock_irqsave(&(nb
->conns_waiting
.lock
), iflags
);
438 if (list_empty(&(nb
->conns_waiting
.lh
)) != 0) {
439 int done
= _resume_neighbors_nextpass(nb
);
440 spin_unlock_irqrestore(&(nb
->conns_waiting
.lock
),
442 return done
? QOS_RESUME_DONE
: QOS_RESUME_NEXTNEIGHBOR
;
444 BUG_ON(nb
->conns_waiting
.cnt
== 0);
446 cn
= container_of(nb
->conns_waiting
.lh
.next
, struct conn
,
448 BUG_ON(cn
->targettype
!= TARGET_OUT
);
449 BUG_ON(cn
->target
.out
.rb
.lh
.prev
!= &(nb
->conns_waiting
.lh
));
450 BUG_ON((cn
->target
.out
.rb
.lh
.next
== &(nb
->conns_waiting
.lh
)) &&
451 (nb
->conns_waiting
.lh
.prev
!=
452 &(cn
->target
.out
.rb
.lh
)));
453 list_del(&(cn
->target
.out
.rb
.lh
));
454 list_add_tail(&(cn
->target
.out
.rb
.lh
),
455 &(nb
->conns_waiting
.lh_nextpass
));
456 kref_get(&(cn
->ref
));
457 spin_unlock_irqrestore(&(nb
->conns_waiting
.lock
), iflags
);
460 priority
= refresh_conn_priority(cn
, 0);
462 spin_lock_bh(&(cn
->rcv_lock
));
464 if (unlikely(cn
->targettype
!= TARGET_OUT
)) {
465 spin_unlock_bh(&(cn
->rcv_lock
));
469 maxsend
= _resume_conns_maxsend(q
, cn
, priority
,
470 &maxsend_forcedelay
);
471 if (cn
->target
.out
.maxsend_extra
>= maxsend
)
472 maxsend_forcedelay
= 0;
473 maxsend
+= cn
->target
.out
.maxsend_extra
;
474 if (unlikely(maxsend
> U32_MAX
))
476 if (unlikely(maxsend
>= 65536))
477 maxsend_forcedelay
= 0;
479 rc2
= _flush_out(cn
, maxsend
, &sent2
, 1, maxsend_forcedelay
);
481 if (rc2
== RC_FLUSH_CONN_OUT_OK
||
482 rc2
== RC_FLUSH_CONN_OUT_NBNOTACTIVE
) {
483 cn
->target
.out
.maxsend_extra
= 0;
485 } else if (sent2
== 0 && (rc2
== RC_FLUSH_CONN_OUT_CONG
||
486 rc2
== RC_FLUSH_CONN_OUT_OOM
)) {
487 spin_lock_irqsave(&(nb
->conns_waiting
.lock
), iflags
);
488 if (likely(cn
->target
.out
.rb
.in_queue
!=
490 list_del(&(cn
->target
.out
.rb
.lh
));
491 list_add(&(cn
->target
.out
.rb
.lh
),
492 &(nb
->conns_waiting
.lh
));
494 spin_unlock_irqrestore(&(nb
->conns_waiting
.lock
),
496 } else if (rc2
== RC_FLUSH_CONN_OUT_CONG
||
497 rc2
== RC_FLUSH_CONN_OUT_OOM
) {
498 cn
->target
.out
.maxsend_extra
= 0;
499 } else if (likely(rc2
== RC_FLUSH_CONN_OUT_MAXSENT
)) {
500 if (unlikely(maxsend
- sent2
> 65535))
501 cn
->target
.out
.maxsend_extra
= 65535;
503 cn
->target
.out
.maxsend_extra
= maxsend
- sent2
;
506 spin_unlock_bh(&(cn
->rcv_lock
));
513 kref_put(&(cn
->ref
), free_conn
);
515 if (rc2
== RC_FLUSH_CONN_OUT_CONG
||
516 rc2
== RC_FLUSH_CONN_OUT_OOM
) {
517 return QOS_RESUME_CONG
;
522 static int resume_neighbors(struct qos_queue
*q
, int *sent
)
524 unsigned long iflags
;
526 spin_lock_irqsave(&(q
->qlock
), iflags
);
532 if (list_empty(&(q
->neighbors_waiting
)) != 0) {
533 BUG_ON(q
->numconns
!= 0);
534 spin_unlock_irqrestore(&(q
->qlock
), iflags
);
535 return QOS_RESUME_DONE
;
537 BUG_ON(q
->numconns
== 0);
539 nb
= container_of(q
->neighbors_waiting
.next
, struct neighbor
,
542 BUG_ON(nb
->rb
.in_queue
!= RB_INQUEUE_TRUE
);
543 BUG_ON(nb
->rb
.lh
.prev
!= &(q
->neighbors_waiting
));
544 BUG_ON((nb
->rb
.lh
.next
== &(q
->neighbors_waiting
)) &&
545 (q
->neighbors_waiting
.prev
!= &(nb
->rb
.lh
)));
547 kref_get(&(nb
->ref
));
549 spin_unlock_irqrestore(&(q
->qlock
), iflags
);
551 atomic_set(&(nb
->cmsg_delay_conndata
), 1);
553 rc
= _resume_neighbors(q
, nb
, sent
);
554 if (rc
== QOS_RESUME_CONG
) {
555 kref_put(&(nb
->ref
), neighbor_free
);
556 return QOS_RESUME_CONG
;
559 atomic_set(&(nb
->cmsg_delay_conndata
), 0);
560 spin_lock_bh(&(nb
->cmsg_lock
));
561 schedule_controlmsg_timer(nb
);
562 spin_unlock_bh(&(nb
->cmsg_lock
));
564 spin_lock_irqsave(&(q
->qlock
), iflags
);
565 if (rc
== QOS_RESUME_DONE
) {
566 if (nb
->conns_waiting
.cnt
== 0 &&
567 nb
->rb
.in_queue
== RB_INQUEUE_TRUE
) {
568 nb
->rb
.in_queue
= RB_INQUEUE_FALSE
;
569 list_del(&(nb
->rb
.lh
));
570 kref_put(&(nb
->ref
), kreffree_bug
);
572 } else if (rc
== QOS_RESUME_NEXTNEIGHBOR
) {
573 if (nb
->rb
.in_queue
== RB_INQUEUE_TRUE
) {
574 list_del(&(nb
->rb
.lh
));
575 list_add_tail(&(nb
->rb
.lh
),
576 &(q
->neighbors_waiting
));
582 kref_put(&(nb
->ref
), neighbor_free
);
584 if (rc
== QOS_RESUME_NEXTNEIGHBOR
) {
585 spin_unlock_irqrestore(&(q
->qlock
), iflags
);
586 return QOS_RESUME_NEXTNEIGHBOR
;
591 static int send_retrans(struct neighbor
*nb
, int *sent
);
593 static int _qos_resume(struct qos_queue
*q
, int caller
, int *sent
)
595 unsigned long iflags
;
596 int rc
= QOS_RESUME_DONE
;
597 struct list_head
*lh
;
599 spin_lock_irqsave(&(q
->qlock
), iflags
);
601 if (caller
== QOS_CALLER_KPACKET
)
602 lh
= &(q
->kpackets_waiting
);
603 else if (caller
== QOS_CALLER_CONN_RETRANS
)
604 lh
= &(q
->conn_retrans_waiting
);
605 else if (caller
== QOS_CALLER_ANNOUNCE
)
606 lh
= &(q
->announce_waiting
);
610 while (list_empty(lh
) == 0) {
611 struct resume_block
*rb
= container_of(lh
->next
,
612 struct resume_block
, lh
);
613 BUG_ON(rb
->in_queue
!= RB_INQUEUE_TRUE
);
614 rb
->in_queue
= RB_INQUEUE_FALSE
;
617 spin_unlock_irqrestore(&(q
->qlock
), iflags
);
618 if (caller
== QOS_CALLER_KPACKET
) {
619 rc
= send_messages(container_of(rb
, struct neighbor
,
621 } else if (caller
== QOS_CALLER_CONN_RETRANS
) {
622 rc
= send_retrans(container_of(rb
, struct neighbor
,
624 } else if (caller
== QOS_CALLER_ANNOUNCE
) {
625 rc
= _send_announce(container_of(rb
,
626 struct announce_data
, rb
), 1, sent
);
630 spin_lock_irqsave(&(q
->qlock
), iflags
);
632 if (rc
!= QOS_RESUME_DONE
&& rb
->in_queue
== RB_INQUEUE_FALSE
) {
633 rb
->in_queue
= RB_INQUEUE_TRUE
;
634 list_add(&(rb
->lh
), lh
);
638 if (caller
== QOS_CALLER_KPACKET
) {
639 kref_put(&(container_of(rb
, struct neighbor
,
640 rb_kp
)->ref
), neighbor_free
);
641 } else if (caller
== QOS_CALLER_CONN_RETRANS
) {
642 kref_put(&(container_of(rb
, struct neighbor
,
643 rb_cr
)->ref
), neighbor_free
);
644 } else if (caller
== QOS_CALLER_ANNOUNCE
) {
645 kref_put(&(container_of(rb
,
646 struct announce_data
, rb
)->ref
),
652 kref_put(&(q
->ref
), kreffree_bug
);
655 spin_unlock_irqrestore(&(q
->qlock
), iflags
);
660 void qos_resume_taskfunc(unsigned long arg
)
662 struct qos_queue
*q
= (struct qos_queue
*) arg
;
666 unsigned long iflags
;
669 #warning todo limit runtime of resume task
671 spin_lock_irqsave(&(q
->qlock
), iflags
);
674 struct list_head
*lh
;
676 rc
= QOS_RESUME_DONE
;
678 if (i
== QOS_CALLER_KPACKET
)
679 lh
= &(q
->kpackets_waiting
);
680 else if (i
== QOS_CALLER_CONN_RETRANS
)
681 lh
= &(q
->conn_retrans_waiting
);
682 else if (i
== QOS_CALLER_ANNOUNCE
)
683 lh
= &(q
->announce_waiting
);
684 else if (i
== QOS_CALLER_NEIGHBOR
)
685 lh
= &(q
->neighbors_waiting
);
689 if (list_empty(lh
)) {
694 spin_unlock_irqrestore(&(q
->qlock
), iflags
);
695 if (i
== QOS_CALLER_NEIGHBOR
) {
696 rc
= resume_neighbors(q
, &sent
);
698 rc
= _qos_resume(q
, i
, &sent
);
701 spin_lock_irqsave(&(q
->qlock
), iflags
);
705 if (rc
!= QOS_RESUME_DONE
&& rc
!= QOS_RESUME_NEXTNEIGHBOR
)
709 if (rc
== QOS_RESUME_DONE
) {
710 BUG_ON(!list_empty(&(q
->kpackets_waiting
)));
711 BUG_ON(!list_empty(&(q
->conn_retrans_waiting
)));
712 BUG_ON(!list_empty(&(q
->announce_waiting
)));
713 BUG_ON(!list_empty(&(q
->neighbors_waiting
)));
715 q
->qos_resume_scheduled
= 0;
717 unsigned long jiffies_tmp
= jiffies
;
718 unsigned long delay
= (jiffies_tmp
- q
->jiffies_lastprogress
+
721 if (sent
|| unlikely(delay
<= 0)) {
722 q
->jiffies_lastprogress
= jiffies_tmp
;
724 } else if (delay
> HZ
/10) {
725 q
->jiffies_lastprogress
= jiffies_tmp
- (HZ
*4)/10;
729 /* If we retry too fast here, we might starve layer 2 */
730 if (mod_timer(&(q
->qos_resume_timer
), jiffies_tmp
+ delay
) ==
736 qos_queue_set_congstatus(q
);
738 spin_unlock_irqrestore(&(q
->qlock
), iflags
);
741 static inline int qos_queue_is_destroyed(struct qos_queue
*q_locked
)
743 return q_locked
->dev
== 0;
746 #warning todo kref (kref_put if tasklet is scheduled)
747 void qos_resume_timerfunc(struct timer_list
*qos_resume_timer
)
749 unsigned long iflags
;
750 struct qos_queue
*q
= container_of(qos_resume_timer
,
751 struct qos_queue
, qos_resume_timer
);
752 spin_lock_irqsave(&(q
->qlock
), iflags
);
753 if (likely(!qos_queue_is_destroyed(q
)))
754 tasklet_schedule(&(q
->qos_resume_task
));
755 spin_unlock_irqrestore(&(q
->qlock
), iflags
);
757 kref_put(&(q
->ref
), free_qos
);
760 struct qos_queue
*get_queue(struct net_device
*dev
)
762 struct qos_queue
*ret
= 0;
763 struct list_head
*curr
;
765 spin_lock_bh(&(queues_lock
));
767 while (curr
!= (&queues
)) {
768 struct qos_queue
*q
= container_of(curr
,
769 struct qos_queue
, queue_list
);
772 kref_get(&(ret
->ref
));
777 spin_unlock_bh(&(queues_lock
));
781 static void _destroy_queue(struct qos_queue
*q
, int caller
)
783 struct list_head
*lh
;
785 if (caller
== QOS_CALLER_KPACKET
)
786 lh
= &(q
->kpackets_waiting
);
787 else if (caller
== QOS_CALLER_CONN_RETRANS
)
788 lh
= &(q
->conn_retrans_waiting
);
789 else if (caller
== QOS_CALLER_ANNOUNCE
)
790 lh
= &(q
->announce_waiting
);
791 else if (caller
== QOS_CALLER_NEIGHBOR
)
792 lh
= &(q
->neighbors_waiting
);
796 while (list_empty(lh
) == 0) {
797 struct list_head
*curr
= lh
->next
;
798 struct resume_block
*rb
= container_of(curr
,
799 struct resume_block
, lh
);
800 BUG_ON(rb
->in_queue
!= RB_INQUEUE_TRUE
);
801 rb
->in_queue
= RB_INQUEUE_FALSE
;
804 if (caller
== QOS_CALLER_KPACKET
) {
805 kref_put(&(container_of(rb
, struct neighbor
,
806 rb_kp
)->ref
), neighbor_free
);
807 } else if (caller
== QOS_CALLER_CONN_RETRANS
) {
808 kref_put(&(container_of(rb
, struct neighbor
,
809 rb_cr
)->ref
), neighbor_free
);
810 } else if (caller
== QOS_CALLER_ANNOUNCE
) {
811 kref_put(&(container_of(rb
,
812 struct announce_data
, rb
)->ref
),
814 } else if (caller
== QOS_CALLER_NEIGHBOR
) {
815 kref_put(&(container_of(rb
,
816 struct neighbor
, rb
)->ref
),
821 kref_put(&(q
->ref
), kreffree_bug
);
825 static struct qos_queue
*unlink_queue(struct net_device
*dev
)
827 struct qos_queue
*ret
= 0;
828 struct list_head
*curr
;
830 spin_lock_bh(&(queues_lock
));
832 while (curr
!= (&queues
)) {
833 struct qos_queue
*q
= container_of(curr
,
834 struct qos_queue
, queue_list
);
835 if (dev
== 0 || q
->dev
== dev
) {
837 kref_get(&(ret
->ref
));
839 list_del(&(q
->queue_list
));
840 kref_put(&(q
->ref
), kreffree_bug
);
845 spin_unlock_bh(&(queues_lock
));
849 int destroy_queue(struct net_device
*dev
)
852 unsigned long iflags
;
855 struct qos_queue
*q
= unlink_queue(dev
);
862 spin_lock_irqsave(&(q
->qlock
), iflags
);
867 _destroy_queue(q
, QOS_CALLER_KPACKET
);
868 _destroy_queue(q
, QOS_CALLER_CONN_RETRANS
);
869 _destroy_queue(q
, QOS_CALLER_ANNOUNCE
);
870 _destroy_queue(q
, QOS_CALLER_NEIGHBOR
);
871 spin_unlock_irqrestore(&(q
->qlock
), iflags
);
873 tasklet_kill(&(q
->qos_resume_task
));
875 kref_put(&(q
->ref
), free_qos
);
881 int create_queue(struct net_device
*dev
)
883 struct qos_queue
*q
= kmalloc(sizeof(struct qos_queue
), GFP_KERNEL
);
886 printk(KERN_ERR
"cor: unable to allocate memory for device "
887 "queue, not enabling device");
891 memset(q
, 0, sizeof(struct qos_queue
));
893 spin_lock_init(&(q
->qlock
));
895 kref_init(&(q
->ref
));
900 timer_setup(&(q
->qos_resume_timer
), qos_resume_timerfunc
, 0);
901 tasklet_init(&(q
->qos_resume_task
), qos_resume_taskfunc
,
904 INIT_LIST_HEAD(&(q
->kpackets_waiting
));
905 INIT_LIST_HEAD(&(q
->conn_retrans_waiting
));
906 INIT_LIST_HEAD(&(q
->announce_waiting
));
907 INIT_LIST_HEAD(&(q
->neighbors_waiting
));
909 atomic_set(&(q
->cong_status
), 0);
911 spin_lock_bh(&(queues_lock
));
912 list_add(&(q
->queue_list
), &queues
);
913 spin_unlock_bh(&(queues_lock
));
918 static void qos_queue_set_congstatus(struct qos_queue
*q_locked
)
922 if (time_before(q_locked
->jiffies_lastdrop
, jiffies
- HZ
/50)) {
923 newstatus
= CONGSTATUS_NONE
;
924 } else if (list_empty(&(q_locked
->kpackets_waiting
)) == 0) {
925 newstatus
= CONGSTATUS_KPACKETS
;
926 } else if (list_empty(&(q_locked
->conn_retrans_waiting
)) == 0) {
927 newstatus
= CONGSTATUS_RETRANS
;
928 } else if (list_empty(&(q_locked
->announce_waiting
)) == 0) {
929 newstatus
= CONGSTATUS_ANNOUNCE
;
930 } else if (list_empty(&(q_locked
->neighbors_waiting
)) == 0) {
931 newstatus
= CONGSTATUS_CONNDATA
;
933 newstatus
= CONGSTATUS_NONE
;
936 atomic_set(&(q_locked
->cong_status
), newstatus
);
939 void qos_set_lastdrop(struct qos_queue
*q
)
941 unsigned long iflags
;
943 spin_lock_irqsave(&(q
->qlock
), iflags
);
944 q
->jiffies_lastdrop
= jiffies
;
945 qos_queue_set_congstatus(q
);
946 spin_unlock_irqrestore(&(q
->qlock
), iflags
);
950 * if caller == QOS_CALLER_NEIGHBOR, nb->conns_waiting.lock must be held by
953 static void _qos_enqueue(struct qos_queue
*q
, struct resume_block
*rb
,
954 int caller
, int from_nbcongwin_resume
)
958 if (rb
->in_queue
== RB_INQUEUE_TRUE
) {
959 BUG_ON(caller
== QOS_CALLER_NEIGHBOR
);
961 } else if (rb
->in_queue
== RB_INQUEUE_NBCONGWIN
&&
962 from_nbcongwin_resume
== 0) {
966 if (unlikely(qos_queue_is_destroyed(q
)))
969 queues_empty
= list_empty(&(q
->kpackets_waiting
)) &&
970 list_empty(&(q
->conn_retrans_waiting
)) &&
971 list_empty(&(q
->announce_waiting
)) &&
972 list_empty(&(q
->neighbors_waiting
));
974 BUG_ON(!queues_empty
&& q
->qos_resume_scheduled
== 0);
976 rb
->in_queue
= RB_INQUEUE_TRUE
;
978 if (caller
== QOS_CALLER_KPACKET
) {
979 list_add(&(rb
->lh
), &(q
->kpackets_waiting
));
980 kref_get(&(container_of(rb
, struct neighbor
, rb_kp
)->ref
));
981 } else if (caller
== QOS_CALLER_CONN_RETRANS
) {
982 list_add(&(rb
->lh
) , &(q
->conn_retrans_waiting
));
983 kref_get(&(container_of(rb
, struct neighbor
, rb_cr
)->ref
));
984 } else if (caller
== QOS_CALLER_ANNOUNCE
) {
985 list_add(&(rb
->lh
), &(q
->announce_waiting
));
986 kref_get(&(container_of(rb
, struct announce_data
, rb
)->ref
));
987 } else if (caller
== QOS_CALLER_NEIGHBOR
) {
988 struct neighbor
*nb
= container_of(rb
, struct neighbor
, rb
);
989 list_add(&(rb
->lh
), &(q
->neighbors_waiting
));
990 kref_get(&(nb
->ref
));
991 BUG_ON(nb
->conns_waiting
.cnt
== 0);
992 q
->numconns
+= nb
->conns_waiting
.cnt
;
993 q
->priority_sum
+= nb
->conns_waiting
.priority_sum
;
999 if (q
->qos_resume_scheduled
== 0) {
1000 q
->jiffies_lastprogress
= jiffies
;
1001 q
->qos_resume_scheduled
= 1;
1002 if (caller
== QOS_CALLER_KPACKET
|| from_nbcongwin_resume
) {
1003 tasklet_schedule(&(q
->qos_resume_task
));
1005 if (mod_timer(&(q
->qos_resume_timer
), jiffies
+ 1) ==
1007 kref_get(&(q
->ref
));
1012 qos_queue_set_congstatus(q
);
1015 void qos_enqueue(struct qos_queue
*q
, struct resume_block
*rb
, int caller
)
1017 unsigned long iflags
;
1019 spin_lock_irqsave(&(q
->qlock
), iflags
);
1020 _qos_enqueue(q
, rb
, caller
, 0);
1021 spin_unlock_irqrestore(&(q
->qlock
), iflags
);
1024 void qos_remove_conn(struct conn
*trgt_out_lx
)
1026 unsigned long iflags
;
1027 struct neighbor
*nb
= trgt_out_lx
->target
.out
.nb
;
1028 struct qos_queue
*q
= nb
->queue
;
1032 BUG_ON(trgt_out_lx
->targettype
!= TARGET_OUT
);
1035 spin_lock_irqsave(&(nb
->conns_waiting
.lock
), iflags
);
1036 if (trgt_out_lx
->target
.out
.rb
.in_queue
== RB_INQUEUE_FALSE
) {
1037 spin_unlock_irqrestore(&(nb
->conns_waiting
.lock
), iflags
);
1040 spin_lock(&(q
->qlock
));
1042 trgt_out_lx
->target
.out
.rb
.in_queue
= RB_INQUEUE_FALSE
;
1043 list_del(&(trgt_out_lx
->target
.out
.rb
.lh
));
1044 BUG_ON(nb
->conns_waiting
.cnt
== 0);
1045 nb
->conns_waiting
.cnt
--;
1046 if (nb
->rb
.in_queue
== RB_INQUEUE_TRUE
) {
1047 BUG_ON(q
->numconns
== 0);
1051 BUG_ON(nb
->conns_waiting
.priority_sum
<
1052 trgt_out_lx
->target
.out
.rb_priority
);
1053 BUG_ON(q
->priority_sum
< trgt_out_lx
->target
.out
.rb_priority
);
1054 nb
->conns_waiting
.priority_sum
-=
1055 trgt_out_lx
->target
.out
.rb_priority
;
1056 q
->priority_sum
-= trgt_out_lx
->target
.out
.rb_priority
;
1057 trgt_out_lx
->target
.out
.rb_priority
= 0;
1059 if (list_empty(&(nb
->conns_waiting
.lh
)) &&
1060 list_empty(&(nb
->conns_waiting
.lh_nextpass
))) {
1061 BUG_ON(nb
->conns_waiting
.priority_sum
!= 0);
1062 BUG_ON(nb
->conns_waiting
.cnt
!= 0);
1064 BUG_ON(nb
->conns_waiting
.cnt
== 0);
1067 if (list_empty(&(nb
->conns_waiting
.lh
)) &&
1068 list_empty(&(nb
->conns_waiting
.lh_nextpass
)) &&
1069 nb
->rb
.in_queue
== RB_INQUEUE_TRUE
) {
1070 nb
->rb
.in_queue
= RB_INQUEUE_FALSE
;
1071 list_del(&(nb
->rb
.lh
));
1072 if (atomic_read(&(nb
->cmsg_delay_conndata
)) != 0) {
1073 atomic_set(&(nb
->cmsg_delay_conndata
), 0);
1079 BUG_ON(list_empty(&(q
->neighbors_waiting
)) && q
->numconns
!= 0);
1080 BUG_ON(list_empty(&(q
->neighbors_waiting
)) &&
1081 q
->priority_sum
!= 0);
1083 qos_queue_set_congstatus(q
);
1086 spin_unlock(&(q
->qlock
));
1087 spin_unlock_irqrestore(&(nb
->conns_waiting
.lock
), iflags
);
1090 spin_lock_bh(&(nb
->cmsg_lock
));
1091 schedule_controlmsg_timer(nb
);
1092 spin_unlock_bh(&(nb
->cmsg_lock
));
1095 kref_put(&(trgt_out_lx
->ref
), kreffree_bug
);
1098 kref_put(&(nb
->ref
), neighbor_free
);
1101 static void qos_enqueue_conn(struct conn
*trgt_out_lx
)
1103 unsigned long iflags
;
1104 struct neighbor
*nb
= trgt_out_lx
->target
.out
.nb
;
1105 struct qos_queue
*q
;
1107 BUG_ON(trgt_out_lx
->data_buf
.read_remaining
== 0);
1109 spin_lock_irqsave(&(nb
->conns_waiting
.lock
), iflags
);
1111 if (trgt_out_lx
->target
.out
.rb
.in_queue
!= RB_INQUEUE_FALSE
)
1114 trgt_out_lx
->target
.out
.rb
.in_queue
= RB_INQUEUE_TRUE
;
1115 list_add(&(trgt_out_lx
->target
.out
.rb
.lh
), &(nb
->conns_waiting
.lh
));
1116 kref_get(&(trgt_out_lx
->ref
));
1117 nb
->conns_waiting
.cnt
++;
1119 q
= trgt_out_lx
->target
.out
.nb
->queue
;
1120 spin_lock(&(q
->qlock
));
1121 if (nb
->rb
.in_queue
== RB_INQUEUE_TRUE
) {
1124 _qos_enqueue(q
, &(nb
->rb
), QOS_CALLER_NEIGHBOR
, 0);
1126 spin_unlock(&(q
->qlock
));
1129 spin_unlock_irqrestore(&(nb
->conns_waiting
.lock
), iflags
);
1132 static struct sk_buff
*create_packet(struct neighbor
*nb
, int size
,
1135 struct sk_buff
*ret
;
1137 ret
= alloc_skb(size
+ LL_RESERVED_SPACE(nb
->dev
) +
1138 nb
->dev
->needed_tailroom
, alloc_flags
);
1139 if (unlikely(ret
== 0))
1142 ret
->protocol
= htons(ETH_P_COR
);
1145 skb_reserve(ret
, LL_RESERVED_SPACE(nb
->dev
));
1146 if(unlikely(dev_hard_header(ret
, nb
->dev
, ETH_P_COR
, nb
->mac
,
1147 nb
->dev
->dev_addr
, ret
->len
) < 0))
1149 skb_reset_network_header(ret
);
1154 struct sk_buff
*create_packet_cmsg(struct neighbor
*nb
, int size
,
1155 gfp_t alloc_flags
, __u64 seqno
)
1157 struct sk_buff
*ret
;
1160 ret
= create_packet(nb
, size
+ 7, alloc_flags
);
1161 if (unlikely(ret
== 0))
1164 dest
= skb_put(ret
, 7);
1167 dest
[0] = PACKET_TYPE_CMSG
;
1170 put_u48(dest
, seqno
);
1176 struct sk_buff
*create_packet_conndata(struct neighbor
*nb
, int size
,
1177 gfp_t alloc_flags
, __u32 conn_id
, __u64 seqno
,
1178 __u8 snd_delayed_lowbuf
, __u8 flush
)
1180 struct sk_buff
*ret
;
1183 ret
= create_packet(nb
, size
+ 11, alloc_flags
);
1184 if (unlikely(ret
== 0))
1187 dest
= skb_put(ret
, 11);
1191 if (snd_delayed_lowbuf
!= 0) {
1192 dest
[0] = PACKET_TYPE_CONNDATA_LOWBUFDELAYED_FLUSH
;
1194 dest
[0] = PACKET_TYPE_CONNDATA_FLUSH
;
1197 if (snd_delayed_lowbuf
!= 0) {
1198 dest
[0] = PACKET_TYPE_CONNDATA_LOWBUFDELAYED
;
1200 dest
[0] = PACKET_TYPE_CONNDATA
;
1205 put_u32(dest
, conn_id
);
1207 put_u48(dest
, seqno
);
1213 void reschedule_conn_retrans_timer(struct neighbor
*nb_retransconnlocked
)
1215 struct conn_retrans
*cr
= 0;
1217 if (list_empty(&(nb_retransconnlocked
->retrans_conn_list
)))
1220 cr
= container_of(nb_retransconnlocked
->retrans_conn_list
.next
,
1221 struct conn_retrans
, timeout_list
);
1223 if (time_before_eq(cr
->timeout
, jiffies
)) {
1224 qos_enqueue(nb_retransconnlocked
->queue
,
1225 &(nb_retransconnlocked
->rb_cr
),
1226 QOS_CALLER_CONN_RETRANS
);
1228 if (mod_timer(&(nb_retransconnlocked
->retrans_conn_timer
),
1229 cr
->timeout
) == 0) {
1230 kref_get(&(nb_retransconnlocked
->ref
));
1237 * caller must also call kref_get/put, see reschedule_conn_retrans_timer
1239 static void cancel_conn_retrans(struct neighbor
*nb_retransconnlocked
,
1240 struct conn
*trgt_out_lx
, struct conn_retrans
*cr
,
1243 if (unlikely(cr
->state
== CONN_RETRANS_ACKED
))
1246 if (cr
->state
== CONN_RETRANS_SCHEDULED
) {
1247 list_del(&(cr
->timeout_list
));
1248 } else if (cr
->state
== CONN_RETRANS_LOWWINDOW
) {
1249 BUG_ON(trgt_out_lx
->target
.out
.retrans_lowwindow
== 0);
1250 if (likely(trgt_out_lx
->target
.out
.retrans_lowwindow
!= 65535))
1251 trgt_out_lx
->target
.out
.retrans_lowwindow
--;
1254 if (cr
->state
!= CONN_RETRANS_INITIAL
)
1255 *bytes_acked
+= cr
->length
;
1257 list_del(&(cr
->conn_list
));
1258 cr
->state
= CONN_RETRANS_ACKED
;
1260 kref_put(&(cr
->ref
), free_connretrans
);
1264 * nb->retrans_conn_lock must be held when calling this
1265 * (see schedule_retransmit_conn())
1267 static void cancel_acked_conn_retrans(struct conn
*trgt_out_l
,
1270 __u64 seqno_acked
= trgt_out_l
->target
.out
.seqno_acked
;
1272 while (list_empty(&(trgt_out_l
->target
.out
.retrans_list
)) == 0) {
1273 struct conn_retrans
*cr
= container_of(
1274 trgt_out_l
->target
.out
.retrans_list
.next
,
1275 struct conn_retrans
, conn_list
);
1277 if (seqno_after(cr
->seqno
+ cr
->length
, seqno_acked
)) {
1278 if (seqno_before(cr
->seqno
, seqno_acked
)) {
1279 *bytes_acked
+= seqno_clean(seqno_acked
-
1281 cr
->length
-= seqno_clean(seqno_acked
-
1283 cr
->seqno
= seqno_acked
;
1288 cancel_conn_retrans(trgt_out_l
->target
.out
.nb
, trgt_out_l
, cr
,
1292 reschedule_conn_retrans_timer(trgt_out_l
->target
.out
.nb
);
1295 void cancel_all_conn_retrans(struct conn
*trgt_out_lx
)
1297 struct neighbor
*nb
= trgt_out_lx
->target
.out
.nb
;
1298 __u64 bytes_acked
= 0;
1300 spin_lock_bh(&(nb
->retrans_conn_lock
));
1302 while (list_empty(&(trgt_out_lx
->target
.out
.retrans_list
)) == 0) {
1303 struct conn_retrans
*cr
= container_of(
1304 trgt_out_lx
->target
.out
.retrans_list
.next
,
1305 struct conn_retrans
, conn_list
);
1306 BUG_ON(cr
->trgt_out_o
!= trgt_out_lx
);
1308 cancel_conn_retrans(nb
, trgt_out_lx
, cr
, &bytes_acked
);
1311 reschedule_conn_retrans_timer(nb
);
1313 spin_unlock_bh(&(nb
->retrans_conn_lock
));
1315 if (bytes_acked
> 0)
1316 nbcongwin_data_acked(nb
, bytes_acked
);
1319 static void cancel_all_conn_retrans_nb(struct neighbor
*nb
)
1321 __u64 bytes_acked
= 0;
1324 struct conn_retrans
*cr
;
1326 spin_lock_bh(&(nb
->retrans_conn_lock
));
1328 if (list_empty(&(nb
->retrans_conn_list
))) {
1329 spin_unlock_bh(&(nb
->retrans_conn_lock
));
1333 cr
= container_of(nb
->retrans_conn_list
.next
,
1334 struct conn_retrans
, timeout_list
);
1336 kref_get(&(cr
->ref
));
1338 spin_unlock_bh(&(nb
->retrans_conn_lock
));
1341 spin_lock_bh(&(cr
->trgt_out_o
->rcv_lock
));
1342 spin_lock_bh(&(nb
->retrans_conn_lock
));
1344 if (likely(cr
== container_of(nb
->retrans_conn_list
.next
,
1345 struct conn_retrans
, timeout_list
)))
1346 cancel_conn_retrans(nb
, cr
->trgt_out_o
, cr
,
1349 spin_unlock_bh(&(nb
->retrans_conn_lock
));
1350 spin_unlock_bh(&(cr
->trgt_out_o
->rcv_lock
));
1352 kref_put(&(cr
->ref
), free_connretrans
);
1355 if (bytes_acked
> 0)
1356 nbcongwin_data_acked(nb
, bytes_acked
);
1359 static struct conn_retrans
*prepare_conn_retrans(struct conn
*trgt_out_l
,
1360 __u64 seqno
, __u32 len
, __u8 snd_delayed_lowbuf
,
1361 struct conn_retrans
*cr_splitted
, int retransconnlocked
)
1363 struct neighbor
*nb
= trgt_out_l
->target
.out
.nb
;
1365 struct conn_retrans
*cr
= kmem_cache_alloc(connretrans_slab
,
1368 if (unlikely(cr
== 0))
1371 BUG_ON(trgt_out_l
->isreset
!= 0);
1373 memset(cr
, 0, sizeof (struct conn_retrans
));
1374 cr
->trgt_out_o
= trgt_out_l
;
1375 kref_get(&(trgt_out_l
->ref
));
1378 cr
->snd_delayed_lowbuf
= snd_delayed_lowbuf
;
1379 kref_init(&(cr
->ref
));
1381 kref_get(&(cr
->ref
));
1382 if (retransconnlocked
== 0)
1383 spin_lock_bh(&(nb
->retrans_conn_lock
));
1385 if (cr_splitted
!= 0)
1386 list_add(&(cr
->conn_list
), &(cr_splitted
->conn_list
));
1388 list_add_tail(&(cr
->conn_list
),
1389 &(cr
->trgt_out_o
->target
.out
.retrans_list
));
1391 if (retransconnlocked
== 0)
1392 spin_unlock_bh(&(nb
->retrans_conn_lock
));
1397 #define RC_SENDRETRANS_OK 0
1398 #define RC_SENDRETRANS_OOM 1
1399 #define RC_SENDRETRANS_QUEUEFULL 2
1400 #define RC_SENDRETRANS_QUEUEFULLDROPPED 3
1402 static int __send_retrans(struct neighbor
*nb
, struct conn
*trgt_out_l
,
1403 struct conn_retrans
*cr
, __u64
*bytes_sent
)
1407 BUG_ON(cr
->length
== 0);
1409 if (trgt_out_l
->flush
!= 0 && seqno_eq(cr
->seqno
+ cr
->length
,
1410 trgt_out_l
->target
.out
.seqno_nextsend
) &&
1411 trgt_out_l
->data_buf
.read_remaining
== 0)
1414 if (send_conndata_as_skb(nb
, cr
->length
)) {
1415 struct sk_buff
*skb
;
1419 skb
= create_packet_conndata(nb
, cr
->length
, GFP_ATOMIC
,
1420 trgt_out_l
->target
.out
.conn_id
, cr
->seqno
,
1421 cr
->snd_delayed_lowbuf
, flush
);
1422 if (unlikely(skb
== 0))
1423 return RC_SENDRETRANS_OOM
;
1425 dst
= skb_put(skb
, cr
->length
);
1427 databuf_pullold(trgt_out_l
, cr
->seqno
, dst
, cr
->length
);
1429 rc
= cor_dev_queue_xmit(skb
, nb
->queue
,
1430 QOS_CALLER_CONN_RETRANS
);
1431 if (rc
== NET_XMIT_DROP
)
1432 return RC_SENDRETRANS_QUEUEFULLDROPPED
;
1433 schedule_retransmit_conn(cr
, 1, 0);
1434 if (rc
!= NET_XMIT_SUCCESS
)
1435 return RC_SENDRETRANS_QUEUEFULL
;
1438 struct control_msg_out
*cm
;
1441 buf
= kmalloc(cr
->length
, GFP_ATOMIC
);
1442 if (unlikely(buf
== 0))
1443 return RC_SENDRETRANS_OOM
;
1445 cm
= alloc_control_msg(nb
, ACM_PRIORITY_LOW
);
1446 if (unlikely(cm
== 0)) {
1448 return RC_SENDRETRANS_OOM
;
1451 databuf_pullold(trgt_out_l
, cr
->seqno
, buf
, cr
->length
);
1453 send_conndata(cm
, trgt_out_l
->target
.out
.conn_id
,
1454 cr
->seqno
, buf
, buf
, cr
->length
,
1455 cr
->snd_delayed_lowbuf
, flush
,
1456 trgt_out_l
->is_highlatency
, cr
);
1459 *bytes_sent
+= cr
->length
;
1461 return RC_SENDRETRANS_OK
;
1464 static int _send_retrans_splitcr_ifneeded(struct neighbor
*nb_retransconnlocked
,
1465 struct conn
*trgt_out_l
, struct conn_retrans
*cr
)
1467 __u32 targetmss
= mss_conndata(nb_retransconnlocked
,
1468 trgt_out_l
->is_highlatency
!= 0);
1469 __u64 windowlimit
= seqno_clean(
1470 trgt_out_l
->target
.out
.seqno_windowlimit
-
1472 __u32 maxsize
= targetmss
;
1473 if (windowlimit
< maxsize
)
1474 maxsize
= windowlimit
;
1476 if (unlikely(cr
->length
> maxsize
)) {
1477 struct conn_retrans
*cr2
= prepare_conn_retrans(trgt_out_l
,
1478 cr
->seqno
+ maxsize
, cr
->length
- maxsize
,
1479 cr
->snd_delayed_lowbuf
, cr
, 1);
1480 if (unlikely(cr2
== 0))
1481 return RC_SENDRETRANS_OOM
;
1483 cr2
->timeout
= cr
->timeout
;
1485 list_add(&(cr2
->timeout_list
),
1486 &(nb_retransconnlocked
->retrans_conn_list
));
1487 cr2
->state
= CONN_RETRANS_SCHEDULED
;
1489 cr
->length
= maxsize
;
1492 return RC_SENDRETRANS_OK
;
1495 static int _send_retrans(struct neighbor
*nb
, struct conn_retrans
*cr
,
1499 struct conn
*trgt_out_o
= cr
->trgt_out_o
;
1500 int rc
= RC_SENDRETRANS_OK
;
1502 spin_lock_bh(&(trgt_out_o
->rcv_lock
));
1504 BUG_ON(trgt_out_o
->targettype
!= TARGET_OUT
);
1505 BUG_ON(trgt_out_o
->target
.out
.nb
!= nb
);
1507 spin_lock_bh(&(nb
->retrans_conn_lock
));
1508 if (unlikely(cr
->state
== CONN_RETRANS_ACKED
)) {
1509 spin_unlock_bh(&(nb
->retrans_conn_lock
));
1513 BUG_ON(trgt_out_o
->isreset
!= 0);
1515 BUG_ON(seqno_before(cr
->seqno
, trgt_out_o
->target
.out
.seqno_acked
));
1517 if (seqno_after_eq(cr
->seqno
,
1518 trgt_out_o
->target
.out
.seqno_windowlimit
)) {
1519 BUG_ON(cr
->state
!= CONN_RETRANS_SENDING
);
1520 cr
->state
= CONN_RETRANS_LOWWINDOW
;
1521 if (likely(trgt_out_o
->target
.out
.retrans_lowwindow
!= 65535))
1522 trgt_out_o
->target
.out
.retrans_lowwindow
++;
1524 spin_unlock_bh(&(nb
->retrans_conn_lock
));
1528 rc
= _send_retrans_splitcr_ifneeded(nb
, trgt_out_o
, cr
);
1530 spin_unlock_bh(&(nb
->retrans_conn_lock
));
1532 kref_get(&(trgt_out_o
->ref
));
1534 if (rc
== RC_SENDRETRANS_OK
)
1535 rc
= __send_retrans(nb
, trgt_out_o
, cr
, bytes_sent
);
1537 if (rc
== RC_SENDRETRANS_OOM
|| rc
== RC_SENDRETRANS_QUEUEFULLDROPPED
) {
1538 spin_lock_bh(&(nb
->retrans_conn_lock
));
1539 if (unlikely(cr
->state
== CONN_RETRANS_ACKED
)) {
1540 } else if (likely(cr
->state
== CONN_RETRANS_SENDING
)) {
1541 if (rc
== RC_SENDRETRANS_OOM
)
1542 cr
->timeout
= jiffies
+ 1;
1543 list_add(&(cr
->timeout_list
), &(nb
->retrans_conn_list
));
1544 cr
->state
= CONN_RETRANS_SCHEDULED
;
1548 spin_unlock_bh(&(nb
->retrans_conn_lock
));
1552 spin_unlock_bh(&(trgt_out_o
->rcv_lock
));
1554 kref_put(&(trgt_out_o
->ref
), free_conn
);
1556 return (rc
== RC_SENDRETRANS_OOM
||
1557 rc
== RC_SENDRETRANS_QUEUEFULL
||
1558 rc
== RC_SENDRETRANS_QUEUEFULLDROPPED
);
1561 static int send_retrans(struct neighbor
*nb
, int *sent
)
1564 int nbstate
= get_neigh_state(nb
);
1565 __u64 bytes_sent
= 0;
1567 if (unlikely(nbstate
== NEIGHBOR_STATE_STALLED
)) {
1568 return QOS_RESUME_DONE
;
1569 } else if (unlikely(nbstate
== NEIGHBOR_STATE_KILLED
)) {
1571 * cancel_all_conn_retrans_nb should not be needed, because
1572 * reset_all_conns calls cancel_all_conn_retrans
1574 cancel_all_conn_retrans_nb(nb
);
1575 return QOS_RESUME_DONE
;
1579 struct conn_retrans
*cr
= 0;
1581 spin_lock_bh(&(nb
->retrans_conn_lock
));
1583 if (list_empty(&(nb
->retrans_conn_list
))) {
1584 spin_unlock_bh(&(nb
->retrans_conn_lock
));
1588 cr
= container_of(nb
->retrans_conn_list
.next
,
1589 struct conn_retrans
, timeout_list
);
1591 BUG_ON(cr
->state
!= CONN_RETRANS_SCHEDULED
);
1593 if (time_after(cr
->timeout
, jiffies
)) {
1594 spin_unlock_bh(&(nb
->retrans_conn_lock
));
1598 kref_get(&(cr
->ref
));
1599 list_del(&(cr
->timeout_list
));
1600 cr
->state
= CONN_RETRANS_SENDING
;
1602 spin_unlock_bh(&(nb
->retrans_conn_lock
));
1604 queuefull
= _send_retrans(nb
, cr
, &bytes_sent
);
1605 kref_put(&(cr
->ref
), free_connretrans
);
1614 nbcongwin_data_retransmitted(nb
, bytes_sent
);
1616 return queuefull
? QOS_RESUME_CONG
: QOS_RESUME_DONE
;
1619 void retransmit_conn_timerfunc(struct timer_list
*retrans_conn_timer
)
1621 struct neighbor
*nb
= container_of(retrans_conn_timer
,
1622 struct neighbor
, retrans_conn_timer
);
1623 qos_enqueue(nb
->queue
, &(nb
->rb_cr
), QOS_CALLER_CONN_RETRANS
);
1624 kref_put(&(nb
->ref
), neighbor_free
);
1627 static void conn_ack_ooo_rcvd_splitcr(struct conn
*trgt_out_l
,
1628 struct conn_retrans
*cr
, __u64 seqno_ooo
, __u32 length
,
1631 struct conn_retrans
*cr2
;
1632 __u64 seqno_cr2start
;
1633 __u32 oldcrlenght
= cr
->length
;
1635 if (cr
->state
!= CONN_RETRANS_SCHEDULED
&&
1636 cr
->state
!= CONN_RETRANS_LOWWINDOW
)
1639 seqno_cr2start
= seqno_ooo
+length
;
1640 cr2
= prepare_conn_retrans(trgt_out_l
, seqno_cr2start
,
1641 seqno_clean(cr
->seqno
+ cr
->length
- seqno_cr2start
),
1642 cr
->snd_delayed_lowbuf
, cr
, 1);
1644 if (unlikely(cr2
== 0))
1647 BUG_ON(cr2
->length
> cr
->length
);
1649 cr2
->timeout
= cr
->timeout
;
1650 cr2
->state
= cr
->state
;
1652 if (cr
->state
!= CONN_RETRANS_SCHEDULED
)
1653 list_add(&(cr2
->timeout_list
), &(cr
->timeout_list
));
1655 BUG_ON(seqno_clean(seqno_ooo
- cr
->seqno
) > cr
->length
);
1657 cr
->length
-= seqno_clean(seqno_ooo
- cr
->seqno
);
1658 BUG_ON(cr
->length
+ length
+ cr2
->length
!= oldcrlenght
);
1660 *bytes_acked
+= length
;
1663 void conn_ack_ooo_rcvd(struct neighbor
*nb
, __u32 conn_id
,
1664 struct conn
*trgt_out
, __u64 seqno_ooo
, __u32 length
,
1667 struct list_head
*curr
;
1669 if (unlikely(length
== 0))
1672 spin_lock_bh(&(trgt_out
->rcv_lock
));
1674 if (unlikely(trgt_out
->targettype
!= TARGET_OUT
))
1676 if (unlikely(trgt_out
->target
.out
.nb
!= nb
))
1678 if (unlikely(trgt_out
->target
.out
.conn_id
!= conn_id
))
1681 kref_get(&(nb
->ref
));
1682 spin_lock_bh(&(nb
->retrans_conn_lock
));
1684 curr
= trgt_out
->target
.out
.retrans_list
.next
;
1686 while (curr
!= &(trgt_out
->target
.out
.retrans_list
)) {
1687 struct conn_retrans
*cr
= container_of(curr
,
1688 struct conn_retrans
, conn_list
);
1690 int ack_covers_start
= seqno_after_eq(cr
->seqno
, seqno_ooo
);
1691 int ack_covers_end
= seqno_before_eq(cr
->seqno
+ cr
->length
,
1692 seqno_ooo
+ length
);
1696 if (seqno_before(cr
->seqno
+ cr
->length
, seqno_ooo
))
1699 if (seqno_after(cr
->seqno
, seqno_ooo
+ length
))
1702 if (likely(ack_covers_start
&& ack_covers_end
)) {
1703 cancel_conn_retrans(nb
, trgt_out
, cr
, bytes_acked
);
1704 reschedule_conn_retrans_timer(nb
);
1705 } else if (ack_covers_start
) {
1706 __u32 diff
= seqno_ooo
+ length
- cr
->seqno
-
1708 BUG_ON(diff
>= cr
->length
);
1711 *bytes_acked
=+ diff
;
1712 } else if (ack_covers_end
) {
1713 __u32 diff
= seqno_ooo
+ length
- cr
->seqno
;
1714 BUG_ON(diff
>= length
);
1716 *bytes_acked
+= diff
;
1718 conn_ack_ooo_rcvd_splitcr(trgt_out
, cr
, seqno_ooo
,
1719 length
, bytes_acked
);
1724 if (unlikely(list_empty(&(trgt_out
->target
.out
.retrans_list
)) == 0)) {
1725 trgt_out
->target
.out
.seqno_acked
=
1726 trgt_out
->target
.out
.seqno_nextsend
;
1728 struct conn_retrans
*cr
= container_of(
1729 trgt_out
->target
.out
.retrans_list
.next
,
1730 struct conn_retrans
, conn_list
);
1731 if (seqno_after(cr
->seqno
, trgt_out
->target
.out
.seqno_acked
))
1732 trgt_out
->target
.out
.seqno_acked
= cr
->seqno
;
1735 spin_unlock_bh(&(nb
->retrans_conn_lock
));
1736 kref_put(&(nb
->ref
), neighbor_free
);
1739 spin_unlock_bh(&(trgt_out
->rcv_lock
));
1742 static void _conn_ack_rcvd_nosendwin(struct conn
*trgt_out_l
)
1744 if (trgt_out_l
->bufsize
.state
== BUFSIZE_INCR
||
1745 trgt_out_l
->bufsize
.state
== BUFSIZE_INCR_FAST
)
1746 trgt_out_l
->bufsize
.state
= BUFSIZE_NOACTION
;
1748 if (trgt_out_l
->bufsize
.state
== BUFSIZE_NOACTION
)
1749 trgt_out_l
->bufsize
.act
.noact
.bytesleft
= max(
1750 trgt_out_l
->bufsize
.act
.noact
.bytesleft
,
1751 (__u32
) BUF_OUT_WIN_NOK_NOINCR
);
1753 trgt_out_l
->bufsize
.ignore_rcv_lowbuf
= max(
1754 trgt_out_l
->bufsize
.ignore_rcv_lowbuf
,
1755 (__u32
) BUF_OUT_WIN_NOK_NOINCR
);
1759 * nb->retrans_conn_lock must be held when calling this
1760 * (see schedule_retransmit_conn())
1762 static void reschedule_lowwindow_retrans(struct conn
*trgt_out_l
)
1764 struct list_head
*lh
= trgt_out_l
->target
.out
.retrans_list
.next
;
1767 while (trgt_out_l
->target
.out
.retrans_lowwindow
> 0 && cnt
< 100) {
1768 struct conn_retrans
*cr
;
1770 if (unlikely(lh
== &(trgt_out_l
->target
.out
.retrans_list
))) {
1771 BUG_ON(trgt_out_l
->target
.out
.retrans_lowwindow
!=
1773 trgt_out_l
->target
.out
.retrans_lowwindow
= 0;
1777 cr
= container_of(lh
, struct conn_retrans
, conn_list
);
1779 if (seqno_after_eq(cr
->seqno
,
1780 trgt_out_l
->target
.out
.seqno_windowlimit
)) {
1784 if (cr
->state
== CONN_RETRANS_LOWWINDOW
)
1785 schedule_retransmit_conn(cr
, 1, 1);
1792 void conn_ack_rcvd(struct neighbor
*nb
, __u32 conn_id
, struct conn
*trgt_out
,
1793 __u64 seqno
, int setwindow
, __u8 window
, __u64
*bytes_acked
)
1795 int seqno_advanced
= 0;
1796 int window_enlarged
= 0;
1798 spin_lock_bh(&(trgt_out
->rcv_lock
));
1800 if (unlikely(trgt_out
->isreset
!= 0))
1802 if (unlikely(trgt_out
->targettype
!= TARGET_OUT
))
1804 if (unlikely(trgt_out
->target
.out
.nb
!= nb
))
1806 if (unlikely(trgt_out
->reversedir
->source
.in
.conn_id
!= conn_id
))
1809 if (unlikely(seqno_after(seqno
, trgt_out
->target
.out
.seqno_nextsend
) ||
1810 seqno_before(seqno
, trgt_out
->target
.out
.seqno_acked
)))
1814 __u64 windowdec
= dec_log_64_7(window
);
1815 if (likely(seqno_after(seqno
,
1816 trgt_out
->target
.out
.seqno_acked
)) ||
1817 seqno_after(seqno
+ windowdec
,
1818 trgt_out
->target
.out
.seqno_windowlimit
)) {
1819 trgt_out
->target
.out
.seqno_windowlimit
= seqno
+
1821 window_enlarged
= 1;
1825 if (seqno_after(seqno
, trgt_out
->target
.out
.seqno_acked
))
1828 if (seqno_advanced
== 0 && window_enlarged
== 0)
1831 kref_get(&(nb
->ref
));
1832 spin_lock_bh(&(nb
->retrans_conn_lock
));
1834 if (seqno_advanced
) {
1835 trgt_out
->target
.out
.seqno_acked
= seqno
;
1836 cancel_acked_conn_retrans(trgt_out
, bytes_acked
);
1839 if (window_enlarged
)
1840 reschedule_lowwindow_retrans(trgt_out
);
1842 spin_unlock_bh(&(nb
->retrans_conn_lock
));
1843 kref_put(&(nb
->ref
), neighbor_free
);
1846 databuf_ack(trgt_out
, trgt_out
->target
.out
.seqno_acked
);
1848 if (seqno_eq(trgt_out
->target
.out
.seqno_acked
,
1849 trgt_out
->target
.out
.seqno_nextsend
))
1850 _conn_ack_rcvd_nosendwin(trgt_out
);
1853 if (seqno_advanced
|| window_enlarged
)
1854 flush_buf(trgt_out
);
1856 spin_unlock_bh(&(trgt_out
->rcv_lock
));
1858 wake_sender(trgt_out
);
1861 static void try_combine_conn_retrans_prev(struct neighbor
*nb_retransconnlocked
,
1862 struct conn
*trgt_out_lx
, struct conn_retrans
*cr
)
1864 struct conn_retrans
*cr_prev
;
1865 __u64 bytes_dummyacked
= 0;
1867 BUG_ON(cr
->state
!= CONN_RETRANS_SCHEDULED
);
1869 if (cr
->conn_list
.prev
== &(trgt_out_lx
->target
.out
.retrans_list
))
1872 cr_prev
= container_of(cr
->conn_list
.prev
, struct conn_retrans
,
1875 if (cr_prev
->state
!= CONN_RETRANS_SCHEDULED
)
1877 if (cr_prev
->timeout
!= cr
->timeout
)
1879 if (!seqno_eq(cr_prev
->seqno
+ cr_prev
->length
, cr
->seqno
))
1882 cr
->seqno
-= cr_prev
->length
;
1883 cr
->length
+= cr_prev
->length
;
1885 cancel_conn_retrans(nb_retransconnlocked
, trgt_out_lx
, cr_prev
,
1889 static void try_combine_conn_retrans_next(struct neighbor
*nb_retranslocked
,
1890 struct conn
*trgt_out_lx
, struct conn_retrans
*cr
)
1892 struct conn_retrans
*cr_next
;
1893 __u64 bytes_dummyacked
= 0;
1895 BUG_ON(cr
->state
!= CONN_RETRANS_SCHEDULED
);
1897 if (cr
->conn_list
.next
== &(trgt_out_lx
->target
.out
.retrans_list
))
1900 cr_next
= container_of(cr
->conn_list
.next
, struct conn_retrans
,
1903 if (cr_next
->state
!= CONN_RETRANS_SCHEDULED
)
1905 if (cr_next
->timeout
!= cr
->timeout
)
1907 if (!seqno_eq(cr
->seqno
+ cr
->length
, cr_next
->seqno
))
1910 cr
->length
+= cr_next
->length
;
1912 cancel_conn_retrans(nb_retranslocked
, trgt_out_lx
, cr_next
,
1916 void schedule_retransmit_conn(struct conn_retrans
*cr
, int connlocked
,
1917 int nbretransconn_locked
)
1919 struct conn
*trgt_out_o
= cr
->trgt_out_o
;
1920 struct neighbor
*nb
;
1923 if (connlocked
== 0)
1924 spin_lock_bh(&(trgt_out_o
->rcv_lock
));
1926 BUG_ON(trgt_out_o
->targettype
!= TARGET_OUT
);
1927 nb
= trgt_out_o
->target
.out
.nb
;
1929 cr
->timeout
= calc_timeout(atomic_read(&(nb
->latency_retrans_us
)),
1930 atomic_read(&(nb
->latency_stddev_retrans_us
)),
1931 atomic_read(&(nb
->max_remote_ackconn_delay_us
)));
1933 if (nbretransconn_locked
== 0)
1934 spin_lock_bh(&(nb
->retrans_conn_lock
));
1936 kref_get(&(nb
->ref
));
1938 BUG_ON(cr
->state
== CONN_RETRANS_SCHEDULED
);
1940 if (unlikely(cr
->state
== CONN_RETRANS_ACKED
)) {
1942 } else if (unlikely(cr
->state
== CONN_RETRANS_LOWWINDOW
)) {
1943 BUG_ON(trgt_out_o
->target
.out
.retrans_lowwindow
== 0);
1944 if (likely(trgt_out_o
->target
.out
.retrans_lowwindow
!= 65535))
1945 trgt_out_o
->target
.out
.retrans_lowwindow
--;
1948 first
= unlikely(list_empty(&(nb
->retrans_conn_list
)));
1949 list_add_tail(&(cr
->timeout_list
), &(nb
->retrans_conn_list
));
1950 cr
->state
= CONN_RETRANS_SCHEDULED
;
1952 if (unlikely(first
)) {
1953 reschedule_conn_retrans_timer(nb
);
1955 try_combine_conn_retrans_prev(nb
, trgt_out_o
, cr
);
1956 try_combine_conn_retrans_next(nb
, trgt_out_o
, cr
);
1960 if (nbretransconn_locked
== 0)
1961 spin_unlock_bh(&(nb
->retrans_conn_lock
));
1963 kref_put(&(nb
->ref
), neighbor_free
);
1965 if (connlocked
== 0)
1966 spin_unlock_bh(&(trgt_out_o
->rcv_lock
));
1969 static int _flush_out_skb(struct conn
*trgt_out_lx
, __u32 len
,
1970 __u8 snd_delayed_lowbuf
)
1972 struct neighbor
*nb
= trgt_out_lx
->target
.out
.nb
;
1975 struct conn_retrans
*cr
;
1976 struct sk_buff
*skb
;
1981 if (trgt_out_lx
->flush
!= 0 &&
1982 trgt_out_lx
->data_buf
.read_remaining
== len
)
1985 seqno
= trgt_out_lx
->target
.out
.seqno_nextsend
;
1986 skb
= create_packet_conndata(trgt_out_lx
->target
.out
.nb
, len
,
1987 GFP_ATOMIC
, trgt_out_lx
->target
.out
.conn_id
, seqno
,
1988 snd_delayed_lowbuf
, flush
);
1989 if (unlikely(skb
== 0))
1990 return RC_FLUSH_CONN_OUT_OOM
;
1992 cr
= prepare_conn_retrans(trgt_out_lx
, seqno
, len
, snd_delayed_lowbuf
,
1994 if (unlikely(cr
== 0)) {
1996 return RC_FLUSH_CONN_OUT_OOM
;
1999 dst
= skb_put(skb
, len
);
2001 databuf_pull(trgt_out_lx
, dst
, len
);
2003 rc
= cor_dev_queue_xmit(skb
, nb
->queue
, QOS_CALLER_NEIGHBOR
);
2004 if (rc
== NET_XMIT_DROP
) {
2005 databuf_unpull(trgt_out_lx
, len
);
2006 spin_lock_bh(&(nb
->retrans_conn_lock
));
2007 cancel_conn_retrans(nb
, trgt_out_lx
, cr
, 0);
2008 spin_unlock_bh(&(nb
->retrans_conn_lock
));
2009 kref_put(&(cr
->ref
), free_connretrans
);
2010 return RC_FLUSH_CONN_OUT_CONG
;
2013 trgt_out_lx
->target
.out
.seqno_nextsend
+= len
;
2014 nbcongwin_data_sent(nb
, len
);
2015 schedule_retransmit_conn(cr
, 1, 0);
2016 if (trgt_out_lx
->sourcetype
== SOURCE_SOCK
)
2017 update_src_sock_sndspeed(trgt_out_lx
, len
);
2019 kref_put(&(cr
->ref
), free_connretrans
);
2021 return (rc
== NET_XMIT_SUCCESS
) ?
2022 RC_FLUSH_CONN_OUT_OK
: RC_FLUSH_CONN_OUT_SENT_CONG
;
2025 static int _flush_out_conndata(struct conn
*trgt_out_lx
, __u32 len
,
2026 __u8 snd_delayed_lowbuf
)
2029 struct control_msg_out
*cm
;
2030 struct conn_retrans
*cr
;
2034 if (trgt_out_lx
->flush
!= 0 &&
2035 trgt_out_lx
->data_buf
.read_remaining
== len
)
2038 buf
= kmalloc(len
, GFP_ATOMIC
);
2040 if (unlikely(buf
== 0))
2041 return RC_FLUSH_CONN_OUT_OOM
;
2043 cm
= alloc_control_msg(trgt_out_lx
->target
.out
.nb
, ACM_PRIORITY_LOW
);
2044 if (unlikely(cm
== 0)) {
2046 return RC_FLUSH_CONN_OUT_OOM
;
2049 seqno
= trgt_out_lx
->target
.out
.seqno_nextsend
;
2051 cr
= prepare_conn_retrans(trgt_out_lx
, seqno
, len
, snd_delayed_lowbuf
,
2053 if (unlikely(cr
== 0)) {
2055 free_control_msg(cm
);
2056 return RC_FLUSH_CONN_OUT_OOM
;
2059 databuf_pull(trgt_out_lx
, buf
, len
);
2060 trgt_out_lx
->target
.out
.seqno_nextsend
+= len
;
2061 nbcongwin_data_sent(trgt_out_lx
->target
.out
.nb
, len
);
2062 if (trgt_out_lx
->sourcetype
== SOURCE_SOCK
)
2063 update_src_sock_sndspeed(trgt_out_lx
, len
);
2065 send_conndata(cm
, trgt_out_lx
->target
.out
.conn_id
, seqno
, buf
, buf
, len
,
2066 snd_delayed_lowbuf
, flush
, trgt_out_lx
->is_highlatency
,
2069 return RC_FLUSH_CONN_OUT_OK
;
2072 int srcin_buflimit_reached(struct conn
*src_in_lx
)
2076 if (unlikely(seqno_before(src_in_lx
->source
.in
.window_seqnolimit
,
2077 src_in_lx
->source
.in
.next_seqno
)))
2080 window_left
= seqno_clean(src_in_lx
->source
.in
.window_seqnolimit
-
2081 src_in_lx
->source
.in
.next_seqno
);
2083 if (window_left
< WINDOW_ENCODE_MIN
)
2086 if (window_left
/2 < src_in_lx
->data_buf
.read_remaining
)
2092 static __u32
maxsend_left_to_len(__u32 maxsend_left
)
2095 if (maxsend_left
< 128)
2096 return maxsend_left
;
2098 for (i
=128;i
<4096;) {
2099 if (i
*2 > maxsend_left
)
2104 return maxsend_left
- maxsend_left
%4096;
2107 static int seqno_low_sendlimit(struct conn
*trgt_out_lx
, __u64 windowlimit
,
2110 __u64 bytes_ackpending
;
2112 BUG_ON(seqno_before(trgt_out_lx
->target
.out
.seqno_nextsend
,
2113 trgt_out_lx
->target
.out
.seqno_acked
));
2115 bytes_ackpending
= seqno_clean(trgt_out_lx
->target
.out
.seqno_nextsend
-
2116 trgt_out_lx
->target
.out
.seqno_acked
);
2118 if (windowlimit
<= sndlen
)
2121 if (unlikely(bytes_ackpending
+ sndlen
< bytes_ackpending
))
2124 if (trgt_out_lx
->is_highlatency
!= 0)
2125 return (windowlimit
- sndlen
< (bytes_ackpending
+ sndlen
) / 4)
2128 return (windowlimit
- sndlen
< (bytes_ackpending
+ sndlen
) / 8)
2132 static void _flush_out_ignore_lowbuf(struct conn
*trgt_out_lx
)
2134 trgt_out_lx
->bufsize
.ignore_rcv_lowbuf
= max(
2135 trgt_out_lx
->bufsize
.ignore_rcv_lowbuf
,
2136 trgt_out_lx
->bufsize
.bufsize
>> BUFSIZE_SHIFT
);
2139 static __u64
get_windowlimit(struct conn
*trgt_out_lx
)
2141 if (unlikely(seqno_before(trgt_out_lx
->target
.out
.seqno_windowlimit
,
2142 trgt_out_lx
->target
.out
.seqno_nextsend
)))
2145 return seqno_clean(trgt_out_lx
->target
.out
.seqno_windowlimit
-
2146 trgt_out_lx
->target
.out
.seqno_nextsend
);
2149 static int _flush_out(struct conn
*trgt_out_lx
, __u32 maxsend
, __u32
*sent
,
2150 int from_qos
, int maxsend_forcedelay
)
2152 struct neighbor
*nb
= trgt_out_lx
->target
.out
.nb
;
2158 __u8 snd_delayed_lowbuf
= trgt_out_lx
->target
.out
.windowlimit_reached
;
2160 __u32 maxsend_left
= maxsend
;
2162 trgt_out_lx
->target
.out
.windowlimit_reached
= 0;
2164 BUG_ON(trgt_out_lx
->targettype
!= TARGET_OUT
);
2166 if (unlikely(trgt_out_lx
->target
.out
.established
== 0))
2167 return RC_FLUSH_CONN_OUT_OK
;
2169 if (unlikely(trgt_out_lx
->isreset
!= 0))
2170 return RC_FLUSH_CONN_OUT_OK
;
2172 BUG_ON(trgt_out_lx
->target
.out
.conn_id
== 0);
2174 if (unlikely(trgt_out_lx
->data_buf
.read_remaining
== 0))
2175 return RC_FLUSH_CONN_OUT_OK
;
2177 #warning todo burst queue
2178 if (from_qos
== 0 && qos_fastsend_allowed_conn(trgt_out_lx
) == 0)
2179 return RC_FLUSH_CONN_OUT_CONG
;
2181 spin_lock_bh(&(nb
->stalledconn_lock
));
2182 nbstate
= get_neigh_state(nb
);
2183 if (unlikely(nbstate
== NEIGHBOR_STATE_STALLED
)) {
2184 BUG_ON(trgt_out_lx
->target
.out
.nbstalled_lh
.prev
== 0 &&
2185 trgt_out_lx
->target
.out
.nbstalled_lh
.next
!= 0);
2186 BUG_ON(trgt_out_lx
->target
.out
.nbstalled_lh
.prev
!= 0 &&
2187 trgt_out_lx
->target
.out
.nbstalled_lh
.next
== 0);
2189 if (trgt_out_lx
->target
.out
.nbstalled_lh
.prev
== 0) {
2190 kref_get(&(trgt_out_lx
->ref
));
2191 list_add_tail(&(trgt_out_lx
->target
.out
.nbstalled_lh
),
2192 &(nb
->stalledconn_list
));
2195 spin_unlock_bh(&(nb
->stalledconn_lock
));
2197 if (unlikely(nbstate
!= NEIGHBOR_STATE_ACTIVE
))
2198 return RC_FLUSH_CONN_OUT_NBNOTACTIVE
;
2200 /* printk(KERN_ERR "flush %p %llu %u", trgt_out_l,
2201 get_windowlimit(trgt_out_l),
2202 trgt_out_l->data_buf.read_remaining); */
2204 targetmss
= mss_conndata(nb
, trgt_out_lx
->is_highlatency
!= 0);
2206 while (trgt_out_lx
->data_buf
.read_remaining
>= targetmss
) {
2207 __u64 windowlimit
= get_windowlimit(trgt_out_lx
);
2210 if (maxsend_left
< targetmss
)
2213 if (windowlimit
< targetmss
) {
2214 trgt_out_lx
->target
.out
.windowlimit_reached
= 1;
2215 snd_delayed_lowbuf
= 1;
2216 _flush_out_ignore_lowbuf(trgt_out_lx
);
2220 if (nbcongwin_send_allowed(nb
) == 0)
2221 return RC_FLUSH_CONN_OUT_CONG
;
2223 if (seqno_low_sendlimit(trgt_out_lx
, windowlimit
, targetmss
)) {
2224 trgt_out_lx
->target
.out
.windowlimit_reached
= 1;
2225 snd_delayed_lowbuf
= 1;
2228 if (likely(send_conndata_as_skb(nb
, targetmss
)))
2229 rc
= _flush_out_skb(trgt_out_lx
, targetmss
,
2230 snd_delayed_lowbuf
);
2232 rc
= _flush_out_conndata(trgt_out_lx
, targetmss
,
2233 snd_delayed_lowbuf
);
2235 if (rc
== RC_FLUSH_CONN_OUT_OK
||
2236 rc
== RC_FLUSH_CONN_OUT_SENT_CONG
) {
2237 maxsend_left
-= targetmss
;
2241 if (rc
== RC_FLUSH_CONN_OUT_SENT_CONG
)
2242 return RC_FLUSH_CONN_OUT_CONG
;
2243 if (rc
!= RC_FLUSH_CONN_OUT_OK
)
2247 if (trgt_out_lx
->data_buf
.read_remaining
> 0) {
2248 __u32 len
= trgt_out_lx
->data_buf
.read_remaining
;
2249 __u64 windowlimit
= get_windowlimit(trgt_out_lx
);
2252 if (maxsend_left
< len
) {
2253 if (maxsend_left
>= 65536 || (
2254 maxsend_left
== maxsend
&&
2255 maxsend_left
>= 128 &&
2256 trgt_out_lx
->is_highlatency
== 0 &&
2257 !maxsend_forcedelay
)) {
2258 len
= maxsend_left_to_len(maxsend_left
);
2260 return RC_FLUSH_CONN_OUT_MAXSENT
;
2264 if (trgt_out_lx
->flush
== 0 &&
2265 trgt_out_lx
->sourcetype
== SOURCE_SOCK
&&
2266 cor_sock_sndbufavailable(trgt_out_lx
) != 0)
2269 if (trgt_out_lx
->flush
== 0 &&
2270 trgt_out_lx
->sourcetype
== SOURCE_IN
&&
2271 srcin_buflimit_reached(trgt_out_lx
)
2273 seqno_eq(trgt_out_lx
->target
.out
.seqno_nextsend
,
2274 trgt_out_lx
->target
.out
.seqno_acked
) == 0 ||
2275 trgt_out_lx
->is_highlatency
!= 0))
2278 if (trgt_out_lx
->flush
== 0 &&
2279 trgt_out_lx
->sourcetype
== SOURCE_UNCONNECTED
&&
2280 cpacket_write_allowed(trgt_out_lx
) != 0)
2283 if (windowlimit
== 0 || (windowlimit
< len
&&
2284 seqno_eq(trgt_out_lx
->target
.out
.seqno_nextsend
,
2285 trgt_out_lx
->target
.out
.seqno_acked
) == 0)) {
2286 trgt_out_lx
->target
.out
.windowlimit_reached
= 1;
2287 snd_delayed_lowbuf
= 1;
2288 _flush_out_ignore_lowbuf(trgt_out_lx
);
2292 if (nbcongwin_send_allowed(nb
) == 0)
2293 return RC_FLUSH_CONN_OUT_CONG
;
2295 if (seqno_low_sendlimit(trgt_out_lx
, windowlimit
, len
)) {
2296 trgt_out_lx
->target
.out
.windowlimit_reached
= 1;
2297 snd_delayed_lowbuf
= 1;
2300 if (len
> windowlimit
) {
2302 _flush_out_ignore_lowbuf(trgt_out_lx
);
2305 if (send_conndata_as_skb(nb
, len
))
2306 rc
= _flush_out_skb(trgt_out_lx
, len
,
2307 snd_delayed_lowbuf
);
2309 rc
= _flush_out_conndata(trgt_out_lx
, len
,
2310 snd_delayed_lowbuf
);
2313 if (rc
== RC_FLUSH_CONN_OUT_OK
||
2314 rc
== RC_FLUSH_CONN_OUT_SENT_CONG
) {
2315 maxsend_left
-= len
;
2319 if (rc
== RC_FLUSH_CONN_OUT_SENT_CONG
)
2320 return RC_FLUSH_CONN_OUT_CONG
;
2321 if (rc
!= RC_FLUSH_CONN_OUT_OK
)
2326 return RC_FLUSH_CONN_OUT_OK
;
2329 int flush_out(struct conn
*trgt_out_lx
, __u32
*sent
)
2331 int rc
= _flush_out(trgt_out_lx
, 1 << 30, sent
, 0, 0);
2333 if (rc
== RC_FLUSH_CONN_OUT_CONG
|| rc
== RC_FLUSH_CONN_OUT_MAXSENT
||
2334 rc
== RC_FLUSH_CONN_OUT_OOM
)
2335 qos_enqueue_conn(trgt_out_lx
);
2340 void resume_nbstalled_conns(struct work_struct
*work
)
2342 struct neighbor
*nb
= container_of(work
, struct neighbor
,
2344 int rc
= RC_FLUSH_CONN_OUT_OK
;
2346 spin_lock_bh(&(nb
->stalledconn_lock
));
2347 nb
->stalledconn_work_scheduled
= 0;
2348 while (rc
!= RC_FLUSH_CONN_OUT_NBNOTACTIVE
&&
2349 list_empty(&(nb
->stalledconn_list
)) == 0) {
2350 struct list_head
*lh
= nb
->stalledconn_list
.next
;
2351 struct conn
*trgt_out
= container_of(lh
, struct conn
,
2352 target
.out
.nbstalled_lh
);
2354 BUG_ON(trgt_out
->targettype
!= TARGET_OUT
);
2359 spin_unlock_bh(&(nb
->stalledconn_lock
));
2361 spin_lock_bh(&(trgt_out
->rcv_lock
));
2362 if (likely(trgt_out
->targettype
== TARGET_OUT
))
2363 rc
= flush_out(trgt_out
, &sent
);
2364 spin_unlock_bh(&(trgt_out
->rcv_lock
));
2367 wake_sender(trgt_out
);
2369 kref_put(&(trgt_out
->ref
), free_conn
);
2371 spin_lock_bh(&(nb
->stalledconn_lock
));
2373 spin_unlock_bh(&(nb
->stalledconn_lock
));
2375 kref_put(&(nb
->ref
), neighbor_free
);
2378 int __init
cor_snd_init(void)
2380 connretrans_slab
= kmem_cache_create("cor_connretrans",
2381 sizeof(struct conn_retrans
), 8, 0, 0);
2382 if (unlikely(connretrans_slab
== 0))
2388 MODULE_LICENSE("GPL");