From 767ce963b819810c6e5106fd9de68190c24462af Mon Sep 17 00:00:00 2001 From: Michael Blizek Date: Sat, 25 Mar 2023 11:43:47 +0100 Subject: [PATCH] split dev_queue --- net/cor/Makefile | 2 +- net/cor/conn.c | 2 +- net/cor/conn_trgt_out.c | 27 +++ net/cor/cor.h | 18 +- net/cor/dev_queue.c | 450 +------------------------------------------ net/cor/neigh_waitingconns.c | 438 +++++++++++++++++++++++++++++++++++++++++ 6 files changed, 481 insertions(+), 456 deletions(-) create mode 100644 net/cor/neigh_waitingconns.c diff --git a/net/cor/Makefile b/net/cor/Makefile index 66e10e4ccc80..0e6be5edaf55 100644 --- a/net/cor/Makefile +++ b/net/cor/Makefile @@ -1,4 +1,4 @@ obj-y := config.o dev.o dev_queue.o util.o \ - neigh.o neigh_ann_snd.o neigh_ann_rcv.o neigh_rcv.o neigh_snd.o neigh_congwin.o \ + neigh.o neigh_ann_snd.o neigh_ann_rcv.o neigh_rcv.o neigh_snd.o neigh_congwin.o neigh_waitingconns.o \ conn.o conn_src_in.o conn_src_sock.o conn_trgt_unconn.o conn_trgt_out.o conn_trgt_sock.o conn_databuf.o \ sock.o sock_managed.o sock_rdaemon.o sock_raw.o diff --git a/net/cor/conn.c b/net/cor/conn.c index 1b7932b42a9c..21ca087b10c6 100644 --- a/net/cor/conn.c +++ b/net/cor/conn.c @@ -860,7 +860,7 @@ static void _cor_reset_conn(struct cor_conn *cn_ll, int trgt_out_resetneeded) cor_cancel_all_conn_retrans(cn_ll); - cor_qos_remove_conn(cn_ll); + cor_neigh_waitingconns_remove_conn(cn_ll); } else if (cn_ll->targettype == TARGET_SOCK) { if (likely(cn_ll->trgt.sock.cs != 0)) { if (cn_ll->trgt.sock.socktype == SOCKTYPE_RAW) { diff --git a/net/cor/conn_trgt_out.c b/net/cor/conn_trgt_out.c index ed45ef626a34..da04f79b6829 100644 --- a/net/cor/conn_trgt_out.c +++ b/net/cor/conn_trgt_out.c @@ -1215,6 +1215,33 @@ int _cor_flush_out(struct cor_conn *trgt_out_lx, __u32 maxsend, __u32 *sent, return RC_FLUSH_CONN_OUT_OK; } +unsigned long cor_get_conn_idletime(struct cor_conn *trgt_out_lx) +{ + unsigned long jiffies_shifted = jiffies << JIFFIES_LAST_IDLE_SHIFT; + __u32 burst_maxidle_secs; + __u32 burst_maxidle_hz_shifted; + unsigned long idletime_hz_shifted; + + if (trgt_out_lx->is_highlatency != 0) + burst_maxidle_secs = BURSTPRIO_MAXIDLETIME_HIGHLATENCY_SECS; + else + burst_maxidle_secs = BURSTPRIO_MAXIDLETIME_LOWLATENCY_SECS; + + burst_maxidle_hz_shifted = (burst_maxidle_secs * HZ) << + JIFFIES_LAST_IDLE_SHIFT; + + idletime_hz_shifted = jiffies_shifted - + trgt_out_lx->trgt.out.jiffies_idle_since; + + if (unlikely(idletime_hz_shifted > burst_maxidle_hz_shifted)) { + idletime_hz_shifted = burst_maxidle_hz_shifted; + trgt_out_lx->trgt.out.jiffies_idle_since = jiffies_shifted - + burst_maxidle_hz_shifted; + } + + return idletime_hz_shifted; +} + int __init cor_snd_init(void) { cor_connretrans_slab = kmem_cache_create("cor_connretrans", diff --git a/net/cor/cor.h b/net/cor/cor.h index 686eee6ad330..cbd64c4139e3 100644 --- a/net/cor/cor.h +++ b/net/cor/cor.h @@ -1359,8 +1359,6 @@ int __init cor_dev_init(void); void __exit cor_dev_exit1(void); /* dev_queue.c */ -unsigned long cor_get_conn_idletime(struct cor_conn *trgt_out_lx); - void cor_dev_queue_destroy(struct cor_dev *cd); int cor_dev_queue_init(struct cor_dev *cd); @@ -1390,10 +1388,6 @@ void cor_dev_queue_enqueue(struct cor_dev *cd, struct cor_resume_block *rb, unsigned long cmsg_send_start_j, ktime_t cmsg_send_start_kt, int caller, int from_nbnotactive_resume); -void cor_qos_remove_conn(struct cor_conn *trgt_out_l); - -void cor_qos_enqueue_conn(struct cor_conn *trgt_out_lx); - static inline int cor_dev_queue_xmit(struct sk_buff *skb, struct cor_dev *cd, int caller) @@ -1585,6 +1579,14 @@ void cor_nbcongwin_data_sent(struct cor_neighbor *nb, __u32 bytes_sent); int cor_nbcongwin_send_allowed(struct cor_neighbor *nb); +/* neigh_waitingconns.c */ +int cor_neigh_waitingsconns_resume(struct cor_dev *cd, struct cor_neighbor *nb, + unsigned long jiffies_nb_lastduration, int *progress); + +void cor_neigh_waitingconns_remove_conn(struct cor_conn *trgt_out_l); + +void cor_neigh_waitingconns_enqueue_conn(struct cor_conn *trgt_out_lx); + /* conn.c */ extern struct kmem_cache *cor_connid_reuse_slab; @@ -1723,11 +1725,13 @@ static inline int cor_flush_out(struct cor_conn *trgt_out_lx, __u32 *sent) if (rc == RC_FLUSH_CONN_OUT_CONG || rc == RC_FLUSH_CONN_OUT_MAXSENT || rc == RC_FLUSH_CONN_OUT_OOM || rc == RC_FLUSH_CONN_OUT_NBNOTACTIVE) - cor_qos_enqueue_conn(trgt_out_lx); + cor_neigh_waitingconns_enqueue_conn(trgt_out_lx); return rc; } +unsigned long cor_get_conn_idletime(struct cor_conn *trgt_out_lx); + int __init cor_snd_init(void); void __exit cor_snd_exit2(void); diff --git a/net/cor/dev_queue.c b/net/cor/dev_queue.c index 45d69d060146..6f21919fceb0 100644 --- a/net/cor/dev_queue.c +++ b/net/cor/dev_queue.c @@ -17,336 +17,6 @@ #include "cor.h" -static void _cor_resume_conns_accountbusytime(struct cor_conn *trgt_out_l, - __u32 priority, __u32 burstprio, - unsigned long jiffies_nb_lastduration) -{ - unsigned long jiffies_tmp = jiffies; - __u64 jiffies_nb_lastduration_shifted = (jiffies_nb_lastduration << - JIFFIES_LAST_IDLE_SHIFT); - __u64 burstfactor; - - if (trgt_out_l->is_highlatency != 0) { - burstfactor = 2048; - } else { - BUG_ON(burstprio < priority); - burstfactor = div_u64(1024LL * (__u64) burstprio, priority) * 2 - - 1024; - BUG_ON(burstfactor < 1024); - } - - trgt_out_l->trgt.out.jiffies_idle_since += - (jiffies_nb_lastduration_shifted * burstfactor) / 1024; - - if (time_before(jiffies_tmp << JIFFIES_LAST_IDLE_SHIFT, - trgt_out_l->trgt.out.jiffies_idle_since)) - trgt_out_l->trgt.out.jiffies_idle_since = - jiffies_tmp << JIFFIES_LAST_IDLE_SHIFT; -} - -static int _cor_resume_conns_nbnotactive(struct cor_neighbor *nb) -{ - struct cor_dev *cd = nb->cd; - int krefput_cd = 0; - - spin_lock(&cd->send_queue.qlock); - - if (unlikely(cor_get_neigh_state(nb) == NEIGHBOR_STATE_ACTIVE)) { - spin_unlock(&cd->send_queue.qlock); - return 1; - } - - if (likely(nb->rb.in_queue == RB_INQUEUE_TRUE)) { - list_del(&nb->rb.lh); - cor_nb_kref_put_bug(nb, "qos_queue_nb"); - nb->rb.in_queue = RB_INQUEUE_NBNOTACTIVE; - BUG_ON(cd->send_queue.numconns < nb->conns_waiting.cnt); - cd->send_queue.numconns -= nb->conns_waiting.cnt; - BUG_ON(nb->conns_waiting.priority_sum > - cd->send_queue.priority_sum); - cd->send_queue.priority_sum -= nb->conns_waiting.priority_sum; - - if (list_empty(&cd->send_queue.neighbors_waiting) && - list_empty(&cd->send_queue.neighbors_waiting_nextpass)) { - BUG_ON(cd->send_queue.numconns != 0); - BUG_ON(cd->send_queue.priority_sum != 0); - } - - krefput_cd = 1; - - cor_dev_queue_set_congstatus(cd); - } - - spin_unlock(&cd->send_queue.qlock); - - if (krefput_cd != 0) - kref_put(&cd->ref, cor_dev_free); - - return 0; -} - -unsigned long cor_get_conn_idletime(struct cor_conn *trgt_out_lx) -{ - unsigned long jiffies_shifted = jiffies << JIFFIES_LAST_IDLE_SHIFT; - __u32 burst_maxidle_secs; - __u32 burst_maxidle_hz_shifted; - unsigned long idletime_hz_shifted; - - if (trgt_out_lx->is_highlatency != 0) - burst_maxidle_secs = BURSTPRIO_MAXIDLETIME_HIGHLATENCY_SECS; - else - burst_maxidle_secs = BURSTPRIO_MAXIDLETIME_LOWLATENCY_SECS; - - burst_maxidle_hz_shifted = (burst_maxidle_secs * HZ) << - JIFFIES_LAST_IDLE_SHIFT; - - idletime_hz_shifted = jiffies_shifted - - trgt_out_lx->trgt.out.jiffies_idle_since; - - if (unlikely(idletime_hz_shifted > burst_maxidle_hz_shifted)) { - idletime_hz_shifted = burst_maxidle_hz_shifted; - trgt_out_lx->trgt.out.jiffies_idle_since = jiffies_shifted - - burst_maxidle_hz_shifted; - } - - return idletime_hz_shifted; -} - -static __u32 _cor_resume_conns_burstprio(struct cor_conn *trgt_out_l, - __u32 priority) -{ - unsigned long idletime_hz_shifted = cor_get_conn_idletime(trgt_out_l); - __u32 idletime_msecs = jiffies_to_msecs(idletime_hz_shifted >> - JIFFIES_LAST_IDLE_SHIFT); - __u64 newprio; - - if (trgt_out_l->is_highlatency != 0) { - /** - * negative burst for high latency conns: - * 50% if idle - * 125% if busy - */ - - __u32 burstfactor; - - BUG_ON(idletime_msecs > - BURSTPRIO_MAXIDLETIME_HIGHLATENCY_SECS * 1000); - BUG_ON(BURSTPRIO_MAXIDLETIME_HIGHLATENCY_SECS * 1000LL > - U32_MAX / 1024); - - burstfactor = 768 - (768 * idletime_msecs) / - (BURSTPRIO_MAXIDLETIME_HIGHLATENCY_SECS * 1000); - - newprio = (((__u64) priority) * (512 + burstfactor)) / - 1024; - } else { - __u32 burstfactor; - - BUG_ON(idletime_msecs > - BURSTPRIO_MAXIDLETIME_LOWLATENCY_SECS * 1000); - BUG_ON(BURSTPRIO_MAXIDLETIME_LOWLATENCY_SECS * 1000LL > - U32_MAX / 1024); - - burstfactor = (1024 * idletime_msecs) / - (BURSTPRIO_MAXIDLETIME_LOWLATENCY_SECS * 1000); - - newprio = (((__u64) priority) * (1024 + burstfactor * 2)) / - 1024; - } - - BUG_ON(newprio > U32_MAX); - return (__u32) newprio; -} - -static __u32 _cor_resume_conns_maxsend(struct cor_conn *trgt_out_l, - __u32 newpriority, int *maxsend_forcedelay) -{ - unsigned long iflags; - - struct cor_neighbor *nb = trgt_out_l->trgt.out.nb; - struct cor_dev *cd = nb->cd; - __u64 priority_sum; - __u32 numconns; - __u64 bytes_per_round; - __u64 ret; - - spin_lock_irqsave(&nb->conns_waiting.lock, iflags); - spin_lock(&cd->send_queue.qlock); - - if (unlikely(unlikely(trgt_out_l->trgt.out.rb.in_queue != - RB_INQUEUE_TRUE) || - unlikely(nb->rb.in_queue != RB_INQUEUE_TRUE))) { - spin_unlock(&cd->send_queue.qlock); - spin_unlock_irqrestore(&nb->conns_waiting.lock, iflags); - - return 1024; - } - - BUG_ON(nb->conns_waiting.priority_sum < - trgt_out_l->trgt.out.rb_priority); - BUG_ON(cd->send_queue.priority_sum < trgt_out_l->trgt.out.rb_priority); - nb->conns_waiting.priority_sum -= trgt_out_l->trgt.out.rb_priority; - cd->send_queue.priority_sum -= trgt_out_l->trgt.out.rb_priority; - - BUG_ON(nb->conns_waiting.priority_sum + newpriority < - nb->conns_waiting.priority_sum); - BUG_ON(cd->send_queue.priority_sum + newpriority < cd->send_queue.priority_sum); - nb->conns_waiting.priority_sum += newpriority; - cd->send_queue.priority_sum += newpriority; - trgt_out_l->trgt.out.rb_priority = newpriority; - - priority_sum = cd->send_queue.priority_sum; - numconns = cd->send_queue.numconns; - - spin_unlock(&cd->send_queue.qlock); - spin_unlock_irqrestore(&nb->conns_waiting.lock, iflags); - - if (numconns <= 4) { - *maxsend_forcedelay = 1; - bytes_per_round = 2048; - } else { - *maxsend_forcedelay = 0; - bytes_per_round = 1024; - } - - ret = div_u64(bytes_per_round * ((__u64) newpriority) * - ((__u64) numconns), priority_sum); - if (unlikely(ret > U32_MAX)) - return U32_MAX; - return (__u32) ret; -} - -static int _cor_resume_neighbors_nextpass( - struct cor_neighbor *nb_waitingconnslocked) -{ - BUG_ON(list_empty(&nb_waitingconnslocked->conns_waiting.lh) == 0); - - if (list_empty(&nb_waitingconnslocked->conns_waiting.lh_nextpass)) { - BUG_ON(nb_waitingconnslocked->conns_waiting.cnt != 0); - return 1; - } - - BUG_ON(nb_waitingconnslocked->conns_waiting.cnt == 0); - - cor_swap_list_items(&nb_waitingconnslocked->conns_waiting.lh, - &nb_waitingconnslocked->conns_waiting.lh_nextpass); - - return 0; -} - -static int _cor_resume_neighbors(struct cor_dev *cd, - struct cor_neighbor *nb, unsigned long jiffies_nb_lastduration, - int *progress) -{ - unsigned long iflags; - - while (1) { - __u32 priority; - __u32 burstprio; - __u32 maxsend; - int maxsend_forcedelay = 0; - - int rc2; - __u32 sent2 = 0; - - struct cor_conn *cn = 0; - - spin_lock_irqsave(&nb->conns_waiting.lock, iflags); - if (list_empty(&nb->conns_waiting.lh) != 0) { - int done = _cor_resume_neighbors_nextpass(nb); - - spin_unlock_irqrestore(&nb->conns_waiting.lock, iflags); - return done ? QOS_RESUME_DONE : QOS_RESUME_NEXTNEIGHBOR; - } - BUG_ON(nb->conns_waiting.cnt == 0); - - cn = container_of(nb->conns_waiting.lh.next, struct cor_conn, - trgt.out.rb.lh); - BUG_ON(cn->targettype != TARGET_OUT); - BUG_ON(cn->trgt.out.rb.lh.prev != &nb->conns_waiting.lh); - BUG_ON((cn->trgt.out.rb.lh.next == &nb->conns_waiting.lh) && - (nb->conns_waiting.lh.prev != - &cn->trgt.out.rb.lh)); - list_del(&cn->trgt.out.rb.lh); - list_add_tail(&cn->trgt.out.rb.lh, - &nb->conns_waiting.lh_nextpass); - cor_conn_kref_get(cn, "stack"); - spin_unlock_irqrestore(&nb->conns_waiting.lock, iflags); - - - priority = cor_conn_refresh_priority(cn, 0); - - spin_lock_bh(&cn->rcv_lock); - - if (unlikely(cn->targettype != TARGET_OUT)) { - spin_unlock_bh(&cn->rcv_lock); - continue; - } - - burstprio = _cor_resume_conns_burstprio(cn, priority); - - maxsend = _cor_resume_conns_maxsend(cn, burstprio, - &maxsend_forcedelay); - if (cn->trgt.out.maxsend_extra >= maxsend) - maxsend_forcedelay = 0; - maxsend += cn->trgt.out.maxsend_extra; - if (unlikely(maxsend > U32_MAX)) - maxsend = U32_MAX; - if (unlikely(maxsend >= 65536)) - maxsend_forcedelay = 0; - -retry: - rc2 = _cor_flush_out(cn, maxsend, &sent2, 1, - maxsend_forcedelay); - - if (rc2 == RC_FLUSH_CONN_OUT_OK) { - cn->trgt.out.maxsend_extra = 0; - cor_qos_remove_conn(cn); - } else if (unlikely(rc2 == RC_FLUSH_CONN_OUT_NBNOTACTIVE)) { - if (_cor_resume_conns_nbnotactive(nb) != 0) - goto retry; - } else if (sent2 == 0 && (rc2 == RC_FLUSH_CONN_OUT_CONG || - unlikely(rc2 == RC_FLUSH_CONN_OUT_OOM))) { - spin_lock_irqsave(&nb->conns_waiting.lock, iflags); - if (likely(cn->trgt.out.rb.in_queue != - RB_INQUEUE_FALSE)) { - list_del(&cn->trgt.out.rb.lh); - list_add(&cn->trgt.out.rb.lh, - &nb->conns_waiting.lh); - } - spin_unlock_irqrestore(&nb->conns_waiting.lock, iflags); - } else if (rc2 == RC_FLUSH_CONN_OUT_CONG || - unlikely(rc2 == RC_FLUSH_CONN_OUT_OOM)) { - cn->trgt.out.maxsend_extra = 0; - } else if (likely(rc2 == RC_FLUSH_CONN_OUT_MAXSENT)) { - if (unlikely(maxsend - sent2 > 65535)) - cn->trgt.out.maxsend_extra = 65535; - else - cn->trgt.out.maxsend_extra = maxsend - sent2; - } - - if (sent2 != 0) - _cor_resume_conns_accountbusytime(cn, priority, - burstprio, jiffies_nb_lastduration); - - spin_unlock_bh(&cn->rcv_lock); - - if (sent2 != 0) { - *progress = 1; - cor_wake_sender(cn); - } - - cor_conn_kref_put(cn, "stack"); - - if (rc2 == RC_FLUSH_CONN_OUT_CONG || - unlikely(rc2 == RC_FLUSH_CONN_OUT_OOM)) { - return QOS_RESUME_CONG; - } else if (unlikely(rc2 == RC_FLUSH_CONN_OUT_NBNOTACTIVE)) { - return QOS_RESUME_NEXTNEIGHBOR; - } - } -} - static struct cor_neighbor *cor_resume_neighbors_peeknextnb( struct cor_dev *cd, unsigned long *jiffies_nb_lastduration) { @@ -413,7 +83,8 @@ static int cor_resume_neighbors(struct cor_dev *cd, int *sent) atomic_set(&nb->cmsg_delay_conndata, 1); - rc = _cor_resume_neighbors(cd, nb, jiffies_nb_lastduration, sent); + rc = cor_neigh_waitingsconns_resume(cd, nb, jiffies_nb_lastduration, + sent); if (rc == QOS_RESUME_CONG) { cor_nb_kref_put(nb, "stack"); return QOS_RESUME_CONG; @@ -425,6 +96,7 @@ static int cor_resume_neighbors(struct cor_dev *cd, int *sent) cor_schedule_controlmsg_timer(nb); spin_unlock_bh(&nb->cmsg_lock); + #warning todo rename qos_* spin_lock_irqsave(&cd->send_queue.qlock, iflags); if (likely(nb->rb.in_queue == RB_INQUEUE_TRUE)) { if (nb->conns_waiting.cnt == 0) { @@ -543,7 +215,6 @@ static int _cor_qos_resume(struct cor_dev *cd, int *sent) spin_lock_irqsave(&cd->send_queue.qlock, iflags); while (1) { - #warning todo exit if (unlikely(cd->send_queue.is_destroyed == 1)) { rc = QOS_RESUME_EXIT; break; @@ -891,118 +562,3 @@ void cor_dev_queue_enqueue(struct cor_dev *cd, struct cor_resume_block *rb, caller, 0, from_nbnotactive_resume); spin_unlock_irqrestore(&cd->send_queue.qlock, iflags); } - -void cor_qos_remove_conn(struct cor_conn *trgt_out_lx) -{ - unsigned long iflags; - struct cor_neighbor *nb = trgt_out_lx->trgt.out.nb; - struct cor_dev *cd = nb->cd; - int sched_cmsg = 0; - int krefput_nb = 0; - - BUG_ON(trgt_out_lx->targettype != TARGET_OUT); - BUG_ON(cd == 0); - - spin_lock_irqsave(&nb->conns_waiting.lock, iflags); - if (trgt_out_lx->trgt.out.rb.in_queue == RB_INQUEUE_FALSE) { - spin_unlock_irqrestore(&nb->conns_waiting.lock, iflags); - return; - } - spin_lock(&cd->send_queue.qlock); - - trgt_out_lx->trgt.out.rb.in_queue = RB_INQUEUE_FALSE; - list_del(&trgt_out_lx->trgt.out.rb.lh); - BUG_ON(nb->conns_waiting.cnt == 0); - nb->conns_waiting.cnt--; - if (nb->rb.in_queue == RB_INQUEUE_TRUE) { - BUG_ON(cd->send_queue.numconns == 0); - cd->send_queue.numconns--; - } - - BUG_ON(nb->conns_waiting.priority_sum < - trgt_out_lx->trgt.out.rb_priority); - BUG_ON(cd->send_queue.priority_sum < trgt_out_lx->trgt.out.rb_priority); - nb->conns_waiting.priority_sum -= - trgt_out_lx->trgt.out.rb_priority; - cd->send_queue.priority_sum -= trgt_out_lx->trgt.out.rb_priority; - trgt_out_lx->trgt.out.rb_priority = 0; - - if (list_empty(&nb->conns_waiting.lh) && - list_empty(&nb->conns_waiting.lh_nextpass)) { - BUG_ON(nb->conns_waiting.priority_sum != 0); - BUG_ON(nb->conns_waiting.cnt != 0); - } else { - BUG_ON(nb->conns_waiting.cnt == 0); - } - - if (list_empty(&nb->conns_waiting.lh) && - list_empty(&nb->conns_waiting.lh_nextpass) && - nb->rb.in_queue == RB_INQUEUE_TRUE) { - BUG_ON(nb->conns_waiting.priority_sum != 0); - - nb->rb.in_queue = RB_INQUEUE_FALSE; - list_del(&nb->rb.lh); - if (atomic_read(&nb->cmsg_delay_conndata) != 0) { - atomic_set(&nb->cmsg_delay_conndata, 0); - sched_cmsg = 1; - } - krefput_nb = 1; - - if (list_empty(&cd->send_queue.neighbors_waiting) && - list_empty(&cd->send_queue.neighbors_waiting_nextpass)) { - BUG_ON(cd->send_queue.numconns != 0); - BUG_ON(cd->send_queue.priority_sum != 0); - } - - cor_dev_queue_set_congstatus(cd); - } - - spin_unlock(&cd->send_queue.qlock); - spin_unlock_irqrestore(&nb->conns_waiting.lock, iflags); - - if (sched_cmsg) { - spin_lock_bh(&nb->cmsg_lock); - cor_schedule_controlmsg_timer(nb); - spin_unlock_bh(&nb->cmsg_lock); - } - - cor_conn_kref_put_bug(trgt_out_lx, "qos_queue"); - - if (krefput_nb) - cor_nb_kref_put(nb, "qos_queue_nb"); -} - -void cor_qos_enqueue_conn(struct cor_conn *trgt_out_lx) -{ - unsigned long iflags; - struct cor_neighbor *nb = trgt_out_lx->trgt.out.nb; - struct cor_dev *cd; - - BUG_ON(trgt_out_lx->data_buf.read_remaining == 0); - - spin_lock_irqsave(&nb->conns_waiting.lock, iflags); - - if (trgt_out_lx->trgt.out.rb.in_queue != RB_INQUEUE_FALSE) - goto out; - - BUG_ON(trgt_out_lx->trgt.out.rb_priority != 0); - - trgt_out_lx->trgt.out.rb.in_queue = RB_INQUEUE_TRUE; - list_add_tail(&trgt_out_lx->trgt.out.rb.lh, - &nb->conns_waiting.lh); - cor_conn_kref_get(trgt_out_lx, "qos_queue"); - nb->conns_waiting.cnt++; - - cd = trgt_out_lx->trgt.out.nb->cd; - spin_lock(&cd->send_queue.qlock); - if (nb->rb.in_queue == RB_INQUEUE_TRUE) { - cd->send_queue.numconns++; - } else { - _cor_dev_queue_enqueue(cd, &nb->rb, 0, ns_to_ktime(0), - QOS_CALLER_NEIGHBOR, 0, 0); - } - spin_unlock(&cd->send_queue.qlock); - -out: - spin_unlock_irqrestore(&nb->conns_waiting.lock, iflags); -} diff --git a/net/cor/neigh_waitingconns.c b/net/cor/neigh_waitingconns.c new file mode 100644 index 000000000000..83f55a2ffe33 --- /dev/null +++ b/net/cor/neigh_waitingconns.c @@ -0,0 +1,438 @@ +/** + * Connection oriented routing + * Copyright (C) 2007-2023 Michael Blizek + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ + +#include "cor.h" + + +static void _cor_neigh_waitingsconns_resume_accountbusytime( + struct cor_conn *trgt_out_l, __u32 priority, __u32 burstprio, + unsigned long jiffies_nb_lastduration) +{ + unsigned long jiffies_tmp = jiffies; + __u64 jiffies_nb_lastduration_shifted = (jiffies_nb_lastduration << + JIFFIES_LAST_IDLE_SHIFT); + __u64 burstfactor; + + if (trgt_out_l->is_highlatency != 0) { + burstfactor = 2048; + } else { + BUG_ON(burstprio < priority); + burstfactor = div_u64(1024LL * (__u64) burstprio, priority) * 2 - + 1024; + BUG_ON(burstfactor < 1024); + } + + trgt_out_l->trgt.out.jiffies_idle_since += + (jiffies_nb_lastduration_shifted * burstfactor) / 1024; + + if (time_before(jiffies_tmp << JIFFIES_LAST_IDLE_SHIFT, + trgt_out_l->trgt.out.jiffies_idle_since)) + trgt_out_l->trgt.out.jiffies_idle_since = + jiffies_tmp << JIFFIES_LAST_IDLE_SHIFT; +} + +static int _cor_neigh_waitingsconns_resume_nbnotactive(struct cor_neighbor *nb) +{ + struct cor_dev *cd = nb->cd; + int krefput_cd = 0; + + spin_lock(&cd->send_queue.qlock); + + if (unlikely(cor_get_neigh_state(nb) == NEIGHBOR_STATE_ACTIVE)) { + spin_unlock(&cd->send_queue.qlock); + return 1; + } + + if (likely(nb->rb.in_queue == RB_INQUEUE_TRUE)) { + list_del(&nb->rb.lh); + cor_nb_kref_put_bug(nb, "qos_queue_nb"); + nb->rb.in_queue = RB_INQUEUE_NBNOTACTIVE; + BUG_ON(cd->send_queue.numconns < nb->conns_waiting.cnt); + cd->send_queue.numconns -= nb->conns_waiting.cnt; + BUG_ON(nb->conns_waiting.priority_sum > + cd->send_queue.priority_sum); + cd->send_queue.priority_sum -= nb->conns_waiting.priority_sum; + + if (list_empty(&cd->send_queue.neighbors_waiting) && + list_empty(&cd->send_queue.neighbors_waiting_nextpass)) { + BUG_ON(cd->send_queue.numconns != 0); + BUG_ON(cd->send_queue.priority_sum != 0); + } + + krefput_cd = 1; + + cor_dev_queue_set_congstatus(cd); + } + + spin_unlock(&cd->send_queue.qlock); + + if (krefput_cd != 0) + kref_put(&cd->ref, cor_dev_free); + + return 0; +} + +static __u32 _cor_neigh_waitingsconns_resume_burstprio( + struct cor_conn *trgt_out_l, __u32 priority) +{ + unsigned long idletime_hz_shifted = cor_get_conn_idletime(trgt_out_l); + __u32 idletime_msecs = jiffies_to_msecs(idletime_hz_shifted >> + JIFFIES_LAST_IDLE_SHIFT); + __u64 newprio; + + if (trgt_out_l->is_highlatency != 0) { + /** + * negative burst for high latency conns: + * 50% if idle + * 125% if busy + */ + + __u32 burstfactor; + + BUG_ON(idletime_msecs > + BURSTPRIO_MAXIDLETIME_HIGHLATENCY_SECS * 1000); + BUG_ON(BURSTPRIO_MAXIDLETIME_HIGHLATENCY_SECS * 1000LL > + U32_MAX / 1024); + + burstfactor = 768 - (768 * idletime_msecs) / + (BURSTPRIO_MAXIDLETIME_HIGHLATENCY_SECS * 1000); + + newprio = (((__u64) priority) * (512 + burstfactor)) / + 1024; + } else { + __u32 burstfactor; + + BUG_ON(idletime_msecs > + BURSTPRIO_MAXIDLETIME_LOWLATENCY_SECS * 1000); + BUG_ON(BURSTPRIO_MAXIDLETIME_LOWLATENCY_SECS * 1000LL > + U32_MAX / 1024); + + burstfactor = (1024 * idletime_msecs) / + (BURSTPRIO_MAXIDLETIME_LOWLATENCY_SECS * 1000); + + newprio = (((__u64) priority) * (1024 + burstfactor * 2)) / + 1024; + } + + BUG_ON(newprio > U32_MAX); + return (__u32) newprio; +} + +static __u32 _cor_neigh_waitingsconns_resume_maxsend( + struct cor_conn *trgt_out_l, __u32 newpriority, + int *maxsend_forcedelay) +{ + unsigned long iflags; + + struct cor_neighbor *nb = trgt_out_l->trgt.out.nb; + struct cor_dev *cd = nb->cd; + __u64 priority_sum; + __u32 numconns; + __u64 bytes_per_round; + __u64 ret; + + spin_lock_irqsave(&nb->conns_waiting.lock, iflags); + spin_lock(&cd->send_queue.qlock); + + if (unlikely(unlikely(trgt_out_l->trgt.out.rb.in_queue != + RB_INQUEUE_TRUE) || + unlikely(nb->rb.in_queue != RB_INQUEUE_TRUE))) { + spin_unlock(&cd->send_queue.qlock); + spin_unlock_irqrestore(&nb->conns_waiting.lock, iflags); + + return 1024; + } + + BUG_ON(nb->conns_waiting.priority_sum < + trgt_out_l->trgt.out.rb_priority); + BUG_ON(cd->send_queue.priority_sum < trgt_out_l->trgt.out.rb_priority); + nb->conns_waiting.priority_sum -= trgt_out_l->trgt.out.rb_priority; + cd->send_queue.priority_sum -= trgt_out_l->trgt.out.rb_priority; + + #warning todo remove priority_sum + BUG_ON(nb->conns_waiting.priority_sum + newpriority < + nb->conns_waiting.priority_sum); + BUG_ON(cd->send_queue.priority_sum + newpriority < cd->send_queue.priority_sum); + nb->conns_waiting.priority_sum += newpriority; + cd->send_queue.priority_sum += newpriority; + trgt_out_l->trgt.out.rb_priority = newpriority; + + priority_sum = cd->send_queue.priority_sum; + numconns = cd->send_queue.numconns; + + spin_unlock(&cd->send_queue.qlock); + spin_unlock_irqrestore(&nb->conns_waiting.lock, iflags); + + if (numconns <= 4) { + *maxsend_forcedelay = 1; + bytes_per_round = 2048; + } else { + *maxsend_forcedelay = 0; + bytes_per_round = 1024; + } + + ret = div_u64(bytes_per_round * ((__u64) newpriority) * + ((__u64) numconns), priority_sum); + if (unlikely(ret > U32_MAX)) + return U32_MAX; + return (__u32) ret; +} + +static int _cor_neigh_waitingsconns_resume_nextpass( + struct cor_neighbor *nb_waitingconnslocked) +{ + BUG_ON(list_empty(&nb_waitingconnslocked->conns_waiting.lh) == 0); + + if (list_empty(&nb_waitingconnslocked->conns_waiting.lh_nextpass)) { + BUG_ON(nb_waitingconnslocked->conns_waiting.cnt != 0); + return 1; + } + + BUG_ON(nb_waitingconnslocked->conns_waiting.cnt == 0); + + cor_swap_list_items(&nb_waitingconnslocked->conns_waiting.lh, + &nb_waitingconnslocked->conns_waiting.lh_nextpass); + + return 0; +} + +int cor_neigh_waitingsconns_resume(struct cor_dev *cd, struct cor_neighbor *nb, + unsigned long jiffies_nb_lastduration, int *progress) +{ + unsigned long iflags; + + while (1) { + __u32 priority; + __u32 burstprio; + __u32 maxsend; + int maxsend_forcedelay = 0; + + int rc2; + __u32 sent2 = 0; + + struct cor_conn *cn = 0; + + spin_lock_irqsave(&nb->conns_waiting.lock, iflags); + if (list_empty(&nb->conns_waiting.lh) != 0) { + int done = _cor_neigh_waitingsconns_resume_nextpass(nb); + + spin_unlock_irqrestore(&nb->conns_waiting.lock, iflags); + return done ? QOS_RESUME_DONE : QOS_RESUME_NEXTNEIGHBOR; + } + BUG_ON(nb->conns_waiting.cnt == 0); + + cn = container_of(nb->conns_waiting.lh.next, struct cor_conn, + trgt.out.rb.lh); + BUG_ON(cn->targettype != TARGET_OUT); + BUG_ON(cn->trgt.out.rb.lh.prev != &nb->conns_waiting.lh); + BUG_ON((cn->trgt.out.rb.lh.next == &nb->conns_waiting.lh) && + (nb->conns_waiting.lh.prev != + &cn->trgt.out.rb.lh)); + list_del(&cn->trgt.out.rb.lh); + list_add_tail(&cn->trgt.out.rb.lh, + &nb->conns_waiting.lh_nextpass); + cor_conn_kref_get(cn, "stack"); + spin_unlock_irqrestore(&nb->conns_waiting.lock, iflags); + + + priority = cor_conn_refresh_priority(cn, 0); + + spin_lock_bh(&cn->rcv_lock); + + if (unlikely(cn->targettype != TARGET_OUT)) { + spin_unlock_bh(&cn->rcv_lock); + continue; + } + + burstprio = _cor_neigh_waitingsconns_resume_burstprio(cn, + priority); + + maxsend = _cor_neigh_waitingsconns_resume_maxsend(cn, burstprio, + &maxsend_forcedelay); + if (cn->trgt.out.maxsend_extra >= maxsend) + maxsend_forcedelay = 0; + maxsend += cn->trgt.out.maxsend_extra; + if (unlikely(maxsend > U32_MAX)) + maxsend = U32_MAX; + if (unlikely(maxsend >= 65536)) + maxsend_forcedelay = 0; + +retry: + rc2 = _cor_flush_out(cn, maxsend, &sent2, 1, + maxsend_forcedelay); + + if (rc2 == RC_FLUSH_CONN_OUT_OK) { + cn->trgt.out.maxsend_extra = 0; + cor_neigh_waitingconns_remove_conn(cn); + } else if (unlikely(rc2 == RC_FLUSH_CONN_OUT_NBNOTACTIVE)) { + if (_cor_neigh_waitingsconns_resume_nbnotactive(nb) != 0) + goto retry; + } else if (sent2 == 0 && (rc2 == RC_FLUSH_CONN_OUT_CONG || + unlikely(rc2 == RC_FLUSH_CONN_OUT_OOM))) { + spin_lock_irqsave(&nb->conns_waiting.lock, iflags); + if (likely(cn->trgt.out.rb.in_queue != + RB_INQUEUE_FALSE)) { + list_del(&cn->trgt.out.rb.lh); + list_add(&cn->trgt.out.rb.lh, + &nb->conns_waiting.lh); + } + spin_unlock_irqrestore(&nb->conns_waiting.lock, iflags); + } else if (rc2 == RC_FLUSH_CONN_OUT_CONG || + unlikely(rc2 == RC_FLUSH_CONN_OUT_OOM)) { + cn->trgt.out.maxsend_extra = 0; + } else if (likely(rc2 == RC_FLUSH_CONN_OUT_MAXSENT)) { + if (unlikely(maxsend - sent2 > 65535)) + cn->trgt.out.maxsend_extra = 65535; + else + cn->trgt.out.maxsend_extra = maxsend - sent2; + } + + if (sent2 != 0) + _cor_neigh_waitingsconns_resume_accountbusytime(cn, + priority, burstprio, + jiffies_nb_lastduration); + + spin_unlock_bh(&cn->rcv_lock); + + if (sent2 != 0) { + *progress = 1; + cor_wake_sender(cn); + } + + cor_conn_kref_put(cn, "stack"); + + if (rc2 == RC_FLUSH_CONN_OUT_CONG || + unlikely(rc2 == RC_FLUSH_CONN_OUT_OOM)) { + return QOS_RESUME_CONG; + } else if (unlikely(rc2 == RC_FLUSH_CONN_OUT_NBNOTACTIVE)) { + return QOS_RESUME_NEXTNEIGHBOR; + } + } +} + +void cor_neigh_waitingconns_remove_conn(struct cor_conn *trgt_out_lx) +{ + unsigned long iflags; + struct cor_neighbor *nb = trgt_out_lx->trgt.out.nb; + struct cor_dev *cd = nb->cd; + int sched_cmsg = 0; + int krefput_nb = 0; + + BUG_ON(trgt_out_lx->targettype != TARGET_OUT); + BUG_ON(cd == 0); + + spin_lock_irqsave(&nb->conns_waiting.lock, iflags); + if (trgt_out_lx->trgt.out.rb.in_queue == RB_INQUEUE_FALSE) { + spin_unlock_irqrestore(&nb->conns_waiting.lock, iflags); + return; + } + spin_lock(&cd->send_queue.qlock); + + trgt_out_lx->trgt.out.rb.in_queue = RB_INQUEUE_FALSE; + list_del(&trgt_out_lx->trgt.out.rb.lh); + BUG_ON(nb->conns_waiting.cnt == 0); + nb->conns_waiting.cnt--; + if (nb->rb.in_queue == RB_INQUEUE_TRUE) { + BUG_ON(cd->send_queue.numconns == 0); + cd->send_queue.numconns--; + } + + BUG_ON(nb->conns_waiting.priority_sum < + trgt_out_lx->trgt.out.rb_priority); + BUG_ON(cd->send_queue.priority_sum < trgt_out_lx->trgt.out.rb_priority); + nb->conns_waiting.priority_sum -= + trgt_out_lx->trgt.out.rb_priority; + cd->send_queue.priority_sum -= trgt_out_lx->trgt.out.rb_priority; + trgt_out_lx->trgt.out.rb_priority = 0; + + if (list_empty(&nb->conns_waiting.lh) && + list_empty(&nb->conns_waiting.lh_nextpass)) { + BUG_ON(nb->conns_waiting.priority_sum != 0); + BUG_ON(nb->conns_waiting.cnt != 0); + } else { + BUG_ON(nb->conns_waiting.cnt == 0); + } + + if (list_empty(&nb->conns_waiting.lh) && + list_empty(&nb->conns_waiting.lh_nextpass) && + nb->rb.in_queue == RB_INQUEUE_TRUE) { + BUG_ON(nb->conns_waiting.priority_sum != 0); + + nb->rb.in_queue = RB_INQUEUE_FALSE; + list_del(&nb->rb.lh); + if (atomic_read(&nb->cmsg_delay_conndata) != 0) { + atomic_set(&nb->cmsg_delay_conndata, 0); + sched_cmsg = 1; + } + krefput_nb = 1; + + if (list_empty(&cd->send_queue.neighbors_waiting) && + list_empty(&cd->send_queue.neighbors_waiting_nextpass)) { + BUG_ON(cd->send_queue.numconns != 0); + BUG_ON(cd->send_queue.priority_sum != 0); + } + + cor_dev_queue_set_congstatus(cd); + } + + spin_unlock(&cd->send_queue.qlock); + spin_unlock_irqrestore(&nb->conns_waiting.lock, iflags); + + if (sched_cmsg) { + spin_lock_bh(&nb->cmsg_lock); + cor_schedule_controlmsg_timer(nb); + spin_unlock_bh(&nb->cmsg_lock); + } + + cor_conn_kref_put_bug(trgt_out_lx, "qos_queue"); + + if (krefput_nb) + cor_nb_kref_put(nb, "qos_queue_nb"); +} + +void cor_neigh_waitingconns_enqueue_conn(struct cor_conn *trgt_out_lx) +{ + unsigned long iflags; + struct cor_neighbor *nb = trgt_out_lx->trgt.out.nb; + struct cor_dev *cd; + + BUG_ON(trgt_out_lx->data_buf.read_remaining == 0); + + spin_lock_irqsave(&nb->conns_waiting.lock, iflags); + + if (trgt_out_lx->trgt.out.rb.in_queue != RB_INQUEUE_FALSE) + goto out; + + BUG_ON(trgt_out_lx->trgt.out.rb_priority != 0); + + trgt_out_lx->trgt.out.rb.in_queue = RB_INQUEUE_TRUE; + list_add_tail(&trgt_out_lx->trgt.out.rb.lh, + &nb->conns_waiting.lh); + cor_conn_kref_get(trgt_out_lx, "qos_queue"); + nb->conns_waiting.cnt++; + + cd = trgt_out_lx->trgt.out.nb->cd; + spin_lock(&cd->send_queue.qlock); + if (nb->rb.in_queue == RB_INQUEUE_TRUE) { + cd->send_queue.numconns++; + } else { + _cor_dev_queue_enqueue(cd, &nb->rb, 0, ns_to_ktime(0), + QOS_CALLER_NEIGHBOR, 0, 0); + } + spin_unlock(&cd->send_queue.qlock); + +out: + spin_unlock_irqrestore(&nb->conns_waiting.lock, iflags); +} -- 2.11.4.GIT