net/cor/neigh_waitingconns.c

   1 /**
   2  *      Connection oriented routing
   3  *      Copyright (C) 2007-2023 Michael Blizek
   4  *
   5  *      This program is free software; you can redistribute it and/or
   6  *      modify it under the terms of the GNU General Public License
   7  *      as published by the Free Software Foundation; either version 2
   8  *      of the License, or (at your option) any later version.
   9  *
  10  *      This program is distributed in the hope that it will be useful,
  11  *      but WITHOUT ANY WARRANTY; without even the implied warranty of
  12  *      MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  13  *      GNU General Public License for more details.
  14  */
  15
  16 #include "cor.h"
  17
  18
  19 static void _cor_neigh_waitingsconns_resume_accountbusytime(
  20                 struct cor_conn *trgt_out_l, __u32 priority, __u32 burstprio,
  21                 unsigned long jiffies_nb_lastduration)
  22 {
  23         unsigned long jiffies_tmp = jiffies;
  24         __u64 jiffies_nb_lastduration_shifted = (jiffies_nb_lastduration <<
  25                         JIFFIES_LAST_IDLE_SHIFT);
  26         __u64 burstfactor;
  27
  28         if (trgt_out_l->is_highlatency != 0) {
  29                 burstfactor = 2048;
  30         } else {
  31                 BUG_ON(burstprio < priority);
  32                 burstfactor = div_u64(1024LL * (__u64) burstprio, priority) * 2 -
  33                                 1024;
  34                 BUG_ON(burstfactor < 1024);
  35         }
  36
  37         trgt_out_l->trgt.out.jiffies_idle_since +=
  38                         (jiffies_nb_lastduration_shifted * burstfactor) / 1024;
  39
  40         if (time_before(jiffies_tmp << JIFFIES_LAST_IDLE_SHIFT,
  41                         trgt_out_l->trgt.out.jiffies_idle_since))
  42                 trgt_out_l->trgt.out.jiffies_idle_since =
  43                                 jiffies_tmp << JIFFIES_LAST_IDLE_SHIFT;
  44 }
  45
  46 static int _cor_neigh_waitingsconns_resume_nbnotactive(struct cor_neighbor *nb)
  47 {
  48         struct cor_dev *cd = nb->cd;
  49         int krefput_cd = 0;
  50
  51         spin_lock(&cd->send_queue.qlock);
  52
  53         if (unlikely(cor_get_neigh_state(nb) == NEIGHBOR_STATE_ACTIVE)) {
  54                 spin_unlock(&cd->send_queue.qlock);
  55                 return 1;
  56         }
  57
  58         if (likely(nb->rb.in_queue == RB_INQUEUE_TRUE)) {
  59                 list_del(&nb->rb.lh);
  60                 cor_nb_kref_put_bug(nb, "qos_queue_nb");
  61                 nb->rb.in_queue = RB_INQUEUE_NBNOTACTIVE;
  62                 BUG_ON(cd->send_queue.numconns < nb->conns_waiting.cnt);
  63                 cd->send_queue.numconns -= nb->conns_waiting.cnt;
  64                 BUG_ON(nb->conns_waiting.priority_sum >
  65                                 cd->send_queue.priority_sum);
  66                 cd->send_queue.priority_sum -= nb->conns_waiting.priority_sum;
  67
  68                 if (list_empty(&cd->send_queue.neighbors_waiting) &&
  69                                 list_empty(&cd->send_queue.neighbors_waiting_nextpass)) {
  70                         BUG_ON(cd->send_queue.numconns != 0);
  71                         BUG_ON(cd->send_queue.priority_sum != 0);
  72                 }
  73
  74                 krefput_cd = 1;
  75
  76                 cor_dev_queue_set_congstatus(cd);
  77         }
  78
  79         spin_unlock(&cd->send_queue.qlock);
  80
  81         if (krefput_cd != 0)
  82                 kref_put(&cd->ref, cor_dev_free);
  83
  84         return 0;
  85 }
  86
  87 static __u32 _cor_neigh_waitingsconns_resume_burstprio(
  88                 struct cor_conn *trgt_out_l, __u32 priority)
  89 {
  90         unsigned long idletime_hz_shifted = cor_get_conn_idletime(trgt_out_l);
  91         __u32 idletime_msecs = jiffies_to_msecs(idletime_hz_shifted >>
  92                         JIFFIES_LAST_IDLE_SHIFT);
  93         __u64 newprio;
  94
  95         if (trgt_out_l->is_highlatency != 0) {
  96                 /**
  97                  * negative burst for high latency conns:
  98                  * 50% if idle
  99                  * 125% if busy
 100                  */
 101
 102                 __u32 burstfactor;
 103
 104                 BUG_ON(idletime_msecs >
 105                                 BURSTPRIO_MAXIDLETIME_HIGHLATENCY_SECS * 1000);
 106                 BUG_ON(BURSTPRIO_MAXIDLETIME_HIGHLATENCY_SECS * 1000LL >
 107                                 U32_MAX / 1024);
 108
 109                 burstfactor = 768 - (768 * idletime_msecs) /
 110                                 (BURSTPRIO_MAXIDLETIME_HIGHLATENCY_SECS * 1000);
 111
 112                 newprio =  (((__u64) priority) * (512 + burstfactor)) /
 113                                         1024;
 114         } else {
 115                 __u32 burstfactor;
 116
 117                 BUG_ON(idletime_msecs >
 118                                 BURSTPRIO_MAXIDLETIME_LOWLATENCY_SECS * 1000);
 119                 BUG_ON(BURSTPRIO_MAXIDLETIME_LOWLATENCY_SECS * 1000LL >
 120                                 U32_MAX / 1024);
 121
 122                 burstfactor = (1024 * idletime_msecs) /
 123                                 (BURSTPRIO_MAXIDLETIME_LOWLATENCY_SECS * 1000);
 124
 125                 newprio =  (((__u64) priority) * (1024 + burstfactor * 2)) /
 126                                         1024;
 127         }
 128
 129         BUG_ON(newprio > U32_MAX);
 130         return (__u32) newprio;
 131 }
 132
 133 static __u32 _cor_neigh_waitingsconns_resume_maxsend(
 134                 struct cor_conn *trgt_out_l, __u32 newpriority,
 135                 int *maxsend_forcedelay)
 136 {
 137         unsigned long iflags;
 138
 139         struct cor_neighbor *nb = trgt_out_l->trgt.out.nb;
 140         struct cor_dev *cd = nb->cd;
 141         __u64 priority_sum;
 142         __u32 numconns;
 143         __u64 bytes_per_round;
 144         __u64 ret;
 145
 146         spin_lock_irqsave(&nb->conns_waiting.lock, iflags);
 147         spin_lock(&cd->send_queue.qlock);
 148
 149         if (unlikely(unlikely(trgt_out_l->trgt.out.rb.in_queue !=
 150                         RB_INQUEUE_TRUE) ||
 151                         unlikely(nb->rb.in_queue != RB_INQUEUE_TRUE))) {
 152                 spin_unlock(&cd->send_queue.qlock);
 153                 spin_unlock_irqrestore(&nb->conns_waiting.lock, iflags);
 154
 155                 return 1024;
 156         }
 157
 158         BUG_ON(nb->conns_waiting.priority_sum <
 159                         trgt_out_l->trgt.out.rb_priority);
 160         BUG_ON(cd->send_queue.priority_sum < trgt_out_l->trgt.out.rb_priority);
 161         nb->conns_waiting.priority_sum -= trgt_out_l->trgt.out.rb_priority;
 162         cd->send_queue.priority_sum -= trgt_out_l->trgt.out.rb_priority;
 163
 164         #warning todo remove priority_sum
 165         BUG_ON(nb->conns_waiting.priority_sum + newpriority <
 166                         nb->conns_waiting.priority_sum);
 167         BUG_ON(cd->send_queue.priority_sum + newpriority < cd->send_queue.priority_sum);
 168         nb->conns_waiting.priority_sum += newpriority;
 169         cd->send_queue.priority_sum += newpriority;
 170         trgt_out_l->trgt.out.rb_priority = newpriority;
 171
 172         priority_sum = cd->send_queue.priority_sum;
 173         numconns = cd->send_queue.numconns;
 174
 175         spin_unlock(&cd->send_queue.qlock);
 176         spin_unlock_irqrestore(&nb->conns_waiting.lock, iflags);
 177
 178         if (numconns <= 4) {
 179                 *maxsend_forcedelay = 1;
 180                 bytes_per_round = 2048;
 181         } else {
 182                 *maxsend_forcedelay = 0;
 183                 bytes_per_round = 1024;
 184         }
 185
 186         ret = div_u64(bytes_per_round * ((__u64) newpriority) *
 187                         ((__u64) numconns), priority_sum);
 188         if (unlikely(ret > U32_MAX))
 189                 return U32_MAX;
 190         return (__u32) ret;
 191 }
 192
 193 static int _cor_neigh_waitingsconns_resume_nextpass(
 194                 struct cor_neighbor *nb_waitingconnslocked)
 195 {
 196         BUG_ON(list_empty(&nb_waitingconnslocked->conns_waiting.lh) == 0);
 197
 198         if (list_empty(&nb_waitingconnslocked->conns_waiting.lh_nextpass)) {
 199                 BUG_ON(nb_waitingconnslocked->conns_waiting.cnt != 0);
 200                 return 1;
 201         }
 202
 203         BUG_ON(nb_waitingconnslocked->conns_waiting.cnt == 0);
 204
 205         cor_swap_list_items(&nb_waitingconnslocked->conns_waiting.lh,
 206                         &nb_waitingconnslocked->conns_waiting.lh_nextpass);
 207
 208         return 0;
 209 }
 210
 211 int cor_neigh_waitingsconns_resume(struct cor_dev *cd, struct cor_neighbor *nb,
 212                 unsigned long jiffies_nb_lastduration, int *progress)
 213 {
 214         unsigned long iflags;
 215
 216         while (1) {
 217                 __u32 priority;
 218                 __u32 burstprio;
 219                 __u32 maxsend;
 220                 int maxsend_forcedelay = 0;
 221
 222                 int rc2;
 223                 __u32 sent2 = 0;
 224
 225                 struct cor_conn *cn = 0;
 226
 227                 spin_lock_irqsave(&nb->conns_waiting.lock, iflags);
 228                 if (list_empty(&nb->conns_waiting.lh) != 0) {
 229                         int done = _cor_neigh_waitingsconns_resume_nextpass(nb);
 230
 231                         spin_unlock_irqrestore(&nb->conns_waiting.lock, iflags);
 232                         return done ? QOS_RESUME_DONE : QOS_RESUME_NEXTNEIGHBOR;
 233                 }
 234                 BUG_ON(nb->conns_waiting.cnt == 0);
 235
 236                 cn = container_of(nb->conns_waiting.lh.next, struct cor_conn,
 237                                 trgt.out.rb.lh);
 238                 BUG_ON(cn->targettype != TARGET_OUT);
 239                 BUG_ON(cn->trgt.out.rb.lh.prev != &nb->conns_waiting.lh);
 240                 BUG_ON((cn->trgt.out.rb.lh.next == &nb->conns_waiting.lh) &&
 241                                 (nb->conns_waiting.lh.prev !=
 242                                 &cn->trgt.out.rb.lh));
 243                 list_del(&cn->trgt.out.rb.lh);
 244                 list_add_tail(&cn->trgt.out.rb.lh,
 245                                 &nb->conns_waiting.lh_nextpass);
 246                 cor_conn_kref_get(cn, "stack");
 247                 spin_unlock_irqrestore(&nb->conns_waiting.lock, iflags);
 248
 249
 250                 priority = cor_conn_refresh_priority(cn, 0);
 251
 252                 spin_lock_bh(&cn->rcv_lock);
 253
 254                 if (unlikely(cn->targettype != TARGET_OUT)) {
 255                         spin_unlock_bh(&cn->rcv_lock);
 256                         continue;
 257                 }
 258
 259                 burstprio = _cor_neigh_waitingsconns_resume_burstprio(cn,
 260                                 priority);
 261
 262                 maxsend = _cor_neigh_waitingsconns_resume_maxsend(cn, burstprio,
 263                                 &maxsend_forcedelay);
 264                 if (cn->trgt.out.maxsend_extra >= maxsend)
 265                         maxsend_forcedelay = 0;
 266                 maxsend += cn->trgt.out.maxsend_extra;
 267                 if (unlikely(maxsend > U32_MAX))
 268                         maxsend = U32_MAX;
 269                 if (unlikely(maxsend >= 65536))
 270                         maxsend_forcedelay = 0;
 271
 272 retry:
 273                 rc2 = _cor_flush_out(cn, maxsend, &sent2, 1,
 274                                 maxsend_forcedelay);
 275
 276                 if (rc2 == RC_FLUSH_CONN_OUT_OK) {
 277                         cn->trgt.out.maxsend_extra = 0;
 278                         cor_neigh_waitingconns_remove_conn(cn);
 279                 } else if (unlikely(rc2 == RC_FLUSH_CONN_OUT_NBNOTACTIVE)) {
 280                         if (_cor_neigh_waitingsconns_resume_nbnotactive(nb) != 0)
 281                                 goto retry;
 282                 } else if (sent2 == 0 && (rc2 == RC_FLUSH_CONN_OUT_CONG ||
 283                                 unlikely(rc2 == RC_FLUSH_CONN_OUT_OOM))) {
 284                         spin_lock_irqsave(&nb->conns_waiting.lock, iflags);
 285                         if (likely(cn->trgt.out.rb.in_queue !=
 286                                         RB_INQUEUE_FALSE)) {
 287                                 list_del(&cn->trgt.out.rb.lh);
 288                                 list_add(&cn->trgt.out.rb.lh,
 289                                                 &nb->conns_waiting.lh);
 290                         }
 291                         spin_unlock_irqrestore(&nb->conns_waiting.lock, iflags);
 292                 } else if (rc2 == RC_FLUSH_CONN_OUT_CONG  ||
 293                                 unlikely(rc2 == RC_FLUSH_CONN_OUT_OOM)) {
 294                         cn->trgt.out.maxsend_extra = 0;
 295                 } else if (likely(rc2 == RC_FLUSH_CONN_OUT_MAXSENT)) {
 296                         if (unlikely(maxsend - sent2 > 65535))
 297                                 cn->trgt.out.maxsend_extra = 65535;
 298                         else
 299                                 cn->trgt.out.maxsend_extra = maxsend - sent2;
 300                 }
 301
 302                 if (sent2 != 0)
 303                         _cor_neigh_waitingsconns_resume_accountbusytime(cn,
 304                                         priority, burstprio,
 305                                         jiffies_nb_lastduration);
 306
 307                 spin_unlock_bh(&cn->rcv_lock);
 308
 309                 if (sent2 != 0) {
 310                         *progress = 1;
 311                         cor_wake_sender(cn);
 312                 }
 313
 314                 cor_conn_kref_put(cn, "stack");
 315
 316                 if (rc2 == RC_FLUSH_CONN_OUT_CONG  ||
 317                                 unlikely(rc2 == RC_FLUSH_CONN_OUT_OOM)) {
 318                         return QOS_RESUME_CONG;
 319                 } else if (unlikely(rc2 == RC_FLUSH_CONN_OUT_NBNOTACTIVE)) {
 320                         return QOS_RESUME_NEXTNEIGHBOR;
 321                 }
 322         }
 323 }
 324
 325 void cor_neigh_waitingconns_remove_conn(struct cor_conn *trgt_out_lx)
 326 {
 327         unsigned long iflags;
 328         struct cor_neighbor *nb = trgt_out_lx->trgt.out.nb;
 329         struct cor_dev *cd = nb->cd;
 330         int sched_cmsg = 0;
 331         int krefput_nb = 0;
 332
 333         BUG_ON(trgt_out_lx->targettype != TARGET_OUT);
 334         BUG_ON(cd == 0);
 335
 336         spin_lock_irqsave(&nb->conns_waiting.lock, iflags);
 337         if (trgt_out_lx->trgt.out.rb.in_queue == RB_INQUEUE_FALSE) {
 338                 spin_unlock_irqrestore(&nb->conns_waiting.lock, iflags);
 339                 return;
 340         }
 341         spin_lock(&cd->send_queue.qlock);
 342
 343         trgt_out_lx->trgt.out.rb.in_queue = RB_INQUEUE_FALSE;
 344         list_del(&trgt_out_lx->trgt.out.rb.lh);
 345         BUG_ON(nb->conns_waiting.cnt == 0);
 346         nb->conns_waiting.cnt--;
 347         if (nb->rb.in_queue == RB_INQUEUE_TRUE) {
 348                 BUG_ON(cd->send_queue.numconns == 0);
 349                 cd->send_queue.numconns--;
 350         }
 351
 352         BUG_ON(nb->conns_waiting.priority_sum <
 353                         trgt_out_lx->trgt.out.rb_priority);
 354         BUG_ON(cd->send_queue.priority_sum < trgt_out_lx->trgt.out.rb_priority);
 355         nb->conns_waiting.priority_sum -=
 356                         trgt_out_lx->trgt.out.rb_priority;
 357         cd->send_queue.priority_sum -= trgt_out_lx->trgt.out.rb_priority;
 358         trgt_out_lx->trgt.out.rb_priority = 0;
 359
 360         if (list_empty(&nb->conns_waiting.lh) &&
 361                         list_empty(&nb->conns_waiting.lh_nextpass)) {
 362                 BUG_ON(nb->conns_waiting.priority_sum != 0);
 363                 BUG_ON(nb->conns_waiting.cnt != 0);
 364         } else {
 365                 BUG_ON(nb->conns_waiting.cnt == 0);
 366         }
 367
 368         if (list_empty(&nb->conns_waiting.lh) &&
 369                         list_empty(&nb->conns_waiting.lh_nextpass) &&
 370                         nb->rb.in_queue == RB_INQUEUE_TRUE) {
 371                 BUG_ON(nb->conns_waiting.priority_sum != 0);
 372
 373                 nb->rb.in_queue = RB_INQUEUE_FALSE;
 374                 list_del(&nb->rb.lh);
 375                 if (atomic_read(&nb->cmsg_delay_conndata) != 0) {
 376                         atomic_set(&nb->cmsg_delay_conndata, 0);
 377                         sched_cmsg = 1;
 378                 }
 379                 krefput_nb = 1;
 380
 381                 if (list_empty(&cd->send_queue.neighbors_waiting) &&
 382                                 list_empty(&cd->send_queue.neighbors_waiting_nextpass)) {
 383                         BUG_ON(cd->send_queue.numconns != 0);
 384                         BUG_ON(cd->send_queue.priority_sum != 0);
 385                 }
 386
 387                 cor_dev_queue_set_congstatus(cd);
 388         }
 389
 390         spin_unlock(&cd->send_queue.qlock);
 391         spin_unlock_irqrestore(&nb->conns_waiting.lock, iflags);
 392
 393         if (sched_cmsg) {
 394                 spin_lock_bh(&nb->cmsg_lock);
 395                 cor_schedule_controlmsg_timer(nb);
 396                 spin_unlock_bh(&nb->cmsg_lock);
 397         }
 398
 399         cor_conn_kref_put_bug(trgt_out_lx, "qos_queue");
 400
 401         if (krefput_nb)
 402                 cor_nb_kref_put(nb, "qos_queue_nb");
 403 }
 404
 405 void cor_neigh_waitingconns_enqueue_conn(struct cor_conn *trgt_out_lx)
 406 {
 407         unsigned long iflags;
 408         struct cor_neighbor *nb = trgt_out_lx->trgt.out.nb;
 409         struct cor_dev *cd;
 410
 411         BUG_ON(trgt_out_lx->data_buf.read_remaining == 0);
 412
 413         spin_lock_irqsave(&nb->conns_waiting.lock, iflags);
 414
 415         if (trgt_out_lx->trgt.out.rb.in_queue != RB_INQUEUE_FALSE)
 416                 goto out;
 417
 418         BUG_ON(trgt_out_lx->trgt.out.rb_priority != 0);
 419
 420         trgt_out_lx->trgt.out.rb.in_queue = RB_INQUEUE_TRUE;
 421         list_add_tail(&trgt_out_lx->trgt.out.rb.lh,
 422                         &nb->conns_waiting.lh);
 423         cor_conn_kref_get(trgt_out_lx, "qos_queue");
 424         nb->conns_waiting.cnt++;
 425
 426         cd = trgt_out_lx->trgt.out.nb->cd;
 427         spin_lock(&cd->send_queue.qlock);
 428         if (nb->rb.in_queue == RB_INQUEUE_TRUE) {
 429                 cd->send_queue.numconns++;
 430         } else {
 431                 _cor_dev_queue_enqueue(cd, &nb->rb, 0, ns_to_ktime(0),
 432                                 QOS_CALLER_NEIGHBOR, 0, 0);
 433         }
 434         spin_unlock(&cd->send_queue.qlock);
 435
 436 out:
 437         spin_unlock_irqrestore(&nb->conns_waiting.lock, iflags);
 438 }