net/cor/credits.c

   1 #include "cor.h"
   2
   3 static spinlock_t conn_credits_lock;
   4
   5 /* credit_lock must be held while calling this */
   6 int refresh_credits_state(struct neighbor *nb)
   7 {
   8         long jiffies_tmp = jiffies;
   9
  10         __s64 creditrate = 1000 * ((__s64)(nb->creditrate_initial +
  11                         nb->creditrate_earning - nb->creditrate_spending));
  12
  13         __s64 crediterrorrate = 1000 * ((__s64)(nb->creditrate_spending -
  14                         nb->creditrate_spending_expected)) -
  15                         /**
  16                          * tolerance is 50% of creditsrate_initial + 2.5% of
  17                          * both spending+earning
  18                          */
  19                         500 * ((__s64)(nb->creditrate_initial)) -
  20                         25 * ((__s64)(nb->creditrate_spending_expected)) -
  21                         25 * ((__s64)(nb->creditrate_earning));
  22
  23         __s64 debitrate = 1000 * ((__s64)(nb->debitrate_initial +
  24                         nb->debitrate_initial_adj + nb->debitrate_earning -
  25                         nb->debitrate_spending));
  26
  27
  28         __s64 credits_adj = (jiffies_tmp - nb->jiffies_credit_update) *
  29                         creditrate + nb->credits_fract;
  30
  31         __s64 creditsd_adj = (jiffies_tmp - nb->jiffies_credit_update) *
  32                         crediterrorrate + nb->credits_diff_fract;
  33
  34         __s64 debits_adj = (jiffies_tmp - nb->jiffies_credit_update) *
  35                         debitrate + nb->debits_fract;
  36
  37
  38         nb->jiffies_credit_update = jiffies_tmp;
  39
  40
  41         if (unlikely(credits_adj < 0 && (nb->credits < -(credits_adj/HZ)))) {
  42                 nb->credits = 0;
  43                 nb->credits_fract = 0;
  44         } else {
  45                 nb->credits += credits_adj / HZ;
  46                 nb->credits_fract = credits_adj % HZ;
  47         }
  48
  49         if (unlikely(creditsd_adj < 0 && (nb->credits_diff <
  50                         -(creditsd_adj/HZ)))) {
  51                 nb->credits_diff = 0;
  52                 nb->credits_diff_fract = 0;
  53         } else {
  54                 nb->credits_diff += creditsd_adj / HZ;
  55                 nb->credits_diff_fract = creditsd_adj % HZ;
  56         }
  57
  58         if (unlikely(debits_adj < 0 && (nb->debits < -(debits_adj/HZ)))) {
  59                 nb->debits = 0;
  60                 nb->debits_fract = 0;
  61
  62                 return 1;
  63         } else {
  64                 nb->debits += debits_adj / HZ;
  65                 nb->debits_fract = debits_adj % HZ;
  66         }
  67
  68         return 0;
  69 }
  70
  71 void check_credit_state(struct neighbor *nb)
  72 {
  73         unsigned long iflags;
  74         int resetconns;
  75         int sendcredits = 0;
  76         struct list_head *currlh;
  77
  78 start:
  79         spin_lock_irqsave( &(nb->credits_lock), iflags );
  80         refresh_credits_state(nb);
  81         resetconns = (nb->debits == 0 && ((
  82                         (__s64) nb->debitrate_initial_adj +
  83                         (__s64) nb->debitrate_earning -
  84                         (__s64) nb->debitrate_spending) < 0));
  85         spin_unlock_irqrestore( &(nb->credits_lock), iflags );
  86
  87         if (likely(resetconns == 0)) {
  88                 if (unlikely(sendcredits))
  89                         send_credits(nb);
  90                 return;
  91         } else {
  92                 sendcredits = 1;
  93         }
  94
  95         mutex_lock(&(nb->conn_list_lock));
  96         BUG_ON(list_empty(&(nb->snd_conn_list)) && nb->num_send_conns != 0);
  97         currlh = nb->snd_conn_list.next;
  98
  99         while (currlh != &(nb->snd_conn_list)) {
 100                 int ispaying;
 101                 struct conn *sconn = container_of(currlh, struct conn,
 102                                         target.out.nb_list);
 103                 struct conn *rconn = sconn->reversedir;
 104
 105                 mutex_lock(&(rconn->rcv_lock));
 106                 BUG_ON(sconn->targettype != TARGET_OUT);
 107                 BUG_ON(rconn->sourcetype != SOURCE_IN);
 108
 109                 spin_lock_irqsave(&conn_credits_lock, iflags);
 110                 if (rconn->targettype == TARGET_UNCONNECTED ||
 111                                 (rconn->targettype == TARGET_SOCK &&
 112                                 rconn->target.sock.credituser != 0))
 113                         ispaying = 1;
 114                 else
 115                         ispaying = (sconn->recp_crate > rconn->sender_crate);
 116                 spin_unlock_irqrestore(&conn_credits_lock, iflags );
 117
 118                 mutex_unlock(&(rconn->rcv_lock));
 119
 120                 if (ispaying) {
 121                         /**
 122                         * reset_conn must not be called with conn_list_lock
 123                         * held
 124                         */
 125                         mutex_unlock(&(nb->conn_list_lock));
 126                         reset_conn(rconn);
 127                         goto start;
 128                 }
 129                 currlh = currlh->next;
 130         }
 131
 132         BUG();
 133 }
 134
 135 int debit_adj_needed(struct neighbor *nb)
 136 {
 137         int rc = 0;
 138         unsigned long iflags;
 139         __s32 expected_adj;
 140         spin_lock_irqsave( &(nb->credits_lock), iflags );
 141
 142         refresh_credits_state(nb);
 143
 144         expected_adj = nb->creditrate_initial - nb->credits_diff/10000;
 145         if (unlikely((expected_adj < 0 || nb->debitrate_initial_adj < 0) &&
 146                         ( (expected_adj * 2 < nb->debitrate_initial_adj) ||
 147                         (expected_adj > nb->debitrate_initial_adj * 2) ))) {
 148                 nb->debitrate_initial_adj = (nb->debitrate_initial_adj * 15 +
 149                                 expected_adj) / 16;
 150                 rc = 1;
 151         }
 152
 153         spin_unlock_irqrestore( &(nb->credits_lock), iflags );
 154
 155         return rc;
 156 }
 157
 158 void set_credits(struct neighbor *nb, __u64 credits, __s32 creditrate_initial,
 159                 __u32 creditrate_earning, __u32 creditrate_spending)
 160 {
 161         unsigned long iflags;
 162
 163         spin_lock_irqsave( &(nb->credits_lock), iflags );
 164
 165         refresh_credits_state(nb);
 166
 167         if (nb->credits_diff + nb->credits < credits)
 168                 nb->credits_diff = 0;
 169         else
 170                 nb->credits_diff += nb->credits - credits;
 171         nb->credits = credits;
 172         nb->creditrate_initial = creditrate_initial;
 173         nb->creditrate_earning = creditrate_earning;
 174         nb->creditrate_spending = creditrate_spending;
 175
 176         spin_unlock_irqrestore( &(nb->credits_lock), iflags );
 177 }
 178
 179 void set_debitrate_initial(struct neighbor *nb, __u32 debitrate)
 180 {
 181         unsigned long iflags;
 182
 183         spin_lock_irqsave( &(nb->credits_lock), iflags );
 184
 185         refresh_credits_state(nb);
 186         nb->debitrate_initial = debitrate;
 187
 188         spin_unlock_irqrestore( &(nb->credits_lock), iflags );
 189
 190         send_credits(nb);
 191 }
 192
 193 static __u32 credit_exchange_in(struct neighbor *nb, __u32 crate_in_raw)
 194 {
 195         unsigned long iflags;
 196         __u64 ret;
 197         spin_lock_irqsave( &(nb->credits_lock), iflags );
 198         if (unlikely(nb->debits == 0))
 199                 ret = crate_in_raw;
 200         else
 201                 ret = multiply_div(nb->credits, crate_in_raw, nb->debits);
 202         spin_unlock_irqrestore( &(nb->credits_lock), iflags );
 203
 204         if ((ret >> 32) != 0)
 205                 return -1;
 206         return (__u32) ret;
 207 }
 208
 209 static void refresh_conn_crates(struct conn *conn)
 210 {
 211         if (conn->sourcetype == SOURCE_NONE) {
 212                 conn->sender_crate = conn->reversedir->recp_crate;
 213         } else if (conn->sourcetype == SOURCE_IN) {
 214                 conn->sender_crate = credit_exchange_in(conn->source.in.nb,
 215                                 conn->source.in.crate_in_raw);
 216         } else if (conn->sourcetype == SOURCE_SOCK) {
 217                 BUG_ON(conn->reversedir->target.sock.credituser == 1 &&
 218                                 conn->targettype == TARGET_SOCK &&
 219                                 conn->reversedir->recp_crate != 0);
 220
 221
 222                 if ((conn->source.sock.crate + conn->reversedir->recp_crate) <
 223                                 conn->reversedir->recp_crate || (
 224                                 conn->reversedir->recp_crate +
 225                                 conn->source.sock.crate) > MAX_CREDITRATE_SOCK)
 226                         conn->sender_crate = MAX_CREDITRATE_SOCK;
 227                 else
 228                         conn->sender_crate = conn->reversedir->recp_crate +
 229                                         conn->source.sock.crate;
 230         } else {
 231                 BUG();
 232         }
 233
 234         if (conn->targettype == TARGET_UNCONNECTED) {
 235                 conn->recp_crate = conn->sender_crate;
 236                 conn->reversedir->sender_crate = conn->recp_crate;
 237         } else if ((conn->targettype == TARGET_SOCK &&
 238                         conn->target.sock.credituser == 0)) {
 239                 conn->recp_crate = conn->sender_crate;
 240                 conn->reversedir->sender_crate = conn->recp_crate +
 241                                 conn->reversedir->source.sock.crate;
 242         }
 243 }
 244
 245 static void _refresh_conn_credits(struct conn *conn)
 246 {
 247         long jiffies_tmp = jiffies;
 248
 249         __u32 scrate = conn->sender_crate;
 250         __u32 rcrate = conn->recp_crate;
 251
 252         __s64 diff;
 253
 254         refresh_conn_crates(conn);
 255
 256         refresh_conn_crates(conn);
 257
 258         /* 500 and not 1000 here: 1000 * (*_crate + *rate)/2 = 500 * ... */
 259         diff = 500 * ((__s64) conn->sender_crate + (__s64) scrate -
 260                         (__s64) conn->recp_crate - (__s64) rcrate) * (
 261                         jiffies_tmp - conn->jiffies_credit_update) +
 262                         conn->credits_fract;
 263
 264         conn->credits += diff / HZ;
 265         conn->credits_fract = diff % HZ;
 266
 267         conn->jiffies_credit_update = jiffies_tmp;
 268 }
 269
 270 /**
 271  * locked for access to credits + source.* + target.*
 272  * lock both sides, if either source or target is not out/in
 273  */
 274 static __u32 credits_lock_conn(struct conn *conn)
 275 {
 276         __u32 rc = 1;
 277         mutex_lock(&(conn->rcv_lock));
 278         if (conn->sourcetype != SOURCE_IN) {
 279                 mutex_unlock(&(conn->rcv_lock));
 280
 281                 mutex_lock(&(conn->reversedir->rcv_lock));
 282                 mutex_lock(&(conn->rcv_lock));
 283
 284                 rc = 6;
 285         } else if (conn->targettype != TARGET_OUT) {
 286                 mutex_lock(&(conn->reversedir->rcv_lock));
 287
 288                 rc = 3;
 289         }
 290         return rc;
 291 }
 292
 293 static void credits_unlock_conn(struct conn *conn, __u32 hints)
 294 {
 295         if ((hints & 1) != 0)
 296                 mutex_unlock(&(conn->rcv_lock));
 297         if ((hints & 2) != 0)
 298                 mutex_unlock(&(conn->reversedir->rcv_lock));
 299         if ((hints & 4) != 0)
 300                 mutex_unlock(&(conn->rcv_lock));
 301 }
 302
 303 void refresh_conn_credits(struct conn *conn)
 304 {
 305         unsigned long iflags;
 306         __u32 unlockhints = credits_lock_conn(conn);
 307
 308         if (unlikely(conn->targettype == TARGET_SOCK &&
 309                         conn->reversedir->targettype == TARGET_SOCK &&
 310                         conn->target.sock.credituser == 0 &&
 311                         conn->reversedir->target.sock.credituser == 0))
 312                 goto out;
 313         spin_lock_irqsave(&conn_credits_lock, iflags);
 314
 315         _refresh_conn_credits(conn);
 316         refresh_conn_crates(conn);
 317
 318         spin_unlock_irqrestore(&conn_credits_lock, iflags );
 319 out:
 320         credits_unlock_conn(conn, unlockhints);
 321 }
 322
 323 void set_conn_in_crate(struct conn *rconn, __u32 crate_in)
 324 {
 325         unsigned long iflags;
 326
 327         __u32 unlockhints = credits_lock_conn(rconn);
 328         spin_lock_irqsave(&conn_credits_lock, iflags);
 329
 330         BUG_ON(rconn->sourcetype != SOURCE_IN);
 331         _refresh_conn_credits(rconn);
 332         rconn->source.in.crate_in_raw = crate_in;
 333         refresh_conn_crates(rconn);
 334
 335         spin_unlock_irqrestore(&conn_credits_lock, iflags );
 336         credits_unlock_conn(rconn, unlockhints);
 337 }
 338
 339 int __init cor_credits_init(void)
 340 {
 341         spin_lock_init(&conn_credits_lock);
 342         return 0;
 343 }