[PATCH] elevator: abstract out the rbtree sort handling
[linux-2.6/suspend2-2.6.18.git] / net / sched / sch_netem.c
blob45939bafbdf894dfcb61a0ed993d1d79204aa5a8
1 /*
2 * net/sched/sch_netem.c Network emulator
4 * This program is free software; you can redistribute it and/or
5 * modify it under the terms of the GNU General Public License
6 * as published by the Free Software Foundation; either version
7 * 2 of the License, or (at your option) any later version.
9 * Many of the algorithms and ideas for this came from
10 * NIST Net which is not copyrighted.
12 * Authors: Stephen Hemminger <shemminger@osdl.org>
13 * Catalin(ux aka Dino) BOIE <catab at umbrella dot ro>
16 #include <linux/module.h>
17 #include <linux/bitops.h>
18 #include <linux/types.h>
19 #include <linux/kernel.h>
20 #include <linux/errno.h>
21 #include <linux/netdevice.h>
22 #include <linux/skbuff.h>
23 #include <linux/rtnetlink.h>
25 #include <net/pkt_sched.h>
27 #define VERSION "1.2"
29 /* Network Emulation Queuing algorithm.
30 ====================================
32 Sources: [1] Mark Carson, Darrin Santay, "NIST Net - A Linux-based
33 Network Emulation Tool
34 [2] Luigi Rizzo, DummyNet for FreeBSD
36 ----------------------------------------------------------------
38 This started out as a simple way to delay outgoing packets to
39 test TCP but has grown to include most of the functionality
40 of a full blown network emulator like NISTnet. It can delay
41 packets and add random jitter (and correlation). The random
42 distribution can be loaded from a table as well to provide
43 normal, Pareto, or experimental curves. Packet loss,
44 duplication, and reordering can also be emulated.
46 This qdisc does not do classification that can be handled in
47 layering other disciplines. It does not need to do bandwidth
48 control either since that can be handled by using token
49 bucket or other rate control.
51 The simulator is limited by the Linux timer resolution
52 and will create packet bursts on the HZ boundary (1ms).
55 struct netem_sched_data {
56 struct Qdisc *qdisc;
57 struct timer_list timer;
59 u32 latency;
60 u32 loss;
61 u32 limit;
62 u32 counter;
63 u32 gap;
64 u32 jitter;
65 u32 duplicate;
66 u32 reorder;
67 u32 corrupt;
69 struct crndstate {
70 unsigned long last;
71 unsigned long rho;
72 } delay_cor, loss_cor, dup_cor, reorder_cor, corrupt_cor;
74 struct disttable {
75 u32 size;
76 s16 table[0];
77 } *delay_dist;
80 /* Time stamp put into socket buffer control block */
81 struct netem_skb_cb {
82 psched_time_t time_to_send;
85 /* init_crandom - initialize correlated random number generator
86 * Use entropy source for initial seed.
88 static void init_crandom(struct crndstate *state, unsigned long rho)
90 state->rho = rho;
91 state->last = net_random();
94 /* get_crandom - correlated random number generator
95 * Next number depends on last value.
96 * rho is scaled to avoid floating point.
98 static unsigned long get_crandom(struct crndstate *state)
100 u64 value, rho;
101 unsigned long answer;
103 if (state->rho == 0) /* no correllation */
104 return net_random();
106 value = net_random();
107 rho = (u64)state->rho + 1;
108 answer = (value * ((1ull<<32) - rho) + state->last * rho) >> 32;
109 state->last = answer;
110 return answer;
113 /* tabledist - return a pseudo-randomly distributed value with mean mu and
114 * std deviation sigma. Uses table lookup to approximate the desired
115 * distribution, and a uniformly-distributed pseudo-random source.
117 static long tabledist(unsigned long mu, long sigma,
118 struct crndstate *state, const struct disttable *dist)
120 long t, x;
121 unsigned long rnd;
123 if (sigma == 0)
124 return mu;
126 rnd = get_crandom(state);
128 /* default uniform distribution */
129 if (dist == NULL)
130 return (rnd % (2*sigma)) - sigma + mu;
132 t = dist->table[rnd % dist->size];
133 x = (sigma % NETEM_DIST_SCALE) * t;
134 if (x >= 0)
135 x += NETEM_DIST_SCALE/2;
136 else
137 x -= NETEM_DIST_SCALE/2;
139 return x / NETEM_DIST_SCALE + (sigma / NETEM_DIST_SCALE) * t + mu;
143 * Insert one skb into qdisc.
144 * Note: parent depends on return value to account for queue length.
145 * NET_XMIT_DROP: queue length didn't change.
146 * NET_XMIT_SUCCESS: one skb was queued.
148 static int netem_enqueue(struct sk_buff *skb, struct Qdisc *sch)
150 struct netem_sched_data *q = qdisc_priv(sch);
151 /* We don't fill cb now as skb_unshare() may invalidate it */
152 struct netem_skb_cb *cb;
153 struct sk_buff *skb2;
154 int ret;
155 int count = 1;
157 pr_debug("netem_enqueue skb=%p\n", skb);
159 /* Random duplication */
160 if (q->duplicate && q->duplicate >= get_crandom(&q->dup_cor))
161 ++count;
163 /* Random packet drop 0 => none, ~0 => all */
164 if (q->loss && q->loss >= get_crandom(&q->loss_cor))
165 --count;
167 if (count == 0) {
168 sch->qstats.drops++;
169 kfree_skb(skb);
170 return NET_XMIT_BYPASS;
174 * If we need to duplicate packet, then re-insert at top of the
175 * qdisc tree, since parent queuer expects that only one
176 * skb will be queued.
178 if (count > 1 && (skb2 = skb_clone(skb, GFP_ATOMIC)) != NULL) {
179 struct Qdisc *rootq = sch->dev->qdisc;
180 u32 dupsave = q->duplicate; /* prevent duplicating a dup... */
181 q->duplicate = 0;
183 rootq->enqueue(skb2, rootq);
184 q->duplicate = dupsave;
188 * Randomized packet corruption.
189 * Make copy if needed since we are modifying
190 * If packet is going to be hardware checksummed, then
191 * do it now in software before we mangle it.
193 if (q->corrupt && q->corrupt >= get_crandom(&q->corrupt_cor)) {
194 if (!(skb = skb_unshare(skb, GFP_ATOMIC))
195 || (skb->ip_summed == CHECKSUM_PARTIAL
196 && skb_checksum_help(skb))) {
197 sch->qstats.drops++;
198 return NET_XMIT_DROP;
201 skb->data[net_random() % skb_headlen(skb)] ^= 1<<(net_random() % 8);
204 cb = (struct netem_skb_cb *)skb->cb;
205 if (q->gap == 0 /* not doing reordering */
206 || q->counter < q->gap /* inside last reordering gap */
207 || q->reorder < get_crandom(&q->reorder_cor)) {
208 psched_time_t now;
209 psched_tdiff_t delay;
211 delay = tabledist(q->latency, q->jitter,
212 &q->delay_cor, q->delay_dist);
214 PSCHED_GET_TIME(now);
215 PSCHED_TADD2(now, delay, cb->time_to_send);
216 ++q->counter;
217 ret = q->qdisc->enqueue(skb, q->qdisc);
218 } else {
220 * Do re-ordering by putting one out of N packets at the front
221 * of the queue.
223 PSCHED_GET_TIME(cb->time_to_send);
224 q->counter = 0;
225 ret = q->qdisc->ops->requeue(skb, q->qdisc);
228 if (likely(ret == NET_XMIT_SUCCESS)) {
229 sch->q.qlen++;
230 sch->bstats.bytes += skb->len;
231 sch->bstats.packets++;
232 } else
233 sch->qstats.drops++;
235 pr_debug("netem: enqueue ret %d\n", ret);
236 return ret;
239 /* Requeue packets but don't change time stamp */
240 static int netem_requeue(struct sk_buff *skb, struct Qdisc *sch)
242 struct netem_sched_data *q = qdisc_priv(sch);
243 int ret;
245 if ((ret = q->qdisc->ops->requeue(skb, q->qdisc)) == 0) {
246 sch->q.qlen++;
247 sch->qstats.requeues++;
250 return ret;
253 static unsigned int netem_drop(struct Qdisc* sch)
255 struct netem_sched_data *q = qdisc_priv(sch);
256 unsigned int len = 0;
258 if (q->qdisc->ops->drop && (len = q->qdisc->ops->drop(q->qdisc)) != 0) {
259 sch->q.qlen--;
260 sch->qstats.drops++;
262 return len;
265 static struct sk_buff *netem_dequeue(struct Qdisc *sch)
267 struct netem_sched_data *q = qdisc_priv(sch);
268 struct sk_buff *skb;
270 skb = q->qdisc->dequeue(q->qdisc);
271 if (skb) {
272 const struct netem_skb_cb *cb
273 = (const struct netem_skb_cb *)skb->cb;
274 psched_time_t now;
276 /* if more time remaining? */
277 PSCHED_GET_TIME(now);
279 if (PSCHED_TLESS(cb->time_to_send, now)) {
280 pr_debug("netem_dequeue: return skb=%p\n", skb);
281 sch->q.qlen--;
282 sch->flags &= ~TCQ_F_THROTTLED;
283 return skb;
284 } else {
285 psched_tdiff_t delay = PSCHED_TDIFF(cb->time_to_send, now);
287 if (q->qdisc->ops->requeue(skb, q->qdisc) != NET_XMIT_SUCCESS) {
288 sch->qstats.drops++;
290 /* After this qlen is confused */
291 printk(KERN_ERR "netem: queue discpline %s could not requeue\n",
292 q->qdisc->ops->id);
294 sch->q.qlen--;
297 mod_timer(&q->timer, jiffies + PSCHED_US2JIFFIE(delay));
298 sch->flags |= TCQ_F_THROTTLED;
302 return NULL;
305 static void netem_watchdog(unsigned long arg)
307 struct Qdisc *sch = (struct Qdisc *)arg;
309 pr_debug("netem_watchdog qlen=%d\n", sch->q.qlen);
310 sch->flags &= ~TCQ_F_THROTTLED;
311 netif_schedule(sch->dev);
314 static void netem_reset(struct Qdisc *sch)
316 struct netem_sched_data *q = qdisc_priv(sch);
318 qdisc_reset(q->qdisc);
319 sch->q.qlen = 0;
320 sch->flags &= ~TCQ_F_THROTTLED;
321 del_timer_sync(&q->timer);
324 /* Pass size change message down to embedded FIFO */
325 static int set_fifo_limit(struct Qdisc *q, int limit)
327 struct rtattr *rta;
328 int ret = -ENOMEM;
330 /* Hack to avoid sending change message to non-FIFO */
331 if (strncmp(q->ops->id + 1, "fifo", 4) != 0)
332 return 0;
334 rta = kmalloc(RTA_LENGTH(sizeof(struct tc_fifo_qopt)), GFP_KERNEL);
335 if (rta) {
336 rta->rta_type = RTM_NEWQDISC;
337 rta->rta_len = RTA_LENGTH(sizeof(struct tc_fifo_qopt));
338 ((struct tc_fifo_qopt *)RTA_DATA(rta))->limit = limit;
340 ret = q->ops->change(q, rta);
341 kfree(rta);
343 return ret;
347 * Distribution data is a variable size payload containing
348 * signed 16 bit values.
350 static int get_dist_table(struct Qdisc *sch, const struct rtattr *attr)
352 struct netem_sched_data *q = qdisc_priv(sch);
353 unsigned long n = RTA_PAYLOAD(attr)/sizeof(__s16);
354 const __s16 *data = RTA_DATA(attr);
355 struct disttable *d;
356 int i;
358 if (n > 65536)
359 return -EINVAL;
361 d = kmalloc(sizeof(*d) + n*sizeof(d->table[0]), GFP_KERNEL);
362 if (!d)
363 return -ENOMEM;
365 d->size = n;
366 for (i = 0; i < n; i++)
367 d->table[i] = data[i];
369 spin_lock_bh(&sch->dev->queue_lock);
370 d = xchg(&q->delay_dist, d);
371 spin_unlock_bh(&sch->dev->queue_lock);
373 kfree(d);
374 return 0;
377 static int get_correlation(struct Qdisc *sch, const struct rtattr *attr)
379 struct netem_sched_data *q = qdisc_priv(sch);
380 const struct tc_netem_corr *c = RTA_DATA(attr);
382 if (RTA_PAYLOAD(attr) != sizeof(*c))
383 return -EINVAL;
385 init_crandom(&q->delay_cor, c->delay_corr);
386 init_crandom(&q->loss_cor, c->loss_corr);
387 init_crandom(&q->dup_cor, c->dup_corr);
388 return 0;
391 static int get_reorder(struct Qdisc *sch, const struct rtattr *attr)
393 struct netem_sched_data *q = qdisc_priv(sch);
394 const struct tc_netem_reorder *r = RTA_DATA(attr);
396 if (RTA_PAYLOAD(attr) != sizeof(*r))
397 return -EINVAL;
399 q->reorder = r->probability;
400 init_crandom(&q->reorder_cor, r->correlation);
401 return 0;
404 static int get_corrupt(struct Qdisc *sch, const struct rtattr *attr)
406 struct netem_sched_data *q = qdisc_priv(sch);
407 const struct tc_netem_corrupt *r = RTA_DATA(attr);
409 if (RTA_PAYLOAD(attr) != sizeof(*r))
410 return -EINVAL;
412 q->corrupt = r->probability;
413 init_crandom(&q->corrupt_cor, r->correlation);
414 return 0;
417 /* Parse netlink message to set options */
418 static int netem_change(struct Qdisc *sch, struct rtattr *opt)
420 struct netem_sched_data *q = qdisc_priv(sch);
421 struct tc_netem_qopt *qopt;
422 int ret;
424 if (opt == NULL || RTA_PAYLOAD(opt) < sizeof(*qopt))
425 return -EINVAL;
427 qopt = RTA_DATA(opt);
428 ret = set_fifo_limit(q->qdisc, qopt->limit);
429 if (ret) {
430 pr_debug("netem: can't set fifo limit\n");
431 return ret;
434 q->latency = qopt->latency;
435 q->jitter = qopt->jitter;
436 q->limit = qopt->limit;
437 q->gap = qopt->gap;
438 q->counter = 0;
439 q->loss = qopt->loss;
440 q->duplicate = qopt->duplicate;
442 /* for compatiablity with earlier versions.
443 * if gap is set, need to assume 100% probablity
445 q->reorder = ~0;
447 /* Handle nested options after initial queue options.
448 * Should have put all options in nested format but too late now.
450 if (RTA_PAYLOAD(opt) > sizeof(*qopt)) {
451 struct rtattr *tb[TCA_NETEM_MAX];
452 if (rtattr_parse(tb, TCA_NETEM_MAX,
453 RTA_DATA(opt) + sizeof(*qopt),
454 RTA_PAYLOAD(opt) - sizeof(*qopt)))
455 return -EINVAL;
457 if (tb[TCA_NETEM_CORR-1]) {
458 ret = get_correlation(sch, tb[TCA_NETEM_CORR-1]);
459 if (ret)
460 return ret;
463 if (tb[TCA_NETEM_DELAY_DIST-1]) {
464 ret = get_dist_table(sch, tb[TCA_NETEM_DELAY_DIST-1]);
465 if (ret)
466 return ret;
469 if (tb[TCA_NETEM_REORDER-1]) {
470 ret = get_reorder(sch, tb[TCA_NETEM_REORDER-1]);
471 if (ret)
472 return ret;
475 if (tb[TCA_NETEM_CORRUPT-1]) {
476 ret = get_corrupt(sch, tb[TCA_NETEM_CORRUPT-1]);
477 if (ret)
478 return ret;
482 return 0;
486 * Special case version of FIFO queue for use by netem.
487 * It queues in order based on timestamps in skb's
489 struct fifo_sched_data {
490 u32 limit;
493 static int tfifo_enqueue(struct sk_buff *nskb, struct Qdisc *sch)
495 struct fifo_sched_data *q = qdisc_priv(sch);
496 struct sk_buff_head *list = &sch->q;
497 const struct netem_skb_cb *ncb
498 = (const struct netem_skb_cb *)nskb->cb;
499 struct sk_buff *skb;
501 if (likely(skb_queue_len(list) < q->limit)) {
502 skb_queue_reverse_walk(list, skb) {
503 const struct netem_skb_cb *cb
504 = (const struct netem_skb_cb *)skb->cb;
506 if (!PSCHED_TLESS(ncb->time_to_send, cb->time_to_send))
507 break;
510 __skb_queue_after(list, skb, nskb);
512 sch->qstats.backlog += nskb->len;
513 sch->bstats.bytes += nskb->len;
514 sch->bstats.packets++;
516 return NET_XMIT_SUCCESS;
519 return qdisc_drop(nskb, sch);
522 static int tfifo_init(struct Qdisc *sch, struct rtattr *opt)
524 struct fifo_sched_data *q = qdisc_priv(sch);
526 if (opt) {
527 struct tc_fifo_qopt *ctl = RTA_DATA(opt);
528 if (RTA_PAYLOAD(opt) < sizeof(*ctl))
529 return -EINVAL;
531 q->limit = ctl->limit;
532 } else
533 q->limit = max_t(u32, sch->dev->tx_queue_len, 1);
535 return 0;
538 static int tfifo_dump(struct Qdisc *sch, struct sk_buff *skb)
540 struct fifo_sched_data *q = qdisc_priv(sch);
541 struct tc_fifo_qopt opt = { .limit = q->limit };
543 RTA_PUT(skb, TCA_OPTIONS, sizeof(opt), &opt);
544 return skb->len;
546 rtattr_failure:
547 return -1;
550 static struct Qdisc_ops tfifo_qdisc_ops = {
551 .id = "tfifo",
552 .priv_size = sizeof(struct fifo_sched_data),
553 .enqueue = tfifo_enqueue,
554 .dequeue = qdisc_dequeue_head,
555 .requeue = qdisc_requeue,
556 .drop = qdisc_queue_drop,
557 .init = tfifo_init,
558 .reset = qdisc_reset_queue,
559 .change = tfifo_init,
560 .dump = tfifo_dump,
563 static int netem_init(struct Qdisc *sch, struct rtattr *opt)
565 struct netem_sched_data *q = qdisc_priv(sch);
566 int ret;
568 if (!opt)
569 return -EINVAL;
571 init_timer(&q->timer);
572 q->timer.function = netem_watchdog;
573 q->timer.data = (unsigned long) sch;
575 q->qdisc = qdisc_create_dflt(sch->dev, &tfifo_qdisc_ops);
576 if (!q->qdisc) {
577 pr_debug("netem: qdisc create failed\n");
578 return -ENOMEM;
581 ret = netem_change(sch, opt);
582 if (ret) {
583 pr_debug("netem: change failed\n");
584 qdisc_destroy(q->qdisc);
586 return ret;
589 static void netem_destroy(struct Qdisc *sch)
591 struct netem_sched_data *q = qdisc_priv(sch);
593 del_timer_sync(&q->timer);
594 qdisc_destroy(q->qdisc);
595 kfree(q->delay_dist);
598 static int netem_dump(struct Qdisc *sch, struct sk_buff *skb)
600 const struct netem_sched_data *q = qdisc_priv(sch);
601 unsigned char *b = skb->tail;
602 struct rtattr *rta = (struct rtattr *) b;
603 struct tc_netem_qopt qopt;
604 struct tc_netem_corr cor;
605 struct tc_netem_reorder reorder;
606 struct tc_netem_corrupt corrupt;
608 qopt.latency = q->latency;
609 qopt.jitter = q->jitter;
610 qopt.limit = q->limit;
611 qopt.loss = q->loss;
612 qopt.gap = q->gap;
613 qopt.duplicate = q->duplicate;
614 RTA_PUT(skb, TCA_OPTIONS, sizeof(qopt), &qopt);
616 cor.delay_corr = q->delay_cor.rho;
617 cor.loss_corr = q->loss_cor.rho;
618 cor.dup_corr = q->dup_cor.rho;
619 RTA_PUT(skb, TCA_NETEM_CORR, sizeof(cor), &cor);
621 reorder.probability = q->reorder;
622 reorder.correlation = q->reorder_cor.rho;
623 RTA_PUT(skb, TCA_NETEM_REORDER, sizeof(reorder), &reorder);
625 corrupt.probability = q->corrupt;
626 corrupt.correlation = q->corrupt_cor.rho;
627 RTA_PUT(skb, TCA_NETEM_CORRUPT, sizeof(corrupt), &corrupt);
629 rta->rta_len = skb->tail - b;
631 return skb->len;
633 rtattr_failure:
634 skb_trim(skb, b - skb->data);
635 return -1;
638 static int netem_dump_class(struct Qdisc *sch, unsigned long cl,
639 struct sk_buff *skb, struct tcmsg *tcm)
641 struct netem_sched_data *q = qdisc_priv(sch);
643 if (cl != 1) /* only one class */
644 return -ENOENT;
646 tcm->tcm_handle |= TC_H_MIN(1);
647 tcm->tcm_info = q->qdisc->handle;
649 return 0;
652 static int netem_graft(struct Qdisc *sch, unsigned long arg, struct Qdisc *new,
653 struct Qdisc **old)
655 struct netem_sched_data *q = qdisc_priv(sch);
657 if (new == NULL)
658 new = &noop_qdisc;
660 sch_tree_lock(sch);
661 *old = xchg(&q->qdisc, new);
662 qdisc_reset(*old);
663 sch->q.qlen = 0;
664 sch_tree_unlock(sch);
666 return 0;
669 static struct Qdisc *netem_leaf(struct Qdisc *sch, unsigned long arg)
671 struct netem_sched_data *q = qdisc_priv(sch);
672 return q->qdisc;
675 static unsigned long netem_get(struct Qdisc *sch, u32 classid)
677 return 1;
680 static void netem_put(struct Qdisc *sch, unsigned long arg)
684 static int netem_change_class(struct Qdisc *sch, u32 classid, u32 parentid,
685 struct rtattr **tca, unsigned long *arg)
687 return -ENOSYS;
690 static int netem_delete(struct Qdisc *sch, unsigned long arg)
692 return -ENOSYS;
695 static void netem_walk(struct Qdisc *sch, struct qdisc_walker *walker)
697 if (!walker->stop) {
698 if (walker->count >= walker->skip)
699 if (walker->fn(sch, 1, walker) < 0) {
700 walker->stop = 1;
701 return;
703 walker->count++;
707 static struct tcf_proto **netem_find_tcf(struct Qdisc *sch, unsigned long cl)
709 return NULL;
712 static struct Qdisc_class_ops netem_class_ops = {
713 .graft = netem_graft,
714 .leaf = netem_leaf,
715 .get = netem_get,
716 .put = netem_put,
717 .change = netem_change_class,
718 .delete = netem_delete,
719 .walk = netem_walk,
720 .tcf_chain = netem_find_tcf,
721 .dump = netem_dump_class,
724 static struct Qdisc_ops netem_qdisc_ops = {
725 .id = "netem",
726 .cl_ops = &netem_class_ops,
727 .priv_size = sizeof(struct netem_sched_data),
728 .enqueue = netem_enqueue,
729 .dequeue = netem_dequeue,
730 .requeue = netem_requeue,
731 .drop = netem_drop,
732 .init = netem_init,
733 .reset = netem_reset,
734 .destroy = netem_destroy,
735 .change = netem_change,
736 .dump = netem_dump,
737 .owner = THIS_MODULE,
741 static int __init netem_module_init(void)
743 pr_info("netem: version " VERSION "\n");
744 return register_qdisc(&netem_qdisc_ops);
746 static void __exit netem_module_exit(void)
748 unregister_qdisc(&netem_qdisc_ops);
750 module_init(netem_module_init)
751 module_exit(netem_module_exit)
752 MODULE_LICENSE("GPL");