MFC r1.6 r1.30 r1.28 (HEAD):
[dragonfly.git] / usr.sbin / pfctl / pfctl_altq.c
blob4587ebf3916f08da7d1c67c687d1bd13b979913c
1 /* $OpenBSD: pfctl_altq.c,v 1.83 2004/03/14 21:51:44 dhartmei Exp $ */
2 /* $DragonFly: src/usr.sbin/pfctl/pfctl_altq.c,v 1.3 2008/04/06 18:58:14 dillon Exp $ */
4 /*
5 * Copyright (c) 2002
6 * Sony Computer Science Laboratories Inc.
7 * Copyright (c) 2002, 2003 Henning Brauer <henning@openbsd.org>
9 * Permission to use, copy, modify, and distribute this software for any
10 * purpose with or without fee is hereby granted, provided that the above
11 * copyright notice and this permission notice appear in all copies.
13 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
14 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
15 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
16 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
17 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
18 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
19 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
22 #include <sys/param.h>
23 #include <sys/ioctl.h>
24 #include <sys/socket.h>
25 #include <sys/sysctl.h>
27 #include <net/if.h>
28 #include <net/if_mib.h>
29 #include <netinet/in.h>
30 #include <net/pf/pfvar.h>
32 #include <err.h>
33 #include <errno.h>
34 #include <limits.h>
35 #include <math.h>
36 #include <stdio.h>
37 #include <stdlib.h>
38 #include <string.h>
39 #include <unistd.h>
41 #include <net/altq/altq.h>
42 #include <net/altq/altq_cbq.h>
43 #include <net/altq/altq_priq.h>
44 #include <net/altq/altq_hfsc.h>
45 #include <net/altq/altq_fairq.h>
47 #include "pfctl_parser.h"
48 #include "pfctl.h"
50 #define is_sc_null(sc) (((sc) == NULL) || ((sc)->m1 == 0 && (sc)->m2 == 0))
52 TAILQ_HEAD(altqs, pf_altq) altqs = TAILQ_HEAD_INITIALIZER(altqs);
53 LIST_HEAD(gen_sc, segment) rtsc, lssc;
55 struct pf_altq *qname_to_pfaltq(const char *, const char *);
56 u_int32_t qname_to_qid(const char *);
58 static int eval_pfqueue_cbq(struct pfctl *, struct pf_altq *);
59 static int cbq_compute_idletime(struct pfctl *, struct pf_altq *);
60 static int check_commit_cbq(int, int, struct pf_altq *);
61 static int print_cbq_opts(const struct pf_altq *);
63 static int eval_pfqueue_priq(struct pfctl *, struct pf_altq *);
64 static int check_commit_priq(int, int, struct pf_altq *);
65 static int print_priq_opts(const struct pf_altq *);
67 static int eval_pfqueue_hfsc(struct pfctl *, struct pf_altq *);
68 static int check_commit_hfsc(int, int, struct pf_altq *);
69 static int print_hfsc_opts(const struct pf_altq *,
70 const struct node_queue_opt *);
72 static int eval_pfqueue_fairq(struct pfctl *, struct pf_altq *);
73 static int print_fairq_opts(const struct pf_altq *,
74 const struct node_queue_opt *);
75 static int check_commit_fairq(int, int, struct pf_altq *);
77 static void gsc_add_sc(struct gen_sc *, struct service_curve *);
78 static int is_gsc_under_sc(struct gen_sc *,
79 struct service_curve *);
80 static void gsc_destroy(struct gen_sc *);
81 static struct segment *gsc_getentry(struct gen_sc *, double);
82 static int gsc_add_seg(struct gen_sc *, double, double, double,
83 double);
84 static double sc_x2y(struct service_curve *, double);
86 u_int32_t getifspeed(const char *);
87 u_long getifmtu(char *);
88 int eval_queue_opts(struct pf_altq *, struct node_queue_opt *,
89 u_int32_t);
90 u_int32_t eval_bwspec(struct node_queue_bw *, u_int32_t);
91 void print_hfsc_sc(const char *, u_int, u_int, u_int,
92 const struct node_hfsc_sc *);
93 void print_fairq_sc(const char *, u_int, u_int, u_int,
94 const struct node_fairq_sc *);
96 void
97 pfaltq_store(struct pf_altq *a)
99 struct pf_altq *altq;
101 if ((altq = malloc(sizeof(*altq))) == NULL)
102 err(1, "malloc");
103 memcpy(altq, a, sizeof(struct pf_altq));
104 TAILQ_INSERT_TAIL(&altqs, altq, entries);
107 void
108 pfaltq_free(struct pf_altq *a)
110 struct pf_altq *altq;
112 TAILQ_FOREACH(altq, &altqs, entries) {
113 if (strncmp(a->ifname, altq->ifname, IFNAMSIZ) == 0 &&
114 strncmp(a->qname, altq->qname, PF_QNAME_SIZE) == 0) {
115 TAILQ_REMOVE(&altqs, altq, entries);
116 free(altq);
117 return;
122 struct pf_altq *
123 pfaltq_lookup(const char *ifname)
125 struct pf_altq *altq;
127 TAILQ_FOREACH(altq, &altqs, entries) {
128 if (strncmp(ifname, altq->ifname, IFNAMSIZ) == 0 &&
129 altq->qname[0] == 0)
130 return (altq);
132 return (NULL);
135 struct pf_altq *
136 qname_to_pfaltq(const char *qname, const char *ifname)
138 struct pf_altq *altq;
140 TAILQ_FOREACH(altq, &altqs, entries) {
141 if (strncmp(ifname, altq->ifname, IFNAMSIZ) == 0 &&
142 strncmp(qname, altq->qname, PF_QNAME_SIZE) == 0)
143 return (altq);
145 return (NULL);
148 u_int32_t
149 qname_to_qid(const char *qname)
151 struct pf_altq *altq;
154 * We guarantee that same named queues on different interfaces
155 * have the same qid, so we do NOT need to limit matching on
156 * one interface!
159 TAILQ_FOREACH(altq, &altqs, entries) {
160 if (strncmp(qname, altq->qname, PF_QNAME_SIZE) == 0)
161 return (altq->qid);
163 return (0);
166 void
167 print_altq(const struct pf_altq *a, unsigned level, struct node_queue_bw *bw,
168 struct node_queue_opt *qopts)
170 if (a->qname[0] != 0) {
171 print_queue(a, level, bw, 0, qopts);
172 return;
175 printf("altq on %s ", a->ifname);
177 switch (a->scheduler) {
178 case ALTQT_CBQ:
179 if (!print_cbq_opts(a))
180 printf("cbq ");
181 break;
182 case ALTQT_PRIQ:
183 if (!print_priq_opts(a))
184 printf("priq ");
185 break;
186 case ALTQT_HFSC:
187 if (!print_hfsc_opts(a, qopts))
188 printf("hfsc ");
189 break;
190 case ALTQT_FAIRQ:
191 if (!print_fairq_opts(a, qopts))
192 printf("hfsc ");
193 break;
196 if (bw != NULL && bw->bw_percent > 0) {
197 if (bw->bw_percent < 100)
198 printf("bandwidth %u%% ", bw->bw_percent);
199 } else
200 printf("bandwidth %s ", rate2str((double)a->ifbandwidth));
202 if (a->qlimit != DEFAULT_QLIMIT)
203 printf("qlimit %u ", a->qlimit);
204 printf("tbrsize %u ", a->tbrsize);
207 void
208 print_queue(const struct pf_altq *a, unsigned level, struct node_queue_bw *bw,
209 int print_interface, struct node_queue_opt *qopts)
211 unsigned i;
213 printf("queue ");
214 for (i = 0; i < level; ++i)
215 printf(" ");
216 printf("%s ", a->qname);
217 if (print_interface)
218 printf("on %s ", a->ifname);
219 if (a->scheduler == ALTQT_CBQ || a->scheduler == ALTQT_HFSC ||
220 a->scheduler == ALTQT_FAIRQ) {
221 if (bw != NULL && bw->bw_percent > 0) {
222 if (bw->bw_percent < 100)
223 printf("bandwidth %u%% ", bw->bw_percent);
224 } else
225 printf("bandwidth %s ", rate2str((double)a->bandwidth));
227 if (a->priority != DEFAULT_PRIORITY)
228 printf("priority %u ", a->priority);
229 if (a->qlimit != DEFAULT_QLIMIT)
230 printf("qlimit %u ", a->qlimit);
231 switch (a->scheduler) {
232 case ALTQT_CBQ:
233 print_cbq_opts(a);
234 break;
235 case ALTQT_PRIQ:
236 print_priq_opts(a);
237 break;
238 case ALTQT_HFSC:
239 print_hfsc_opts(a, qopts);
240 break;
241 case ALTQT_FAIRQ:
242 print_fairq_opts(a, qopts);
243 break;
248 * eval_pfaltq computes the discipline parameters.
251 eval_pfaltq(struct pfctl *pf __unused, struct pf_altq *pa,
252 struct node_queue_bw *bw, struct node_queue_opt *opts)
254 u_int rate, size, errors = 0;
256 if (bw->bw_absolute > 0)
257 pa->ifbandwidth = bw->bw_absolute;
258 else
259 if ((rate = getifspeed(pa->ifname)) == 0) {
260 fprintf(stderr, "cannot determine interface bandwidth "
261 "for %s, specify an absolute bandwidth\n",
262 pa->ifname);
263 errors++;
264 } else if ((pa->ifbandwidth = eval_bwspec(bw, rate)) == 0)
265 pa->ifbandwidth = rate;
267 errors += eval_queue_opts(pa, opts, pa->ifbandwidth);
269 /* if tbrsize is not specified, use heuristics */
270 if (pa->tbrsize == 0) {
271 rate = pa->ifbandwidth;
272 if (rate <= 1 * 1000 * 1000)
273 size = 1;
274 else if (rate <= 10 * 1000 * 1000)
275 size = 4;
276 else if (rate <= 200 * 1000 * 1000)
277 size = 8;
278 else
279 size = 24;
280 size = size * getifmtu(pa->ifname);
281 if (size > 0xffff)
282 size = 0xffff;
283 pa->tbrsize = size;
285 return (errors);
289 * check_commit_altq does consistency check for each interface
292 check_commit_altq(int dev, int opts)
294 struct pf_altq *altq;
295 int error = 0;
297 /* call the discipline check for each interface. */
298 TAILQ_FOREACH(altq, &altqs, entries) {
299 if (altq->qname[0] == 0) {
300 switch (altq->scheduler) {
301 case ALTQT_CBQ:
302 error = check_commit_cbq(dev, opts, altq);
303 break;
304 case ALTQT_PRIQ:
305 error = check_commit_priq(dev, opts, altq);
306 break;
307 case ALTQT_HFSC:
308 error = check_commit_hfsc(dev, opts, altq);
309 break;
310 case ALTQT_FAIRQ:
311 error = check_commit_fairq(dev, opts, altq);
312 break;
313 default:
314 break;
318 return (error);
322 * eval_pfqueue computes the queue parameters.
325 eval_pfqueue(struct pfctl *pf, struct pf_altq *pa, struct node_queue_bw *bw,
326 struct node_queue_opt *opts)
328 /* should be merged with expand_queue */
329 struct pf_altq *if_pa, *parent;
330 int error = 0;
332 /* find the corresponding interface and copy fields used by queues */
333 if ((if_pa = pfaltq_lookup(pa->ifname)) == NULL) {
334 fprintf(stderr, "altq not defined on %s\n", pa->ifname);
335 return (1);
337 pa->scheduler = if_pa->scheduler;
338 pa->ifbandwidth = if_pa->ifbandwidth;
340 if (qname_to_pfaltq(pa->qname, pa->ifname) != NULL) {
341 fprintf(stderr, "queue %s already exists on interface %s\n",
342 pa->qname, pa->ifname);
343 return (1);
345 pa->qid = qname_to_qid(pa->qname);
347 parent = NULL;
348 if (pa->parent[0] != 0) {
349 parent = qname_to_pfaltq(pa->parent, pa->ifname);
350 if (parent == NULL) {
351 fprintf(stderr, "parent %s not found for %s\n",
352 pa->parent, pa->qname);
353 return (1);
355 pa->parent_qid = parent->qid;
357 if (pa->qlimit == 0)
358 pa->qlimit = DEFAULT_QLIMIT;
360 if (pa->scheduler == ALTQT_CBQ || pa->scheduler == ALTQT_HFSC ||
361 pa->scheduler == ALTQT_FAIRQ) {
362 if ((pa->bandwidth = eval_bwspec(bw,
363 parent == NULL ? 0 : parent->bandwidth)) == 0) {
364 fprintf(stderr, "bandwidth for %s invalid (%d / %d)\n",
365 pa->qname, bw->bw_absolute, bw->bw_percent);
366 return (1);
369 if (pa->bandwidth > pa->ifbandwidth) {
370 fprintf(stderr, "bandwidth for %s higher than "
371 "interface\n", pa->qname);
372 return (1);
374 if (parent != NULL && pa->bandwidth > parent->bandwidth) {
375 fprintf(stderr, "bandwidth for %s higher than parent\n",
376 pa->qname);
377 return (1);
381 if (eval_queue_opts(pa, opts, parent == NULL? 0 : parent->bandwidth))
382 return (1);
384 switch (pa->scheduler) {
385 case ALTQT_CBQ:
386 error = eval_pfqueue_cbq(pf, pa);
387 break;
388 case ALTQT_PRIQ:
389 error = eval_pfqueue_priq(pf, pa);
390 break;
391 case ALTQT_HFSC:
392 error = eval_pfqueue_hfsc(pf, pa);
393 break;
394 case ALTQT_FAIRQ:
395 error = eval_pfqueue_fairq(pf, pa);
396 break;
397 default:
398 break;
400 return (error);
404 * CBQ support functions
406 #define RM_FILTER_GAIN 5 /* log2 of gain, e.g., 5 => 31/32 */
407 #define RM_NS_PER_SEC (1000000000)
409 static int
410 eval_pfqueue_cbq(struct pfctl *pf, struct pf_altq *pa)
412 struct cbq_opts *opts;
413 u_int ifmtu;
415 if (pa->priority >= CBQ_MAXPRI) {
416 warnx("priority out of range: max %d", CBQ_MAXPRI - 1);
417 return (-1);
420 ifmtu = getifmtu(pa->ifname);
421 opts = &pa->pq_u.cbq_opts;
423 if (opts->pktsize == 0) { /* use default */
424 opts->pktsize = ifmtu;
425 if (opts->pktsize > MCLBYTES) /* do what TCP does */
426 opts->pktsize &= ~MCLBYTES;
427 } else if (opts->pktsize > ifmtu)
428 opts->pktsize = ifmtu;
429 if (opts->maxpktsize == 0) /* use default */
430 opts->maxpktsize = ifmtu;
431 else if (opts->maxpktsize > ifmtu)
432 opts->pktsize = ifmtu;
434 if (opts->pktsize > opts->maxpktsize)
435 opts->pktsize = opts->maxpktsize;
437 if (pa->parent[0] == 0)
438 opts->flags |= (CBQCLF_ROOTCLASS | CBQCLF_WRR);
440 cbq_compute_idletime(pf, pa);
441 return (0);
445 * compute ns_per_byte, maxidle, minidle, and offtime
447 static int
448 cbq_compute_idletime(struct pfctl *pf, struct pf_altq *pa)
450 struct cbq_opts *opts;
451 double maxidle_s, maxidle, minidle;
452 double offtime, nsPerByte, ifnsPerByte, ptime, cptime;
453 double z, g, f, gton, gtom;
454 u_int minburst, maxburst;
456 opts = &pa->pq_u.cbq_opts;
457 ifnsPerByte = (1.0 / (double)pa->ifbandwidth) * RM_NS_PER_SEC * 8;
458 minburst = opts->minburst;
459 maxburst = opts->maxburst;
461 if (pa->bandwidth == 0)
462 f = 0.0001; /* small enough? */
463 else
464 f = ((double) pa->bandwidth / (double) pa->ifbandwidth);
466 nsPerByte = ifnsPerByte / f;
467 ptime = (double)opts->pktsize * ifnsPerByte;
468 cptime = ptime * (1.0 - f) / f;
470 if (nsPerByte * (double)opts->maxpktsize > (double)INT_MAX) {
472 * this causes integer overflow in kernel!
473 * (bandwidth < 6Kbps when max_pkt_size=1500)
475 if (pa->bandwidth != 0 && (pf->opts & PF_OPT_QUIET) == 0)
476 warnx("queue bandwidth must be larger than %s",
477 rate2str(ifnsPerByte * (double)opts->maxpktsize /
478 (double)INT_MAX * (double)pa->ifbandwidth));
479 fprintf(stderr, "cbq: queue %s is too slow!\n",
480 pa->qname);
481 nsPerByte = (double)(INT_MAX / opts->maxpktsize);
484 if (maxburst == 0) { /* use default */
485 if (cptime > 10.0 * 1000000)
486 maxburst = 4;
487 else
488 maxburst = 16;
490 if (minburst == 0) /* use default */
491 minburst = 2;
492 if (minburst > maxburst)
493 minburst = maxburst;
495 z = (double)(1 << RM_FILTER_GAIN);
496 g = (1.0 - 1.0 / z);
497 gton = pow(g, (double)maxburst);
498 gtom = pow(g, (double)(minburst-1));
499 maxidle = ((1.0 / f - 1.0) * ((1.0 - gton) / gton));
500 maxidle_s = (1.0 - g);
501 if (maxidle > maxidle_s)
502 maxidle = ptime * maxidle;
503 else
504 maxidle = ptime * maxidle_s;
505 if (minburst)
506 offtime = cptime * (1.0 + 1.0/(1.0 - g) * (1.0 - gtom) / gtom);
507 else
508 offtime = cptime;
509 minidle = -((double)opts->maxpktsize * (double)nsPerByte);
511 /* scale parameters */
512 maxidle = ((maxidle * 8.0) / nsPerByte) *
513 pow(2.0, (double)RM_FILTER_GAIN);
514 offtime = (offtime * 8.0) / nsPerByte *
515 pow(2.0, (double)RM_FILTER_GAIN);
516 minidle = ((minidle * 8.0) / nsPerByte) *
517 pow(2.0, (double)RM_FILTER_GAIN);
519 maxidle = maxidle / 1000.0;
520 offtime = offtime / 1000.0;
521 minidle = minidle / 1000.0;
523 opts->minburst = minburst;
524 opts->maxburst = maxburst;
525 opts->ns_per_byte = (u_int)nsPerByte;
526 opts->maxidle = (u_int)fabs(maxidle);
527 opts->minidle = (int)minidle;
528 opts->offtime = (u_int)fabs(offtime);
530 return (0);
533 static int
534 check_commit_cbq(int dev __unused, int opts __unused, struct pf_altq *pa)
536 struct pf_altq *altq;
537 int root_class, default_class;
538 int error = 0;
541 * check if cbq has one root queue and one default queue
542 * for this interface
544 root_class = default_class = 0;
545 TAILQ_FOREACH(altq, &altqs, entries) {
546 if (strncmp(altq->ifname, pa->ifname, IFNAMSIZ) != 0)
547 continue;
548 if (altq->qname[0] == 0) /* this is for interface */
549 continue;
550 if (altq->pq_u.cbq_opts.flags & CBQCLF_ROOTCLASS)
551 root_class++;
552 if (altq->pq_u.cbq_opts.flags & CBQCLF_DEFCLASS)
553 default_class++;
555 if (root_class != 1) {
556 warnx("should have one root queue on %s", pa->ifname);
557 error++;
559 if (default_class != 1) {
560 warnx("should have one default queue on %s", pa->ifname);
561 error++;
563 return (error);
566 static int
567 print_cbq_opts(const struct pf_altq *a)
569 const struct cbq_opts *opts;
571 opts = &a->pq_u.cbq_opts;
572 if (opts->flags) {
573 printf("cbq(");
574 if (opts->flags & CBQCLF_RED)
575 printf(" red");
576 if (opts->flags & CBQCLF_ECN)
577 printf(" ecn");
578 if (opts->flags & CBQCLF_RIO)
579 printf(" rio");
580 if (opts->flags & CBQCLF_CLEARDSCP)
581 printf(" cleardscp");
582 if (opts->flags & CBQCLF_BORROW)
583 printf(" borrow");
584 if (opts->flags & CBQCLF_WRR)
585 printf(" wrr");
586 if (opts->flags & CBQCLF_EFFICIENT)
587 printf(" efficient");
588 if (opts->flags & CBQCLF_ROOTCLASS)
589 printf(" root");
590 if (opts->flags & CBQCLF_DEFCLASS)
591 printf(" default");
592 printf(" ) ");
594 return (1);
595 } else
596 return (0);
600 * PRIQ support functions
602 static int
603 eval_pfqueue_priq(struct pfctl *pf __unused, struct pf_altq *pa)
605 struct pf_altq *altq;
607 if (pa->priority >= PRIQ_MAXPRI) {
608 warnx("priority out of range: max %d", PRIQ_MAXPRI - 1);
609 return (-1);
611 /* the priority should be unique for the interface */
612 TAILQ_FOREACH(altq, &altqs, entries) {
613 if (strncmp(altq->ifname, pa->ifname, IFNAMSIZ) == 0 &&
614 altq->qname[0] != 0 && altq->priority == pa->priority) {
615 warnx("%s and %s have the same priority",
616 altq->qname, pa->qname);
617 return (-1);
621 return (0);
624 static int
625 check_commit_priq(int dev __unused, int opts __unused, struct pf_altq *pa)
627 struct pf_altq *altq;
628 int default_class;
629 int error = 0;
632 * check if priq has one default class for this interface
634 default_class = 0;
635 TAILQ_FOREACH(altq, &altqs, entries) {
636 if (strncmp(altq->ifname, pa->ifname, IFNAMSIZ) != 0)
637 continue;
638 if (altq->qname[0] == 0) /* this is for interface */
639 continue;
640 if (altq->pq_u.priq_opts.flags & PRCF_DEFAULTCLASS)
641 default_class++;
643 if (default_class != 1) {
644 warnx("should have one default queue on %s", pa->ifname);
645 error++;
647 return (error);
650 static int
651 print_priq_opts(const struct pf_altq *a)
653 const struct priq_opts *opts;
655 opts = &a->pq_u.priq_opts;
657 if (opts->flags) {
658 printf("priq(");
659 if (opts->flags & PRCF_RED)
660 printf(" red");
661 if (opts->flags & PRCF_ECN)
662 printf(" ecn");
663 if (opts->flags & PRCF_RIO)
664 printf(" rio");
665 if (opts->flags & PRCF_CLEARDSCP)
666 printf(" cleardscp");
667 if (opts->flags & PRCF_DEFAULTCLASS)
668 printf(" default");
669 printf(" ) ");
671 return (1);
672 } else
673 return (0);
677 * HFSC support functions
679 static int
680 eval_pfqueue_hfsc(struct pfctl *pf __unused, struct pf_altq *pa)
682 struct pf_altq *altq, *parent;
683 struct hfsc_opts *opts;
684 struct service_curve sc;
686 opts = &pa->pq_u.hfsc_opts;
688 if (pa->parent[0] == 0) {
689 /* root queue */
690 opts->lssc_m1 = pa->ifbandwidth;
691 opts->lssc_m2 = pa->ifbandwidth;
692 opts->lssc_d = 0;
693 return (0);
696 LIST_INIT(&rtsc);
697 LIST_INIT(&lssc);
699 /* if link_share is not specified, use bandwidth */
700 if (opts->lssc_m2 == 0)
701 opts->lssc_m2 = pa->bandwidth;
703 if ((opts->rtsc_m1 > 0 && opts->rtsc_m2 == 0) ||
704 (opts->lssc_m1 > 0 && opts->lssc_m2 == 0) ||
705 (opts->ulsc_m1 > 0 && opts->ulsc_m2 == 0)) {
706 warnx("m2 is zero for %s", pa->qname);
707 return (-1);
710 if ((opts->rtsc_m1 < opts->rtsc_m2 && opts->rtsc_m1 != 0) ||
711 (opts->rtsc_m1 < opts->rtsc_m2 && opts->rtsc_m1 != 0) ||
712 (opts->rtsc_m1 < opts->rtsc_m2 && opts->rtsc_m1 != 0)) {
713 warnx("m1 must be zero for convex curve: %s", pa->qname);
714 return (-1);
718 * admission control:
719 * for the real-time service curve, the sum of the service curves
720 * should not exceed 80% of the interface bandwidth. 20% is reserved
721 * not to over-commit the actual interface bandwidth.
722 * for the link-sharing service curve, the sum of the child service
723 * curve should not exceed the parent service curve.
724 * for the upper-limit service curve, the assigned bandwidth should
725 * be smaller than the interface bandwidth, and the upper-limit should
726 * be larger than the real-time service curve when both are defined.
728 parent = qname_to_pfaltq(pa->parent, pa->ifname);
729 if (parent == NULL)
730 errx(1, "parent %s not found for %s", pa->parent, pa->qname);
732 TAILQ_FOREACH(altq, &altqs, entries) {
733 if (strncmp(altq->ifname, pa->ifname, IFNAMSIZ) != 0)
734 continue;
735 if (altq->qname[0] == 0) /* this is for interface */
736 continue;
738 /* if the class has a real-time service curve, add it. */
739 if (opts->rtsc_m2 != 0 && altq->pq_u.hfsc_opts.rtsc_m2 != 0) {
740 sc.m1 = altq->pq_u.hfsc_opts.rtsc_m1;
741 sc.d = altq->pq_u.hfsc_opts.rtsc_d;
742 sc.m2 = altq->pq_u.hfsc_opts.rtsc_m2;
743 gsc_add_sc(&rtsc, &sc);
746 if (strncmp(altq->parent, pa->parent, PF_QNAME_SIZE) != 0)
747 continue;
749 /* if the class has a link-sharing service curve, add it. */
750 if (opts->lssc_m2 != 0 && altq->pq_u.hfsc_opts.lssc_m2 != 0) {
751 sc.m1 = altq->pq_u.hfsc_opts.lssc_m1;
752 sc.d = altq->pq_u.hfsc_opts.lssc_d;
753 sc.m2 = altq->pq_u.hfsc_opts.lssc_m2;
754 gsc_add_sc(&lssc, &sc);
758 /* check the real-time service curve. reserve 20% of interface bw */
759 if (opts->rtsc_m2 != 0) {
760 sc.m1 = 0;
761 sc.d = 0;
762 sc.m2 = pa->ifbandwidth / 100 * 80;
763 if (!is_gsc_under_sc(&rtsc, &sc)) {
764 warnx("real-time sc exceeds the interface bandwidth");
765 goto err_ret;
769 /* check the link-sharing service curve. */
770 if (opts->lssc_m2 != 0) {
771 sc.m1 = parent->pq_u.hfsc_opts.lssc_m1;
772 sc.d = parent->pq_u.hfsc_opts.lssc_d;
773 sc.m2 = parent->pq_u.hfsc_opts.lssc_m2;
774 if (!is_gsc_under_sc(&lssc, &sc)) {
775 warnx("link-sharing sc exceeds parent's sc");
776 goto err_ret;
780 /* check the upper-limit service curve. */
781 if (opts->ulsc_m2 != 0) {
782 if (opts->ulsc_m1 > pa->ifbandwidth ||
783 opts->ulsc_m2 > pa->ifbandwidth) {
784 warnx("upper-limit larger than interface bandwidth");
785 goto err_ret;
787 if (opts->rtsc_m2 != 0 && opts->rtsc_m2 > opts->ulsc_m2) {
788 warnx("upper-limit sc smaller than real-time sc");
789 goto err_ret;
793 gsc_destroy(&rtsc);
794 gsc_destroy(&lssc);
796 return (0);
798 err_ret:
799 gsc_destroy(&rtsc);
800 gsc_destroy(&lssc);
801 return (-1);
805 * FAIRQ support functions
807 static int
808 eval_pfqueue_fairq(struct pfctl *pf __unused, struct pf_altq *pa)
810 struct pf_altq *altq, *parent;
811 struct fairq_opts *opts;
812 struct service_curve sc;
814 opts = &pa->pq_u.fairq_opts;
816 if (pa->parent[0] == 0) {
817 /* root queue */
818 opts->lssc_m1 = pa->ifbandwidth;
819 opts->lssc_m2 = pa->ifbandwidth;
820 opts->lssc_d = 0;
821 return (0);
824 LIST_INIT(&lssc);
826 /* if link_share is not specified, use bandwidth */
827 if (opts->lssc_m2 == 0)
828 opts->lssc_m2 = pa->bandwidth;
831 * admission control:
832 * for the real-time service curve, the sum of the service curves
833 * should not exceed 80% of the interface bandwidth. 20% is reserved
834 * not to over-commit the actual interface bandwidth.
835 * for the link-sharing service curve, the sum of the child service
836 * curve should not exceed the parent service curve.
837 * for the upper-limit service curve, the assigned bandwidth should
838 * be smaller than the interface bandwidth, and the upper-limit should
839 * be larger than the real-time service curve when both are defined.
841 parent = qname_to_pfaltq(pa->parent, pa->ifname);
842 if (parent == NULL)
843 errx(1, "parent %s not found for %s", pa->parent, pa->qname);
845 TAILQ_FOREACH(altq, &altqs, entries) {
846 if (strncmp(altq->ifname, pa->ifname, IFNAMSIZ) != 0)
847 continue;
848 if (altq->qname[0] == 0) /* this is for interface */
849 continue;
851 if (strncmp(altq->parent, pa->parent, PF_QNAME_SIZE) != 0)
852 continue;
854 /* if the class has a link-sharing service curve, add it. */
855 if (opts->lssc_m2 != 0 && altq->pq_u.fairq_opts.lssc_m2 != 0) {
856 sc.m1 = altq->pq_u.fairq_opts.lssc_m1;
857 sc.d = altq->pq_u.fairq_opts.lssc_d;
858 sc.m2 = altq->pq_u.fairq_opts.lssc_m2;
859 gsc_add_sc(&lssc, &sc);
863 /* check the link-sharing service curve. */
864 if (opts->lssc_m2 != 0) {
865 sc.m1 = parent->pq_u.fairq_opts.lssc_m1;
866 sc.d = parent->pq_u.fairq_opts.lssc_d;
867 sc.m2 = parent->pq_u.fairq_opts.lssc_m2;
868 if (!is_gsc_under_sc(&lssc, &sc)) {
869 warnx("link-sharing sc exceeds parent's sc");
870 goto err_ret;
874 gsc_destroy(&lssc);
876 return (0);
878 err_ret:
879 gsc_destroy(&lssc);
880 return (-1);
883 static int
884 check_commit_hfsc(int dev __unused, int opts __unused, struct pf_altq *pa)
886 struct pf_altq *altq, *def = NULL;
887 int default_class;
888 int error = 0;
890 /* check if hfsc has one default queue for this interface */
891 default_class = 0;
892 TAILQ_FOREACH(altq, &altqs, entries) {
893 if (strncmp(altq->ifname, pa->ifname, IFNAMSIZ) != 0)
894 continue;
895 if (altq->qname[0] == 0) /* this is for interface */
896 continue;
897 if (altq->parent[0] == 0) /* dummy root */
898 continue;
899 if (altq->pq_u.hfsc_opts.flags & HFCF_DEFAULTCLASS) {
900 default_class++;
901 def = altq;
904 if (default_class != 1) {
905 warnx("should have one default queue on %s", pa->ifname);
906 return (1);
908 /* make sure the default queue is a leaf */
909 TAILQ_FOREACH(altq, &altqs, entries) {
910 if (strncmp(altq->ifname, pa->ifname, IFNAMSIZ) != 0)
911 continue;
912 if (altq->qname[0] == 0) /* this is for interface */
913 continue;
914 if (strncmp(altq->parent, def->qname, PF_QNAME_SIZE) == 0) {
915 warnx("default queue is not a leaf");
916 error++;
919 return (error);
922 static int
923 check_commit_fairq(int dev __unused, int opts __unused, struct pf_altq *pa)
925 struct pf_altq *altq, *def = NULL;
926 int default_class;
927 int error = 0;
929 /* check if fairq has one default queue for this interface */
930 default_class = 0;
931 TAILQ_FOREACH(altq, &altqs, entries) {
932 if (strncmp(altq->ifname, pa->ifname, IFNAMSIZ) != 0)
933 continue;
934 if (altq->qname[0] == 0) /* this is for interface */
935 continue;
936 if (altq->pq_u.fairq_opts.flags & FARF_DEFAULTCLASS) {
937 default_class++;
938 def = altq;
941 if (default_class != 1) {
942 warnx("should have one default queue on %s", pa->ifname);
943 return (1);
945 /* make sure the default queue is a leaf */
946 TAILQ_FOREACH(altq, &altqs, entries) {
947 if (strncmp(altq->ifname, pa->ifname, IFNAMSIZ) != 0)
948 continue;
949 if (altq->qname[0] == 0) /* this is for interface */
950 continue;
951 if (strncmp(altq->parent, def->qname, PF_QNAME_SIZE) == 0) {
952 warnx("default queue is not a leaf");
953 error++;
956 return (error);
959 static int
960 print_hfsc_opts(const struct pf_altq *a, const struct node_queue_opt *qopts)
962 const struct hfsc_opts *opts;
963 const struct node_hfsc_sc *loc_rtsc, *loc_lssc, *ulsc;
965 opts = &a->pq_u.hfsc_opts;
966 if (qopts == NULL)
967 loc_rtsc = loc_lssc = ulsc = NULL;
968 else {
969 loc_rtsc = &qopts->data.hfsc_opts.realtime;
970 loc_lssc = &qopts->data.hfsc_opts.linkshare;
971 ulsc = &qopts->data.hfsc_opts.upperlimit;
974 if (opts->flags || opts->rtsc_m2 != 0 || opts->ulsc_m2 != 0 ||
975 (opts->lssc_m2 != 0 && (opts->lssc_m2 != a->bandwidth ||
976 opts->lssc_d != 0))) {
977 printf("hfsc(");
978 if (opts->flags & HFCF_RED)
979 printf(" red");
980 if (opts->flags & HFCF_ECN)
981 printf(" ecn");
982 if (opts->flags & HFCF_RIO)
983 printf(" rio");
984 if (opts->flags & HFCF_CLEARDSCP)
985 printf(" cleardscp");
986 if (opts->flags & HFCF_DEFAULTCLASS)
987 printf(" default");
988 if (opts->rtsc_m2 != 0)
989 print_hfsc_sc("realtime", opts->rtsc_m1, opts->rtsc_d,
990 opts->rtsc_m2, loc_rtsc);
991 if (opts->lssc_m2 != 0 && (opts->lssc_m2 != a->bandwidth ||
992 opts->lssc_d != 0))
993 print_hfsc_sc("linkshare", opts->lssc_m1, opts->lssc_d,
994 opts->lssc_m2, loc_lssc);
995 if (opts->ulsc_m2 != 0)
996 print_hfsc_sc("upperlimit", opts->ulsc_m1, opts->ulsc_d,
997 opts->ulsc_m2, ulsc);
998 printf(" ) ");
1000 return (1);
1001 } else
1002 return (0);
1005 static int
1006 print_fairq_opts(const struct pf_altq *a, const struct node_queue_opt *qopts)
1008 const struct fairq_opts *opts;
1009 const struct node_fairq_sc *loc_lssc;
1011 opts = &a->pq_u.fairq_opts;
1012 if (qopts == NULL)
1013 loc_lssc = NULL;
1014 else
1015 loc_lssc = &qopts->data.fairq_opts.linkshare;
1017 if (opts->flags ||
1018 (opts->lssc_m2 != 0 && (opts->lssc_m2 != a->bandwidth ||
1019 opts->lssc_d != 0))) {
1020 printf("fairq(");
1021 if (opts->flags & FARF_RED)
1022 printf(" red");
1023 if (opts->flags & FARF_ECN)
1024 printf(" ecn");
1025 if (opts->flags & FARF_RIO)
1026 printf(" rio");
1027 if (opts->flags & FARF_CLEARDSCP)
1028 printf(" cleardscp");
1029 if (opts->flags & FARF_DEFAULTCLASS)
1030 printf(" default");
1031 if (opts->lssc_m2 != 0 && (opts->lssc_m2 != a->bandwidth ||
1032 opts->lssc_d != 0))
1033 print_fairq_sc("linkshare", opts->lssc_m1, opts->lssc_d,
1034 opts->lssc_m2, loc_lssc);
1035 printf(" ) ");
1037 return (1);
1038 } else
1039 return (0);
1043 * admission control using generalized service curve
1045 #define INFINITY HUGE_VAL /* positive infinity defined in <math.h> */
1047 /* add a new service curve to a generalized service curve */
1048 static void
1049 gsc_add_sc(struct gen_sc *gsc, struct service_curve *sc)
1051 if (is_sc_null(sc))
1052 return;
1053 if (sc->d != 0)
1054 gsc_add_seg(gsc, 0.0, 0.0, (double)sc->d, (double)sc->m1);
1055 gsc_add_seg(gsc, (double)sc->d, 0.0, INFINITY, (double)sc->m2);
1059 * check whether all points of a generalized service curve have
1060 * their y-coordinates no larger than a given two-piece linear
1061 * service curve.
1063 static int
1064 is_gsc_under_sc(struct gen_sc *gsc, struct service_curve *sc)
1066 struct segment *s, *last, *end;
1067 double y;
1069 if (is_sc_null(sc)) {
1070 if (LIST_EMPTY(gsc))
1071 return (1);
1072 LIST_FOREACH(s, gsc, _next) {
1073 if (s->m != 0)
1074 return (0);
1076 return (1);
1079 * gsc has a dummy entry at the end with x = INFINITY.
1080 * loop through up to this dummy entry.
1082 end = gsc_getentry(gsc, INFINITY);
1083 if (end == NULL)
1084 return (1);
1085 last = NULL;
1086 for (s = LIST_FIRST(gsc); s != end; s = LIST_NEXT(s, _next)) {
1087 if (s->y > sc_x2y(sc, s->x))
1088 return (0);
1089 last = s;
1091 /* last now holds the real last segment */
1092 if (last == NULL)
1093 return (1);
1094 if (last->m > sc->m2)
1095 return (0);
1096 if (last->x < sc->d && last->m > sc->m1) {
1097 y = last->y + (sc->d - last->x) * last->m;
1098 if (y > sc_x2y(sc, sc->d))
1099 return (0);
1101 return (1);
1104 static void
1105 gsc_destroy(struct gen_sc *gsc)
1107 struct segment *s;
1109 while ((s = LIST_FIRST(gsc)) != NULL) {
1110 LIST_REMOVE(s, _next);
1111 free(s);
1116 * return a segment entry starting at x.
1117 * if gsc has no entry starting at x, a new entry is created at x.
1119 static struct segment *
1120 gsc_getentry(struct gen_sc *gsc, double x)
1122 struct segment *new, *prev, *s;
1124 prev = NULL;
1125 LIST_FOREACH(s, gsc, _next) {
1126 if (s->x == x)
1127 return (s); /* matching entry found */
1128 else if (s->x < x)
1129 prev = s;
1130 else
1131 break;
1134 /* we have to create a new entry */
1135 if ((new = calloc(1, sizeof(struct segment))) == NULL)
1136 return (NULL);
1138 new->x = x;
1139 if (x == INFINITY || s == NULL)
1140 new->d = 0;
1141 else if (s->x == INFINITY)
1142 new->d = INFINITY;
1143 else
1144 new->d = s->x - x;
1145 if (prev == NULL) {
1146 /* insert the new entry at the head of the list */
1147 new->y = 0;
1148 new->m = 0;
1149 LIST_INSERT_HEAD(gsc, new, _next);
1150 } else {
1152 * the start point intersects with the segment pointed by
1153 * prev. divide prev into 2 segments
1155 if (x == INFINITY) {
1156 prev->d = INFINITY;
1157 if (prev->m == 0)
1158 new->y = prev->y;
1159 else
1160 new->y = INFINITY;
1161 } else {
1162 prev->d = x - prev->x;
1163 new->y = prev->d * prev->m + prev->y;
1165 new->m = prev->m;
1166 LIST_INSERT_AFTER(prev, new, _next);
1168 return (new);
1171 /* add a segment to a generalized service curve */
1172 static int
1173 gsc_add_seg(struct gen_sc *gsc, double x, double y, double d, double m)
1175 struct segment *start, *end, *s;
1176 double x2;
1178 if (d == INFINITY)
1179 x2 = INFINITY;
1180 else
1181 x2 = x + d;
1182 start = gsc_getentry(gsc, x);
1183 end = gsc_getentry(gsc, x2);
1184 if (start == NULL || end == NULL)
1185 return (-1);
1187 for (s = start; s != end; s = LIST_NEXT(s, _next)) {
1188 s->m += m;
1189 s->y += y + (s->x - x) * m;
1192 end = gsc_getentry(gsc, INFINITY);
1193 for (; s != end; s = LIST_NEXT(s, _next)) {
1194 s->y += m * d;
1197 return (0);
1200 /* get y-projection of a service curve */
1201 static double
1202 sc_x2y(struct service_curve *sc, double x)
1204 double y;
1206 if (x <= (double)sc->d)
1207 /* y belongs to the 1st segment */
1208 y = x * (double)sc->m1;
1209 else
1210 /* y belongs to the 2nd segment */
1211 y = (double)sc->d * (double)sc->m1
1212 + (x - (double)sc->d) * (double)sc->m2;
1213 return (y);
1217 * misc utilities
1219 #define R2S_BUFS 8
1220 #define RATESTR_MAX 16
1222 char *
1223 rate2str(double rate)
1225 char *buf;
1226 static char r2sbuf[R2S_BUFS][RATESTR_MAX]; /* ring bufer */
1227 static int idx = 0;
1228 int i;
1229 static const char unit[] = " KMG";
1231 buf = r2sbuf[idx++];
1232 if (idx == R2S_BUFS)
1233 idx = 0;
1235 for (i = 0; rate >= 1000 && i <= 3; i++)
1236 rate /= 1000;
1238 if ((int)(rate * 100) % 100)
1239 snprintf(buf, RATESTR_MAX, "%.2f%cb", rate, unit[i]);
1240 else
1241 snprintf(buf, RATESTR_MAX, "%d%cb", (int)rate, unit[i]);
1243 return (buf);
1246 u_int32_t
1247 getifspeed(const char *ifname)
1249 size_t datalen;
1250 int idx;
1251 struct ifmibdata data;
1252 int name[] = {
1253 CTL_NET,
1254 PF_LINK,
1255 NETLINK_GENERIC,
1256 IFMIB_IFDATA,
1258 IFDATA_GENERAL
1261 if ((idx = (int)if_nametoindex(ifname)) == 0)
1262 err(1, "getifspeed: if_nametoindex");
1263 name[4] = idx;
1265 datalen = sizeof(data);
1266 if (sysctl(name, 6, &data, &datalen, NULL, 0))
1267 err(1, "getifspeed: sysctl");
1269 return(data.ifmd_data.ifi_baudrate);
1272 u_long
1273 getifmtu(char *ifname)
1275 int s;
1276 struct ifreq ifr;
1278 if ((s = socket(AF_INET, SOCK_DGRAM, 0)) < 0)
1279 err(1, "socket");
1280 if (strlcpy(ifr.ifr_name, ifname, sizeof(ifr.ifr_name)) >=
1281 sizeof(ifr.ifr_name))
1282 errx(1, "getifmtu: strlcpy");
1283 if (ioctl(s, SIOCGIFMTU, (caddr_t)&ifr) == -1)
1284 err(1, "SIOCGIFMTU");
1285 if (shutdown(s, SHUT_RDWR) == -1)
1286 err(1, "shutdown");
1287 if (close(s))
1288 err(1, "close");
1289 if (ifr.ifr_mtu > 0)
1290 return (ifr.ifr_mtu);
1291 else {
1292 warnx("could not get mtu for %s, assuming 1500", ifname);
1293 return (1500);
1298 eval_queue_opts(struct pf_altq *pa, struct node_queue_opt *opts,
1299 u_int32_t ref_bw)
1301 int errors = 0;
1303 switch (pa->scheduler) {
1304 case ALTQT_CBQ:
1305 pa->pq_u.cbq_opts = opts->data.cbq_opts;
1306 break;
1307 case ALTQT_PRIQ:
1308 pa->pq_u.priq_opts = opts->data.priq_opts;
1309 break;
1310 case ALTQT_HFSC:
1311 pa->pq_u.hfsc_opts.flags = opts->data.hfsc_opts.flags;
1312 if (opts->data.hfsc_opts.linkshare.used) {
1313 pa->pq_u.hfsc_opts.lssc_m1 =
1314 eval_bwspec(&opts->data.hfsc_opts.linkshare.m1,
1315 ref_bw);
1316 pa->pq_u.hfsc_opts.lssc_m2 =
1317 eval_bwspec(&opts->data.hfsc_opts.linkshare.m2,
1318 ref_bw);
1319 pa->pq_u.hfsc_opts.lssc_d =
1320 opts->data.hfsc_opts.linkshare.d;
1322 if (opts->data.hfsc_opts.realtime.used) {
1323 pa->pq_u.hfsc_opts.rtsc_m1 =
1324 eval_bwspec(&opts->data.hfsc_opts.realtime.m1,
1325 ref_bw);
1326 pa->pq_u.hfsc_opts.rtsc_m2 =
1327 eval_bwspec(&opts->data.hfsc_opts.realtime.m2,
1328 ref_bw);
1329 pa->pq_u.hfsc_opts.rtsc_d =
1330 opts->data.hfsc_opts.realtime.d;
1332 if (opts->data.hfsc_opts.upperlimit.used) {
1333 pa->pq_u.hfsc_opts.ulsc_m1 =
1334 eval_bwspec(&opts->data.hfsc_opts.upperlimit.m1,
1335 ref_bw);
1336 pa->pq_u.hfsc_opts.ulsc_m2 =
1337 eval_bwspec(&opts->data.hfsc_opts.upperlimit.m2,
1338 ref_bw);
1339 pa->pq_u.hfsc_opts.ulsc_d =
1340 opts->data.hfsc_opts.upperlimit.d;
1342 break;
1343 case ALTQT_FAIRQ:
1344 pa->pq_u.fairq_opts.flags = opts->data.fairq_opts.flags;
1345 pa->pq_u.fairq_opts.nbuckets = opts->data.fairq_opts.nbuckets;
1346 pa->pq_u.fairq_opts.hogs_m1 =
1347 eval_bwspec(&opts->data.fairq_opts.hogs_bw, ref_bw);
1349 if (opts->data.fairq_opts.linkshare.used) {
1350 pa->pq_u.fairq_opts.lssc_m1 =
1351 eval_bwspec(&opts->data.fairq_opts.linkshare.m1,
1352 ref_bw);
1353 pa->pq_u.fairq_opts.lssc_m2 =
1354 eval_bwspec(&opts->data.fairq_opts.linkshare.m2,
1355 ref_bw);
1356 pa->pq_u.fairq_opts.lssc_d =
1357 opts->data.fairq_opts.linkshare.d;
1359 break;
1360 default:
1361 warnx("eval_queue_opts: unknown scheduler type %u",
1362 opts->qtype);
1363 errors++;
1364 break;
1367 return (errors);
1370 u_int32_t
1371 eval_bwspec(struct node_queue_bw *bw, u_int32_t ref_bw)
1373 if (bw->bw_absolute > 0)
1374 return (bw->bw_absolute);
1376 if (bw->bw_percent > 0)
1377 return (ref_bw / 100 * bw->bw_percent);
1379 return (0);
1382 void
1383 print_hfsc_sc(const char *scname, u_int m1, u_int d, u_int m2,
1384 const struct node_hfsc_sc *sc)
1386 printf(" %s", scname);
1388 if (d != 0) {
1389 printf("(");
1390 if (sc != NULL && sc->m1.bw_percent > 0)
1391 printf("%u%%", sc->m1.bw_percent);
1392 else
1393 printf("%s", rate2str((double)m1));
1394 printf(" %u", d);
1397 if (sc != NULL && sc->m2.bw_percent > 0)
1398 printf(" %u%%", sc->m2.bw_percent);
1399 else
1400 printf(" %s", rate2str((double)m2));
1402 if (d != 0)
1403 printf(")");
1406 void
1407 print_fairq_sc(const char *scname, u_int m1, u_int d, u_int m2,
1408 const struct node_fairq_sc *sc)
1410 printf(" %s", scname);
1412 if (d != 0) {
1413 printf("(");
1414 if (sc != NULL && sc->m1.bw_percent > 0)
1415 printf("%u%%", sc->m1.bw_percent);
1416 else
1417 printf("%s", rate2str((double)m1));
1418 printf(" %u", d);
1421 if (sc != NULL && sc->m2.bw_percent > 0)
1422 printf(" %u%%", sc->m2.bw_percent);
1423 else
1424 printf(" %s", rate2str((double)m2));
1426 if (d != 0)
1427 printf(")");