boot - Switch boot2 loader path around
[dragonfly.git] / usr.sbin / pfctl / pfctl_altq.c
blobc7efc6368ec9d51506bf196d1cf7fe6ab9abdbf1
1 /* $OpenBSD: pfctl_altq.c,v 1.83 2004/03/14 21:51:44 dhartmei Exp $ */
2 /* $DragonFly: src/usr.sbin/pfctl/pfctl_altq.c,v 1.4 2008/11/03 00:25:45 pavalos Exp $ */
4 /*
5 * Copyright (c) 2002
6 * Sony Computer Science Laboratories Inc.
7 * Copyright (c) 2002, 2003 Henning Brauer <henning@openbsd.org>
9 * Permission to use, copy, modify, and distribute this software for any
10 * purpose with or without fee is hereby granted, provided that the above
11 * copyright notice and this permission notice appear in all copies.
13 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
14 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
15 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
16 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
17 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
18 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
19 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
22 #include <sys/param.h>
23 #include <sys/ioctl.h>
24 #include <sys/socket.h>
25 #include <sys/sysctl.h>
27 #include <net/if.h>
28 #include <net/if_mib.h>
29 #include <netinet/in.h>
30 #include <net/pf/pfvar.h>
32 #include <err.h>
33 #include <errno.h>
34 #include <limits.h>
35 #include <math.h>
36 #include <stdio.h>
37 #include <stdlib.h>
38 #include <string.h>
39 #include <unistd.h>
41 #include <net/altq/altq.h>
42 #include <net/altq/altq_cbq.h>
43 #include <net/altq/altq_priq.h>
44 #include <net/altq/altq_hfsc.h>
45 #include <net/altq/altq_fairq.h>
47 #include "pfctl_parser.h"
48 #include "pfctl.h"
50 #define is_sc_null(sc) (((sc) == NULL) || ((sc)->m1 == 0 && (sc)->m2 == 0))
52 TAILQ_HEAD(altqs, pf_altq) altqs = TAILQ_HEAD_INITIALIZER(altqs);
53 LIST_HEAD(gen_sc, segment) rtsc, lssc;
55 struct pf_altq *qname_to_pfaltq(const char *, const char *);
56 u_int32_t qname_to_qid(const char *);
58 static int eval_pfqueue_cbq(struct pfctl *, struct pf_altq *);
59 static int cbq_compute_idletime(struct pfctl *, struct pf_altq *);
60 static int check_commit_cbq(int, int, struct pf_altq *);
61 static int print_cbq_opts(const struct pf_altq *);
63 static int eval_pfqueue_priq(struct pfctl *, struct pf_altq *);
64 static int check_commit_priq(int, int, struct pf_altq *);
65 static int print_priq_opts(const struct pf_altq *);
67 static int eval_pfqueue_hfsc(struct pfctl *, struct pf_altq *);
68 static int check_commit_hfsc(int, int, struct pf_altq *);
69 static int print_hfsc_opts(const struct pf_altq *,
70 const struct node_queue_opt *);
72 static int eval_pfqueue_fairq(struct pfctl *, struct pf_altq *);
73 static int print_fairq_opts(const struct pf_altq *,
74 const struct node_queue_opt *);
75 static int check_commit_fairq(int, int, struct pf_altq *);
77 static void gsc_add_sc(struct gen_sc *, struct service_curve *);
78 static int is_gsc_under_sc(struct gen_sc *,
79 struct service_curve *);
80 static void gsc_destroy(struct gen_sc *);
81 static struct segment *gsc_getentry(struct gen_sc *, double);
82 static int gsc_add_seg(struct gen_sc *, double, double, double,
83 double);
84 static double sc_x2y(struct service_curve *, double);
86 u_int32_t getifspeed(const char *);
87 u_long getifmtu(char *);
88 int eval_queue_opts(struct pf_altq *, struct node_queue_opt *,
89 u_int32_t);
90 u_int32_t eval_bwspec(struct node_queue_bw *, u_int32_t);
91 void print_hfsc_sc(const char *, u_int, u_int, u_int,
92 const struct node_hfsc_sc *);
93 void print_fairq_sc(const char *, u_int, u_int, u_int,
94 const struct node_fairq_sc *);
96 void
97 pfaltq_store(struct pf_altq *a)
99 struct pf_altq *altq;
101 if ((altq = malloc(sizeof(*altq))) == NULL)
102 err(1, "malloc");
103 memcpy(altq, a, sizeof(struct pf_altq));
104 TAILQ_INSERT_TAIL(&altqs, altq, entries);
107 void
108 pfaltq_free(struct pf_altq *a)
110 struct pf_altq *altq;
112 TAILQ_FOREACH(altq, &altqs, entries) {
113 if (strncmp(a->ifname, altq->ifname, IFNAMSIZ) == 0 &&
114 strncmp(a->qname, altq->qname, PF_QNAME_SIZE) == 0) {
115 TAILQ_REMOVE(&altqs, altq, entries);
116 free(altq);
117 return;
122 struct pf_altq *
123 pfaltq_lookup(const char *ifname)
125 struct pf_altq *altq;
127 TAILQ_FOREACH(altq, &altqs, entries) {
128 if (strncmp(ifname, altq->ifname, IFNAMSIZ) == 0 &&
129 altq->qname[0] == 0)
130 return (altq);
132 return (NULL);
135 struct pf_altq *
136 qname_to_pfaltq(const char *qname, const char *ifname)
138 struct pf_altq *altq;
140 TAILQ_FOREACH(altq, &altqs, entries) {
141 if (strncmp(ifname, altq->ifname, IFNAMSIZ) == 0 &&
142 strncmp(qname, altq->qname, PF_QNAME_SIZE) == 0)
143 return (altq);
145 return (NULL);
148 u_int32_t
149 qname_to_qid(const char *qname)
151 struct pf_altq *altq;
154 * We guarantee that same named queues on different interfaces
155 * have the same qid, so we do NOT need to limit matching on
156 * one interface!
159 TAILQ_FOREACH(altq, &altqs, entries) {
160 if (strncmp(qname, altq->qname, PF_QNAME_SIZE) == 0)
161 return (altq->qid);
163 return (0);
166 void
167 print_altq(const struct pf_altq *a, unsigned level, struct node_queue_bw *bw,
168 struct node_queue_opt *qopts)
170 if (a->qname[0] != 0) {
171 print_queue(a, level, bw, 0, qopts);
172 return;
175 printf("altq on %s ", a->ifname);
177 switch (a->scheduler) {
178 case ALTQT_CBQ:
179 if (!print_cbq_opts(a))
180 printf("cbq ");
181 break;
182 case ALTQT_PRIQ:
183 if (!print_priq_opts(a))
184 printf("priq ");
185 break;
186 case ALTQT_HFSC:
187 if (!print_hfsc_opts(a, qopts))
188 printf("hfsc ");
189 break;
190 case ALTQT_FAIRQ:
191 if (!print_fairq_opts(a, qopts))
192 printf("hfsc ");
193 break;
196 if (bw != NULL && bw->bw_percent > 0) {
197 if (bw->bw_percent < 100)
198 printf("bandwidth %u%% ", bw->bw_percent);
199 } else
200 printf("bandwidth %s ", rate2str((double)a->ifbandwidth));
202 if (a->qlimit != DEFAULT_QLIMIT)
203 printf("qlimit %u ", a->qlimit);
204 printf("tbrsize %u ", a->tbrsize);
207 void
208 print_queue(const struct pf_altq *a, unsigned level, struct node_queue_bw *bw,
209 int print_interface, struct node_queue_opt *qopts)
211 unsigned i;
213 printf("queue ");
214 for (i = 0; i < level; ++i)
215 printf(" ");
216 printf("%s ", a->qname);
217 if (print_interface)
218 printf("on %s ", a->ifname);
219 if (a->scheduler == ALTQT_CBQ || a->scheduler == ALTQT_HFSC ||
220 a->scheduler == ALTQT_FAIRQ) {
221 if (bw != NULL && bw->bw_percent > 0) {
222 if (bw->bw_percent < 100)
223 printf("bandwidth %u%% ", bw->bw_percent);
224 } else
225 printf("bandwidth %s ", rate2str((double)a->bandwidth));
227 if (a->priority != DEFAULT_PRIORITY)
228 printf("priority %u ", a->priority);
229 if (a->qlimit != DEFAULT_QLIMIT)
230 printf("qlimit %u ", a->qlimit);
231 switch (a->scheduler) {
232 case ALTQT_CBQ:
233 print_cbq_opts(a);
234 break;
235 case ALTQT_PRIQ:
236 print_priq_opts(a);
237 break;
238 case ALTQT_HFSC:
239 print_hfsc_opts(a, qopts);
240 break;
241 case ALTQT_FAIRQ:
242 print_fairq_opts(a, qopts);
243 break;
248 * eval_pfaltq computes the discipline parameters.
251 eval_pfaltq(struct pfctl *pf __unused, struct pf_altq *pa,
252 struct node_queue_bw *bw, struct node_queue_opt *opts)
254 u_int rate, size, errors = 0;
256 if (bw->bw_absolute > 0)
257 pa->ifbandwidth = bw->bw_absolute;
258 else
259 if ((rate = getifspeed(pa->ifname)) == 0) {
260 fprintf(stderr, "cannot determine interface bandwidth "
261 "for %s, specify an absolute bandwidth\n",
262 pa->ifname);
263 errors++;
264 } else if ((pa->ifbandwidth = eval_bwspec(bw, rate)) == 0)
265 pa->ifbandwidth = rate;
267 errors += eval_queue_opts(pa, opts, pa->ifbandwidth);
269 /* if tbrsize is not specified, use heuristics */
270 if (pa->tbrsize == 0) {
271 rate = pa->ifbandwidth;
272 if (rate <= 1 * 1000 * 1000)
273 size = 1;
274 else if (rate <= 10 * 1000 * 1000)
275 size = 4;
276 else if (rate <= 200 * 1000 * 1000)
277 size = 8;
278 else
279 size = 24;
280 size = size * getifmtu(pa->ifname);
281 if (size > 0xffff)
282 size = 0xffff;
283 pa->tbrsize = size;
285 return (errors);
289 * check_commit_altq does consistency check for each interface
292 check_commit_altq(int dev, int opts)
294 struct pf_altq *altq;
295 int error = 0;
297 /* call the discipline check for each interface. */
298 TAILQ_FOREACH(altq, &altqs, entries) {
299 if (altq->qname[0] == 0) {
300 switch (altq->scheduler) {
301 case ALTQT_CBQ:
302 error = check_commit_cbq(dev, opts, altq);
303 break;
304 case ALTQT_PRIQ:
305 error = check_commit_priq(dev, opts, altq);
306 break;
307 case ALTQT_HFSC:
308 error = check_commit_hfsc(dev, opts, altq);
309 break;
310 case ALTQT_FAIRQ:
311 error = check_commit_fairq(dev, opts, altq);
312 break;
313 default:
314 break;
318 return (error);
322 * eval_pfqueue computes the queue parameters.
325 eval_pfqueue(struct pfctl *pf, struct pf_altq *pa, struct node_queue_bw *bw,
326 struct node_queue_opt *opts)
328 /* should be merged with expand_queue */
329 struct pf_altq *if_pa, *parent;
330 int error = 0;
332 /* find the corresponding interface and copy fields used by queues */
333 if ((if_pa = pfaltq_lookup(pa->ifname)) == NULL) {
334 fprintf(stderr, "altq not defined on %s\n", pa->ifname);
335 return (1);
337 pa->scheduler = if_pa->scheduler;
338 pa->ifbandwidth = if_pa->ifbandwidth;
340 if (qname_to_pfaltq(pa->qname, pa->ifname) != NULL) {
341 fprintf(stderr, "queue %s already exists on interface %s\n",
342 pa->qname, pa->ifname);
343 return (1);
345 pa->qid = qname_to_qid(pa->qname);
347 parent = NULL;
348 if (pa->parent[0] != 0) {
349 parent = qname_to_pfaltq(pa->parent, pa->ifname);
350 if (parent == NULL) {
351 fprintf(stderr, "parent %s not found for %s\n",
352 pa->parent, pa->qname);
353 return (1);
355 pa->parent_qid = parent->qid;
357 if (pa->qlimit == 0)
358 pa->qlimit = DEFAULT_QLIMIT;
360 if (pa->scheduler == ALTQT_CBQ || pa->scheduler == ALTQT_HFSC ||
361 pa->scheduler == ALTQT_FAIRQ) {
362 if ((pa->bandwidth = eval_bwspec(bw,
363 parent == NULL ? 0 : parent->bandwidth)) == 0) {
364 fprintf(stderr, "bandwidth for %s invalid (%d / %d)\n",
365 pa->qname, bw->bw_absolute, bw->bw_percent);
366 return (1);
369 if (pa->bandwidth > pa->ifbandwidth) {
370 fprintf(stderr, "bandwidth for %s higher than "
371 "interface\n", pa->qname);
372 return (1);
374 if (parent != NULL && pa->bandwidth > parent->bandwidth) {
375 fprintf(stderr, "bandwidth for %s higher than parent\n",
376 pa->qname);
377 return (1);
381 if (eval_queue_opts(pa, opts, parent == NULL? 0 : parent->bandwidth))
382 return (1);
384 switch (pa->scheduler) {
385 case ALTQT_CBQ:
386 error = eval_pfqueue_cbq(pf, pa);
387 break;
388 case ALTQT_PRIQ:
389 error = eval_pfqueue_priq(pf, pa);
390 break;
391 case ALTQT_HFSC:
392 error = eval_pfqueue_hfsc(pf, pa);
393 break;
394 case ALTQT_FAIRQ:
395 error = eval_pfqueue_fairq(pf, pa);
396 break;
397 default:
398 break;
400 return (error);
404 * CBQ support functions
406 #define RM_FILTER_GAIN 5 /* log2 of gain, e.g., 5 => 31/32 */
407 #define RM_NS_PER_SEC (1000000000)
409 static int
410 eval_pfqueue_cbq(struct pfctl *pf, struct pf_altq *pa)
412 struct cbq_opts *opts;
413 u_int ifmtu;
415 if (pa->priority >= CBQ_MAXPRI) {
416 warnx("priority out of range: max %d", CBQ_MAXPRI - 1);
417 return (-1);
420 ifmtu = getifmtu(pa->ifname);
421 opts = &pa->pq_u.cbq_opts;
423 if (opts->pktsize == 0) { /* use default */
424 opts->pktsize = ifmtu;
425 if (opts->pktsize > MCLBYTES) /* do what TCP does */
426 opts->pktsize &= ~MCLBYTES;
427 } else if (opts->pktsize > ifmtu)
428 opts->pktsize = ifmtu;
429 if (opts->maxpktsize == 0) /* use default */
430 opts->maxpktsize = ifmtu;
431 else if (opts->maxpktsize > ifmtu)
432 opts->pktsize = ifmtu;
434 if (opts->pktsize > opts->maxpktsize)
435 opts->pktsize = opts->maxpktsize;
437 if (pa->parent[0] == 0)
438 opts->flags |= (CBQCLF_ROOTCLASS | CBQCLF_WRR);
440 cbq_compute_idletime(pf, pa);
441 return (0);
445 * compute ns_per_byte, maxidle, minidle, and offtime
447 static int
448 cbq_compute_idletime(struct pfctl *pf, struct pf_altq *pa)
450 struct cbq_opts *opts;
451 double maxidle_s, maxidle, minidle;
452 double offtime, nsPerByte, ifnsPerByte, ptime, cptime;
453 double z, g, f, gton, gtom;
454 u_int minburst, maxburst;
456 opts = &pa->pq_u.cbq_opts;
457 ifnsPerByte = (1.0 / (double)pa->ifbandwidth) * RM_NS_PER_SEC * 8;
458 minburst = opts->minburst;
459 maxburst = opts->maxburst;
461 if (pa->bandwidth == 0)
462 f = 0.0001; /* small enough? */
463 else
464 f = ((double) pa->bandwidth / (double) pa->ifbandwidth);
466 nsPerByte = ifnsPerByte / f;
467 ptime = (double)opts->pktsize * ifnsPerByte;
468 cptime = ptime * (1.0 - f) / f;
470 if (nsPerByte * (double)opts->maxpktsize > (double)INT_MAX) {
472 * this causes integer overflow in kernel!
473 * (bandwidth < 6Kbps when max_pkt_size=1500)
475 if (pa->bandwidth != 0 && (pf->opts & PF_OPT_QUIET) == 0)
476 warnx("queue bandwidth must be larger than %s",
477 rate2str(ifnsPerByte * (double)opts->maxpktsize /
478 (double)INT_MAX * (double)pa->ifbandwidth));
479 fprintf(stderr, "cbq: queue %s is too slow!\n",
480 pa->qname);
481 nsPerByte = (double)(INT_MAX / opts->maxpktsize);
484 if (maxburst == 0) { /* use default */
485 if (cptime > 10.0 * 1000000)
486 maxburst = 4;
487 else
488 maxburst = 16;
490 if (minburst == 0) /* use default */
491 minburst = 2;
492 if (minburst > maxburst)
493 minburst = maxburst;
495 z = (double)(1 << RM_FILTER_GAIN);
496 g = (1.0 - 1.0 / z);
497 gton = pow(g, (double)maxburst);
498 gtom = pow(g, (double)(minburst-1));
499 maxidle = ((1.0 / f - 1.0) * ((1.0 - gton) / gton));
500 maxidle_s = (1.0 - g);
501 if (maxidle > maxidle_s)
502 maxidle = ptime * maxidle;
503 else
504 maxidle = ptime * maxidle_s;
505 if (minburst)
506 offtime = cptime * (1.0 + 1.0/(1.0 - g) * (1.0 - gtom) / gtom);
507 else
508 offtime = cptime;
509 minidle = -((double)opts->maxpktsize * (double)nsPerByte);
511 /* scale parameters */
512 maxidle = ((maxidle * 8.0) / nsPerByte) *
513 pow(2.0, (double)RM_FILTER_GAIN);
514 offtime = (offtime * 8.0) / nsPerByte *
515 pow(2.0, (double)RM_FILTER_GAIN);
516 minidle = ((minidle * 8.0) / nsPerByte) *
517 pow(2.0, (double)RM_FILTER_GAIN);
519 maxidle = maxidle / 1000.0;
520 offtime = offtime / 1000.0;
521 minidle = minidle / 1000.0;
523 opts->minburst = minburst;
524 opts->maxburst = maxburst;
525 opts->ns_per_byte = (u_int)nsPerByte;
526 opts->maxidle = (u_int)fabs(maxidle);
527 opts->minidle = (int)minidle;
528 opts->offtime = (u_int)fabs(offtime);
530 return (0);
533 static int
534 check_commit_cbq(int dev __unused, int opts __unused, struct pf_altq *pa)
536 struct pf_altq *altq;
537 int root_class, default_class;
538 int error = 0;
541 * check if cbq has one root queue and one default queue
542 * for this interface
544 root_class = default_class = 0;
545 TAILQ_FOREACH(altq, &altqs, entries) {
546 if (strncmp(altq->ifname, pa->ifname, IFNAMSIZ) != 0)
547 continue;
548 if (altq->qname[0] == 0) /* this is for interface */
549 continue;
550 if (altq->pq_u.cbq_opts.flags & CBQCLF_ROOTCLASS)
551 root_class++;
552 if (altq->pq_u.cbq_opts.flags & CBQCLF_DEFCLASS)
553 default_class++;
555 if (root_class != 1) {
556 warnx("should have one root queue on %s", pa->ifname);
557 error++;
559 if (default_class != 1) {
560 warnx("should have one default queue on %s", pa->ifname);
561 error++;
563 return (error);
566 static int
567 print_cbq_opts(const struct pf_altq *a)
569 const struct cbq_opts *opts;
571 opts = &a->pq_u.cbq_opts;
572 if (opts->flags) {
573 printf("cbq(");
574 if (opts->flags & CBQCLF_RED)
575 printf(" red");
576 if (opts->flags & CBQCLF_ECN)
577 printf(" ecn");
578 if (opts->flags & CBQCLF_RIO)
579 printf(" rio");
580 if (opts->flags & CBQCLF_CLEARDSCP)
581 printf(" cleardscp");
582 if (opts->flags & CBQCLF_BORROW)
583 printf(" borrow");
584 if (opts->flags & CBQCLF_WRR)
585 printf(" wrr");
586 if (opts->flags & CBQCLF_EFFICIENT)
587 printf(" efficient");
588 if (opts->flags & CBQCLF_ROOTCLASS)
589 printf(" root");
590 if (opts->flags & CBQCLF_DEFCLASS)
591 printf(" default");
592 printf(" ) ");
594 return (1);
595 } else
596 return (0);
600 * PRIQ support functions
602 static int
603 eval_pfqueue_priq(struct pfctl *pf __unused, struct pf_altq *pa)
605 struct pf_altq *altq;
607 if (pa->priority >= PRIQ_MAXPRI) {
608 warnx("priority out of range: max %d", PRIQ_MAXPRI - 1);
609 return (-1);
611 /* the priority should be unique for the interface */
612 TAILQ_FOREACH(altq, &altqs, entries) {
613 if (strncmp(altq->ifname, pa->ifname, IFNAMSIZ) == 0 &&
614 altq->qname[0] != 0 && altq->priority == pa->priority) {
615 warnx("%s and %s have the same priority",
616 altq->qname, pa->qname);
617 return (-1);
621 return (0);
624 static int
625 check_commit_priq(int dev __unused, int opts __unused, struct pf_altq *pa)
627 struct pf_altq *altq;
628 int default_class;
629 int error = 0;
632 * check if priq has one default class for this interface
634 default_class = 0;
635 TAILQ_FOREACH(altq, &altqs, entries) {
636 if (strncmp(altq->ifname, pa->ifname, IFNAMSIZ) != 0)
637 continue;
638 if (altq->qname[0] == 0) /* this is for interface */
639 continue;
640 if (altq->pq_u.priq_opts.flags & PRCF_DEFAULTCLASS)
641 default_class++;
643 if (default_class != 1) {
644 warnx("should have one default queue on %s", pa->ifname);
645 error++;
647 return (error);
650 static int
651 print_priq_opts(const struct pf_altq *a)
653 const struct priq_opts *opts;
655 opts = &a->pq_u.priq_opts;
657 if (opts->flags) {
658 printf("priq(");
659 if (opts->flags & PRCF_RED)
660 printf(" red");
661 if (opts->flags & PRCF_ECN)
662 printf(" ecn");
663 if (opts->flags & PRCF_RIO)
664 printf(" rio");
665 if (opts->flags & PRCF_CLEARDSCP)
666 printf(" cleardscp");
667 if (opts->flags & PRCF_DEFAULTCLASS)
668 printf(" default");
669 printf(" ) ");
671 return (1);
672 } else
673 return (0);
677 * HFSC support functions
679 static int
680 eval_pfqueue_hfsc(struct pfctl *pf __unused, struct pf_altq *pa)
682 struct pf_altq *altq, *parent;
683 struct hfsc_opts *opts;
684 struct service_curve sc;
686 opts = &pa->pq_u.hfsc_opts;
688 if (pa->parent[0] == 0) {
689 /* root queue */
690 opts->lssc_m1 = pa->ifbandwidth;
691 opts->lssc_m2 = pa->ifbandwidth;
692 opts->lssc_d = 0;
693 return (0);
696 LIST_INIT(&rtsc);
697 LIST_INIT(&lssc);
699 /* if link_share is not specified, use bandwidth */
700 if (opts->lssc_m2 == 0)
701 opts->lssc_m2 = pa->bandwidth;
703 if ((opts->rtsc_m1 > 0 && opts->rtsc_m2 == 0) ||
704 (opts->lssc_m1 > 0 && opts->lssc_m2 == 0) ||
705 (opts->ulsc_m1 > 0 && opts->ulsc_m2 == 0)) {
706 warnx("m2 is zero for %s", pa->qname);
707 return (-1);
710 if ((opts->rtsc_m1 < opts->rtsc_m2 && opts->rtsc_m1 != 0) ||
711 (opts->rtsc_m1 < opts->rtsc_m2 && opts->rtsc_m1 != 0) ||
712 (opts->rtsc_m1 < opts->rtsc_m2 && opts->rtsc_m1 != 0)) {
713 warnx("m1 must be zero for convex curve: %s", pa->qname);
714 return (-1);
718 * admission control:
719 * for the real-time service curve, the sum of the service curves
720 * should not exceed 80% of the interface bandwidth. 20% is reserved
721 * not to over-commit the actual interface bandwidth.
722 * for the link-sharing service curve, the sum of the child service
723 * curve should not exceed the parent service curve.
724 * for the upper-limit service curve, the assigned bandwidth should
725 * be smaller than the interface bandwidth, and the upper-limit should
726 * be larger than the real-time service curve when both are defined.
728 parent = qname_to_pfaltq(pa->parent, pa->ifname);
729 if (parent == NULL)
730 errx(1, "parent %s not found for %s", pa->parent, pa->qname);
732 TAILQ_FOREACH(altq, &altqs, entries) {
733 if (strncmp(altq->ifname, pa->ifname, IFNAMSIZ) != 0)
734 continue;
735 if (altq->qname[0] == 0) /* this is for interface */
736 continue;
738 /* if the class has a real-time service curve, add it. */
739 if (opts->rtsc_m2 != 0 && altq->pq_u.hfsc_opts.rtsc_m2 != 0) {
740 sc.m1 = altq->pq_u.hfsc_opts.rtsc_m1;
741 sc.d = altq->pq_u.hfsc_opts.rtsc_d;
742 sc.m2 = altq->pq_u.hfsc_opts.rtsc_m2;
743 gsc_add_sc(&rtsc, &sc);
746 if (strncmp(altq->parent, pa->parent, PF_QNAME_SIZE) != 0)
747 continue;
749 /* if the class has a link-sharing service curve, add it. */
750 if (opts->lssc_m2 != 0 && altq->pq_u.hfsc_opts.lssc_m2 != 0) {
751 sc.m1 = altq->pq_u.hfsc_opts.lssc_m1;
752 sc.d = altq->pq_u.hfsc_opts.lssc_d;
753 sc.m2 = altq->pq_u.hfsc_opts.lssc_m2;
754 gsc_add_sc(&lssc, &sc);
758 /* check the real-time service curve. reserve 20% of interface bw */
759 if (opts->rtsc_m2 != 0) {
760 sc.m1 = 0;
761 sc.d = 0;
762 sc.m2 = pa->ifbandwidth / 100 * 80;
763 if (!is_gsc_under_sc(&rtsc, &sc)) {
764 warnx("real-time sc exceeds the interface bandwidth");
765 goto err_ret;
769 /* check the link-sharing service curve. */
770 if (opts->lssc_m2 != 0) {
771 sc.m1 = parent->pq_u.hfsc_opts.lssc_m1;
772 sc.d = parent->pq_u.hfsc_opts.lssc_d;
773 sc.m2 = parent->pq_u.hfsc_opts.lssc_m2;
774 if (!is_gsc_under_sc(&lssc, &sc)) {
775 warnx("link-sharing sc exceeds parent's sc");
776 goto err_ret;
780 /* check the upper-limit service curve. */
781 if (opts->ulsc_m2 != 0) {
782 if (opts->ulsc_m1 > pa->ifbandwidth ||
783 opts->ulsc_m2 > pa->ifbandwidth) {
784 warnx("upper-limit larger than interface bandwidth");
785 goto err_ret;
787 if (opts->rtsc_m2 != 0 && opts->rtsc_m2 > opts->ulsc_m2) {
788 warnx("upper-limit sc smaller than real-time sc");
789 goto err_ret;
793 gsc_destroy(&rtsc);
794 gsc_destroy(&lssc);
796 return (0);
798 err_ret:
799 gsc_destroy(&rtsc);
800 gsc_destroy(&lssc);
801 return (-1);
805 * FAIRQ support functions
807 static int
808 eval_pfqueue_fairq(struct pfctl *pf __unused, struct pf_altq *pa)
810 struct pf_altq *altq, *parent;
811 struct fairq_opts *opts;
812 struct service_curve sc;
814 opts = &pa->pq_u.fairq_opts;
816 if (pa->parent[0] == 0) {
817 /* root queue */
818 opts->lssc_m1 = pa->ifbandwidth;
819 opts->lssc_m2 = pa->ifbandwidth;
820 opts->lssc_d = 0;
821 return (0);
824 LIST_INIT(&lssc);
826 /* if link_share is not specified, use bandwidth */
827 if (opts->lssc_m2 == 0)
828 opts->lssc_m2 = pa->bandwidth;
831 * admission control:
832 * for the real-time service curve, the sum of the service curves
833 * should not exceed 80% of the interface bandwidth. 20% is reserved
834 * not to over-commit the actual interface bandwidth.
835 * for the link-sharing service curve, the sum of the child service
836 * curve should not exceed the parent service curve.
837 * for the upper-limit service curve, the assigned bandwidth should
838 * be smaller than the interface bandwidth, and the upper-limit should
839 * be larger than the real-time service curve when both are defined.
841 parent = qname_to_pfaltq(pa->parent, pa->ifname);
842 if (parent == NULL)
843 errx(1, "parent %s not found for %s", pa->parent, pa->qname);
845 TAILQ_FOREACH(altq, &altqs, entries) {
846 if (strncmp(altq->ifname, pa->ifname, IFNAMSIZ) != 0)
847 continue;
848 if (altq->qname[0] == 0) /* this is for interface */
849 continue;
851 if (strncmp(altq->parent, pa->parent, PF_QNAME_SIZE) != 0)
852 continue;
854 /* if the class has a link-sharing service curve, add it. */
855 if (opts->lssc_m2 != 0 && altq->pq_u.fairq_opts.lssc_m2 != 0) {
856 sc.m1 = altq->pq_u.fairq_opts.lssc_m1;
857 sc.d = altq->pq_u.fairq_opts.lssc_d;
858 sc.m2 = altq->pq_u.fairq_opts.lssc_m2;
859 gsc_add_sc(&lssc, &sc);
863 /* check the link-sharing service curve. */
864 if (opts->lssc_m2 != 0) {
865 sc.m1 = parent->pq_u.fairq_opts.lssc_m1;
866 sc.d = parent->pq_u.fairq_opts.lssc_d;
867 sc.m2 = parent->pq_u.fairq_opts.lssc_m2;
868 if (!is_gsc_under_sc(&lssc, &sc)) {
869 warnx("link-sharing sc exceeds parent's sc");
870 goto err_ret;
874 gsc_destroy(&lssc);
876 return (0);
878 err_ret:
879 gsc_destroy(&lssc);
880 return (-1);
883 static int
884 check_commit_hfsc(int dev __unused, int opts __unused, struct pf_altq *pa)
886 struct pf_altq *altq, *def = NULL;
887 int default_class;
888 int error = 0;
890 /* check if hfsc has one default queue for this interface */
891 default_class = 0;
892 TAILQ_FOREACH(altq, &altqs, entries) {
893 if (strncmp(altq->ifname, pa->ifname, IFNAMSIZ) != 0)
894 continue;
895 if (altq->qname[0] == 0) /* this is for interface */
896 continue;
897 if (altq->parent[0] == 0) /* dummy root */
898 continue;
899 if (altq->pq_u.hfsc_opts.flags & HFCF_DEFAULTCLASS) {
900 default_class++;
901 def = altq;
904 if (default_class != 1) {
905 warnx("should have one default queue on %s", pa->ifname);
906 return (1);
908 /* make sure the default queue is a leaf */
909 TAILQ_FOREACH(altq, &altqs, entries) {
910 if (strncmp(altq->ifname, pa->ifname, IFNAMSIZ) != 0)
911 continue;
912 if (altq->qname[0] == 0) /* this is for interface */
913 continue;
914 if (strncmp(altq->parent, def->qname, PF_QNAME_SIZE) == 0) {
915 warnx("default queue is not a leaf");
916 error++;
919 return (error);
922 static int
923 check_commit_fairq(int dev __unused, int opts __unused, struct pf_altq *pa)
925 struct pf_altq *altq, *def = NULL;
926 int default_class;
927 int error = 0;
929 /* check if fairq has one default queue for this interface */
930 default_class = 0;
931 TAILQ_FOREACH(altq, &altqs, entries) {
932 if (strncmp(altq->ifname, pa->ifname, IFNAMSIZ) != 0)
933 continue;
934 if (altq->qname[0] == 0) /* this is for interface */
935 continue;
936 if (altq->pq_u.fairq_opts.flags & FARF_DEFAULTCLASS) {
937 default_class++;
938 def = altq;
941 if (default_class != 1) {
942 warnx("should have one default queue on %s", pa->ifname);
943 return (1);
945 /* make sure the default queue is a leaf */
946 TAILQ_FOREACH(altq, &altqs, entries) {
947 if (strncmp(altq->ifname, pa->ifname, IFNAMSIZ) != 0)
948 continue;
949 if (altq->qname[0] == 0) /* this is for interface */
950 continue;
951 if (strncmp(altq->parent, def->qname, PF_QNAME_SIZE) == 0) {
952 warnx("default queue is not a leaf");
953 error++;
956 return (error);
959 static int
960 print_hfsc_opts(const struct pf_altq *a, const struct node_queue_opt *qopts)
962 const struct hfsc_opts *opts;
963 const struct node_hfsc_sc *loc_rtsc, *loc_lssc, *ulsc;
965 opts = &a->pq_u.hfsc_opts;
966 if (qopts == NULL)
967 loc_rtsc = loc_lssc = ulsc = NULL;
968 else {
969 loc_rtsc = &qopts->data.hfsc_opts.realtime;
970 loc_lssc = &qopts->data.hfsc_opts.linkshare;
971 ulsc = &qopts->data.hfsc_opts.upperlimit;
974 if (opts->flags || opts->rtsc_m2 != 0 || opts->ulsc_m2 != 0 ||
975 (opts->lssc_m2 != 0 && (opts->lssc_m2 != a->bandwidth ||
976 opts->lssc_d != 0))) {
977 printf("hfsc(");
978 if (opts->flags & HFCF_RED)
979 printf(" red");
980 if (opts->flags & HFCF_ECN)
981 printf(" ecn");
982 if (opts->flags & HFCF_RIO)
983 printf(" rio");
984 if (opts->flags & HFCF_CLEARDSCP)
985 printf(" cleardscp");
986 if (opts->flags & HFCF_DEFAULTCLASS)
987 printf(" default");
988 if (opts->rtsc_m2 != 0)
989 print_hfsc_sc("realtime", opts->rtsc_m1, opts->rtsc_d,
990 opts->rtsc_m2, loc_rtsc);
991 if (opts->lssc_m2 != 0 && (opts->lssc_m2 != a->bandwidth ||
992 opts->lssc_d != 0))
993 print_hfsc_sc("linkshare", opts->lssc_m1, opts->lssc_d,
994 opts->lssc_m2, loc_lssc);
995 if (opts->ulsc_m2 != 0)
996 print_hfsc_sc("upperlimit", opts->ulsc_m1, opts->ulsc_d,
997 opts->ulsc_m2, ulsc);
998 printf(" ) ");
1000 return (1);
1001 } else
1002 return (0);
1005 static int
1006 print_fairq_opts(const struct pf_altq *a, const struct node_queue_opt *qopts)
1008 const struct fairq_opts *opts;
1009 const struct node_fairq_sc *loc_lssc;
1011 opts = &a->pq_u.fairq_opts;
1012 if (qopts == NULL)
1013 loc_lssc = NULL;
1014 else
1015 loc_lssc = &qopts->data.fairq_opts.linkshare;
1017 if (opts->flags ||
1018 (opts->lssc_m2 != 0 && (opts->lssc_m2 != a->bandwidth ||
1019 opts->lssc_d != 0))) {
1020 printf("fairq(");
1021 if (opts->flags & FARF_RED)
1022 printf(" red");
1023 if (opts->flags & FARF_ECN)
1024 printf(" ecn");
1025 if (opts->flags & FARF_RIO)
1026 printf(" rio");
1027 if (opts->flags & FARF_CLEARDSCP)
1028 printf(" cleardscp");
1029 if (opts->flags & FARF_DEFAULTCLASS)
1030 printf(" default");
1031 if (opts->lssc_m2 != 0 && (opts->lssc_m2 != a->bandwidth ||
1032 opts->lssc_d != 0))
1033 print_fairq_sc("linkshare", opts->lssc_m1, opts->lssc_d,
1034 opts->lssc_m2, loc_lssc);
1035 printf(" ) ");
1037 return (1);
1038 } else
1039 return (0);
1043 * admission control using generalized service curve
1045 #ifndef INFINITY
1046 #define INFINITY HUGE_VAL /* positive infinity defined in <math.h> */
1047 #endif
1049 /* add a new service curve to a generalized service curve */
1050 static void
1051 gsc_add_sc(struct gen_sc *gsc, struct service_curve *sc)
1053 if (is_sc_null(sc))
1054 return;
1055 if (sc->d != 0)
1056 gsc_add_seg(gsc, 0.0, 0.0, (double)sc->d, (double)sc->m1);
1057 gsc_add_seg(gsc, (double)sc->d, 0.0, INFINITY, (double)sc->m2);
1061 * check whether all points of a generalized service curve have
1062 * their y-coordinates no larger than a given two-piece linear
1063 * service curve.
1065 static int
1066 is_gsc_under_sc(struct gen_sc *gsc, struct service_curve *sc)
1068 struct segment *s, *last, *end;
1069 double y;
1071 if (is_sc_null(sc)) {
1072 if (LIST_EMPTY(gsc))
1073 return (1);
1074 LIST_FOREACH(s, gsc, _next) {
1075 if (s->m != 0)
1076 return (0);
1078 return (1);
1081 * gsc has a dummy entry at the end with x = INFINITY.
1082 * loop through up to this dummy entry.
1084 end = gsc_getentry(gsc, INFINITY);
1085 if (end == NULL)
1086 return (1);
1087 last = NULL;
1088 for (s = LIST_FIRST(gsc); s != end; s = LIST_NEXT(s, _next)) {
1089 if (s->y > sc_x2y(sc, s->x))
1090 return (0);
1091 last = s;
1093 /* last now holds the real last segment */
1094 if (last == NULL)
1095 return (1);
1096 if (last->m > sc->m2)
1097 return (0);
1098 if (last->x < sc->d && last->m > sc->m1) {
1099 y = last->y + (sc->d - last->x) * last->m;
1100 if (y > sc_x2y(sc, sc->d))
1101 return (0);
1103 return (1);
1106 static void
1107 gsc_destroy(struct gen_sc *gsc)
1109 struct segment *s;
1111 while ((s = LIST_FIRST(gsc)) != NULL) {
1112 LIST_REMOVE(s, _next);
1113 free(s);
1118 * return a segment entry starting at x.
1119 * if gsc has no entry starting at x, a new entry is created at x.
1121 static struct segment *
1122 gsc_getentry(struct gen_sc *gsc, double x)
1124 struct segment *new, *prev, *s;
1126 prev = NULL;
1127 LIST_FOREACH(s, gsc, _next) {
1128 if (s->x == x)
1129 return (s); /* matching entry found */
1130 else if (s->x < x)
1131 prev = s;
1132 else
1133 break;
1136 /* we have to create a new entry */
1137 if ((new = calloc(1, sizeof(struct segment))) == NULL)
1138 return (NULL);
1140 new->x = x;
1141 if (x == INFINITY || s == NULL)
1142 new->d = 0;
1143 else if (s->x == INFINITY)
1144 new->d = INFINITY;
1145 else
1146 new->d = s->x - x;
1147 if (prev == NULL) {
1148 /* insert the new entry at the head of the list */
1149 new->y = 0;
1150 new->m = 0;
1151 LIST_INSERT_HEAD(gsc, new, _next);
1152 } else {
1154 * the start point intersects with the segment pointed by
1155 * prev. divide prev into 2 segments
1157 if (x == INFINITY) {
1158 prev->d = INFINITY;
1159 if (prev->m == 0)
1160 new->y = prev->y;
1161 else
1162 new->y = INFINITY;
1163 } else {
1164 prev->d = x - prev->x;
1165 new->y = prev->d * prev->m + prev->y;
1167 new->m = prev->m;
1168 LIST_INSERT_AFTER(prev, new, _next);
1170 return (new);
1173 /* add a segment to a generalized service curve */
1174 static int
1175 gsc_add_seg(struct gen_sc *gsc, double x, double y, double d, double m)
1177 struct segment *start, *end, *s;
1178 double x2;
1180 if (d == INFINITY)
1181 x2 = INFINITY;
1182 else
1183 x2 = x + d;
1184 start = gsc_getentry(gsc, x);
1185 end = gsc_getentry(gsc, x2);
1186 if (start == NULL || end == NULL)
1187 return (-1);
1189 for (s = start; s != end; s = LIST_NEXT(s, _next)) {
1190 s->m += m;
1191 s->y += y + (s->x - x) * m;
1194 end = gsc_getentry(gsc, INFINITY);
1195 for (; s != end; s = LIST_NEXT(s, _next)) {
1196 s->y += m * d;
1199 return (0);
1202 /* get y-projection of a service curve */
1203 static double
1204 sc_x2y(struct service_curve *sc, double x)
1206 double y;
1208 if (x <= (double)sc->d)
1209 /* y belongs to the 1st segment */
1210 y = x * (double)sc->m1;
1211 else
1212 /* y belongs to the 2nd segment */
1213 y = (double)sc->d * (double)sc->m1
1214 + (x - (double)sc->d) * (double)sc->m2;
1215 return (y);
1219 * misc utilities
1221 #define R2S_BUFS 8
1222 #define RATESTR_MAX 16
1224 char *
1225 rate2str(double rate)
1227 char *buf;
1228 static char r2sbuf[R2S_BUFS][RATESTR_MAX]; /* ring bufer */
1229 static int idx = 0;
1230 int i;
1231 static const char unit[] = " KMG";
1233 buf = r2sbuf[idx++];
1234 if (idx == R2S_BUFS)
1235 idx = 0;
1237 for (i = 0; rate >= 1000 && i <= 3; i++)
1238 rate /= 1000;
1240 if ((int)(rate * 100) % 100)
1241 snprintf(buf, RATESTR_MAX, "%.2f%cb", rate, unit[i]);
1242 else
1243 snprintf(buf, RATESTR_MAX, "%d%cb", (int)rate, unit[i]);
1245 return (buf);
1248 u_int32_t
1249 getifspeed(const char *ifname)
1251 size_t datalen;
1252 int idx;
1253 struct ifmibdata data;
1254 int name[] = {
1255 CTL_NET,
1256 PF_LINK,
1257 NETLINK_GENERIC,
1258 IFMIB_IFDATA,
1260 IFDATA_GENERAL
1263 if ((idx = (int)if_nametoindex(ifname)) == 0)
1264 err(1, "getifspeed: if_nametoindex");
1265 name[4] = idx;
1267 datalen = sizeof(data);
1268 if (sysctl(name, 6, &data, &datalen, NULL, 0))
1269 err(1, "getifspeed: sysctl");
1271 return(data.ifmd_data.ifi_baudrate);
1274 u_long
1275 getifmtu(char *ifname)
1277 int s;
1278 struct ifreq ifr;
1280 if ((s = socket(AF_INET, SOCK_DGRAM, 0)) < 0)
1281 err(1, "socket");
1282 if (strlcpy(ifr.ifr_name, ifname, sizeof(ifr.ifr_name)) >=
1283 sizeof(ifr.ifr_name))
1284 errx(1, "getifmtu: strlcpy");
1285 if (ioctl(s, SIOCGIFMTU, (caddr_t)&ifr) == -1)
1286 err(1, "SIOCGIFMTU");
1287 if (shutdown(s, SHUT_RDWR) == -1)
1288 err(1, "shutdown");
1289 if (close(s))
1290 err(1, "close");
1291 if (ifr.ifr_mtu > 0)
1292 return (ifr.ifr_mtu);
1293 else {
1294 warnx("could not get mtu for %s, assuming 1500", ifname);
1295 return (1500);
1300 eval_queue_opts(struct pf_altq *pa, struct node_queue_opt *opts,
1301 u_int32_t ref_bw)
1303 int errors = 0;
1305 switch (pa->scheduler) {
1306 case ALTQT_CBQ:
1307 pa->pq_u.cbq_opts = opts->data.cbq_opts;
1308 break;
1309 case ALTQT_PRIQ:
1310 pa->pq_u.priq_opts = opts->data.priq_opts;
1311 break;
1312 case ALTQT_HFSC:
1313 pa->pq_u.hfsc_opts.flags = opts->data.hfsc_opts.flags;
1314 if (opts->data.hfsc_opts.linkshare.used) {
1315 pa->pq_u.hfsc_opts.lssc_m1 =
1316 eval_bwspec(&opts->data.hfsc_opts.linkshare.m1,
1317 ref_bw);
1318 pa->pq_u.hfsc_opts.lssc_m2 =
1319 eval_bwspec(&opts->data.hfsc_opts.linkshare.m2,
1320 ref_bw);
1321 pa->pq_u.hfsc_opts.lssc_d =
1322 opts->data.hfsc_opts.linkshare.d;
1324 if (opts->data.hfsc_opts.realtime.used) {
1325 pa->pq_u.hfsc_opts.rtsc_m1 =
1326 eval_bwspec(&opts->data.hfsc_opts.realtime.m1,
1327 ref_bw);
1328 pa->pq_u.hfsc_opts.rtsc_m2 =
1329 eval_bwspec(&opts->data.hfsc_opts.realtime.m2,
1330 ref_bw);
1331 pa->pq_u.hfsc_opts.rtsc_d =
1332 opts->data.hfsc_opts.realtime.d;
1334 if (opts->data.hfsc_opts.upperlimit.used) {
1335 pa->pq_u.hfsc_opts.ulsc_m1 =
1336 eval_bwspec(&opts->data.hfsc_opts.upperlimit.m1,
1337 ref_bw);
1338 pa->pq_u.hfsc_opts.ulsc_m2 =
1339 eval_bwspec(&opts->data.hfsc_opts.upperlimit.m2,
1340 ref_bw);
1341 pa->pq_u.hfsc_opts.ulsc_d =
1342 opts->data.hfsc_opts.upperlimit.d;
1344 break;
1345 case ALTQT_FAIRQ:
1346 pa->pq_u.fairq_opts.flags = opts->data.fairq_opts.flags;
1347 pa->pq_u.fairq_opts.nbuckets = opts->data.fairq_opts.nbuckets;
1348 pa->pq_u.fairq_opts.hogs_m1 =
1349 eval_bwspec(&opts->data.fairq_opts.hogs_bw, ref_bw);
1351 if (opts->data.fairq_opts.linkshare.used) {
1352 pa->pq_u.fairq_opts.lssc_m1 =
1353 eval_bwspec(&opts->data.fairq_opts.linkshare.m1,
1354 ref_bw);
1355 pa->pq_u.fairq_opts.lssc_m2 =
1356 eval_bwspec(&opts->data.fairq_opts.linkshare.m2,
1357 ref_bw);
1358 pa->pq_u.fairq_opts.lssc_d =
1359 opts->data.fairq_opts.linkshare.d;
1361 break;
1362 default:
1363 warnx("eval_queue_opts: unknown scheduler type %u",
1364 opts->qtype);
1365 errors++;
1366 break;
1369 return (errors);
1372 u_int32_t
1373 eval_bwspec(struct node_queue_bw *bw, u_int32_t ref_bw)
1375 if (bw->bw_absolute > 0)
1376 return (bw->bw_absolute);
1378 if (bw->bw_percent > 0)
1379 return (ref_bw / 100 * bw->bw_percent);
1381 return (0);
1384 void
1385 print_hfsc_sc(const char *scname, u_int m1, u_int d, u_int m2,
1386 const struct node_hfsc_sc *sc)
1388 printf(" %s", scname);
1390 if (d != 0) {
1391 printf("(");
1392 if (sc != NULL && sc->m1.bw_percent > 0)
1393 printf("%u%%", sc->m1.bw_percent);
1394 else
1395 printf("%s", rate2str((double)m1));
1396 printf(" %u", d);
1399 if (sc != NULL && sc->m2.bw_percent > 0)
1400 printf(" %u%%", sc->m2.bw_percent);
1401 else
1402 printf(" %s", rate2str((double)m2));
1404 if (d != 0)
1405 printf(")");
1408 void
1409 print_fairq_sc(const char *scname, u_int m1, u_int d, u_int m2,
1410 const struct node_fairq_sc *sc)
1412 printf(" %s", scname);
1414 if (d != 0) {
1415 printf("(");
1416 if (sc != NULL && sc->m1.bw_percent > 0)
1417 printf("%u%%", sc->m1.bw_percent);
1418 else
1419 printf("%s", rate2str((double)m1));
1420 printf(" %u", d);
1423 if (sc != NULL && sc->m2.bw_percent > 0)
1424 printf(" %u%%", sc->m2.bw_percent);
1425 else
1426 printf(" %s", rate2str((double)m2));
1428 if (d != 0)
1429 printf(")");