kernel - cleanup vfs_cache debugging
[dragonfly.git] / sys / net / if_poll.c
blob059a596b635fcbf96be7b55abde04afd984b58b1
1 /*-
2 * Copyright (c) 2001-2002 Luigi Rizzo
4 * Supported by: the Xorp Project (www.xorp.org)
6 * Redistribution and use in source and binary forms, with or without
7 * modification, are permitted provided that the following conditions
8 * are met:
9 * 1. Redistributions of source code must retain the above copyright
10 * notice, this list of conditions and the following disclaimer.
11 * 2. Redistributions in binary form must reproduce the above copyright
12 * notice, this list of conditions and the following disclaimer in the
13 * documentation and/or other materials provided with the distribution.
15 * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND
16 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
17 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
18 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE
19 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
20 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
25 * SUCH DAMAGE.
27 * $FreeBSD: src/sys/kern/kern_poll.c,v 1.2.2.4 2002/06/27 23:26:33 luigi Exp $
30 #include "opt_ifpoll.h"
32 #include <sys/param.h>
33 #include <sys/kernel.h>
34 #include <sys/ktr.h>
35 #include <sys/malloc.h>
36 #include <sys/serialize.h>
37 #include <sys/socket.h>
38 #include <sys/sysctl.h>
39 #include <sys/microtime_pcpu.h>
41 #include <sys/thread2.h>
42 #include <sys/msgport2.h>
44 #include <net/if.h>
45 #include <net/if_poll.h>
46 #include <net/netmsg2.h>
47 #include <net/netisr2.h>
50 * Polling support for network device drivers.
52 * Drivers which support this feature try to register one status polling
53 * handler and several TX/RX polling handlers with the polling code.
54 * If interface's if_npoll is called with non-NULL second argument, then
55 * a register operation is requested, else a deregister operation is
56 * requested. If the requested operation is "register", driver should
57 * setup the ifpoll_info passed in accoding its own needs:
58 * ifpoll_info.ifpi_status.status_func == NULL
59 * No status polling handler will be installed on CPU(0)
60 * ifpoll_info.ifpi_rx[n].poll_func == NULL
61 * No RX polling handler will be installed on CPU(n)
62 * ifpoll_info.ifpi_tx[n].poll_func == NULL
63 * No TX polling handler will be installed on CPU(n)
65 * RX is polled at the specified polling frequency (net.ifpoll.X.pollhz).
66 * TX and status polling could be done at lower frequency than RX frequency
67 * (net.ifpoll.0.status_frac and net.ifpoll.X.tx_frac). To avoid systimer
68 * staggering at high frequency, RX systimer gives TX and status polling a
69 * piggyback (XXX).
71 * All of the registered polling handlers are called only if the interface
72 * is marked as 'IFF_RUNNING and IFF_NPOLLING'. However, the interface's
73 * register and deregister function (ifnet.if_npoll) will be called even
74 * if interface is not marked with 'IFF_RUNNING'.
76 * If registration is successful, the driver must disable interrupts,
77 * and further I/O is performed through the TX/RX polling handler, which
78 * are invoked (at least once per clock tick) with 3 arguments: the "arg"
79 * passed at register time, a struct ifnet pointer, and a "count" limit.
80 * The registered serializer will be held before calling the related
81 * polling handler.
83 * The count limit specifies how much work the handler can do during the
84 * call -- typically this is the number of packets to be received, or
85 * transmitted, etc. (drivers are free to interpret this number, as long
86 * as the max time spent in the function grows roughly linearly with the
87 * count).
89 * A second variable controls the sharing of CPU between polling/kernel
90 * network processing, and other activities (typically userlevel tasks):
91 * net.ifpoll.X.{rx,tx}.user_frac (between 0 and 100, default 50) sets the
92 * share of CPU allocated to user tasks. CPU is allocated proportionally
93 * to the shares, by dynamically adjusting the "count" (poll_burst).
95 * Other parameters can should be left to their default values.
96 * The following constraints hold
98 * 1 <= poll_burst <= poll_burst_max
99 * 1 <= poll_each_burst <= poll_burst_max
100 * MIN_POLL_BURST_MAX <= poll_burst_max <= MAX_POLL_BURST_MAX
103 #define IFPOLL_LIST_LEN 128
104 #define IFPOLL_FREQ_MAX 30000
106 #define MIN_IOPOLL_BURST_MAX 10
107 #define MAX_IOPOLL_BURST_MAX 5000
108 #define IOPOLL_BURST_MAX 250 /* good for 1000Mbit net and HZ=6000 */
110 #define IOPOLL_EACH_BURST 50
111 #define IOPOLL_USER_FRAC 50
113 #define IFPOLL_FREQ_DEFAULT 6000
115 #define IFPOLL_TXFRAC_DEFAULT 1 /* 1/1 of the pollhz */
116 #define IFPOLL_STFRAC_DEFAULT 120 /* 1/120 of the pollhz */
118 #define IFPOLL_RX 0x1
119 #define IFPOLL_TX 0x2
121 struct iopoll_rec {
122 struct lwkt_serialize *serializer;
123 struct ifnet *ifp;
124 void *arg;
125 ifpoll_iofn_t poll_func;
128 struct iopoll_ctx {
129 union microtime_pcpu prev_t;
130 u_long short_ticks; /* statistics */
131 u_long lost_polls; /* statistics */
132 u_long suspect; /* statistics */
133 u_long stalled; /* statistics */
134 uint32_t pending_polls; /* state */
136 struct netmsg_base poll_netmsg;
137 struct netmsg_base poll_more_netmsg;
139 int poll_cpuid;
140 int pollhz;
141 uint32_t phase; /* state */
142 int residual_burst; /* state */
143 uint32_t poll_each_burst; /* tunable */
144 union microtime_pcpu poll_start_t; /* state */
146 uint32_t poll_burst; /* state */
147 uint32_t poll_burst_max; /* tunable */
148 uint32_t user_frac; /* tunable */
149 uint32_t kern_frac; /* state */
151 uint32_t poll_handlers; /* next free entry in pr[]. */
152 struct iopoll_rec pr[IFPOLL_LIST_LEN];
154 struct sysctl_ctx_list poll_sysctl_ctx;
155 struct sysctl_oid *poll_sysctl_tree;
156 } __cachealign;
158 struct poll_comm {
159 struct systimer pollclock;
160 int poll_cpuid;
162 int stfrac_count; /* state */
163 int poll_stfrac; /* tunable */
165 int txfrac_count; /* state */
166 int poll_txfrac; /* tunable */
168 int pollhz; /* tunable */
170 struct sysctl_ctx_list sysctl_ctx;
171 struct sysctl_oid *sysctl_tree;
172 } __cachealign;
174 struct stpoll_rec {
175 struct lwkt_serialize *serializer;
176 struct ifnet *ifp;
177 ifpoll_stfn_t status_func;
180 struct stpoll_ctx {
181 struct netmsg_base poll_netmsg;
183 uint32_t poll_handlers; /* next free entry in pr[]. */
184 struct stpoll_rec pr[IFPOLL_LIST_LEN];
186 struct sysctl_ctx_list poll_sysctl_ctx;
187 struct sysctl_oid *poll_sysctl_tree;
188 } __cachealign;
190 struct iopoll_sysctl_netmsg {
191 struct netmsg_base base;
192 struct iopoll_ctx *ctx;
195 void ifpoll_init_pcpu(int);
196 static void ifpoll_register_handler(netmsg_t);
197 static void ifpoll_deregister_handler(netmsg_t);
200 * Status polling
202 static void stpoll_init(void);
203 static void stpoll_handler(netmsg_t);
204 static void stpoll_clock(struct stpoll_ctx *);
205 static int stpoll_register(struct ifnet *, const struct ifpoll_status *);
206 static int stpoll_deregister(struct ifnet *);
209 * RX/TX polling
211 static struct iopoll_ctx *iopoll_ctx_create(int, int);
212 static void iopoll_init(int);
213 static void rxpoll_handler(netmsg_t);
214 static void txpoll_handler(netmsg_t);
215 static void rxpollmore_handler(netmsg_t);
216 static void txpollmore_handler(netmsg_t);
217 static void iopoll_clock(struct iopoll_ctx *);
218 static int iopoll_register(struct ifnet *, struct iopoll_ctx *,
219 const struct ifpoll_io *);
220 static int iopoll_deregister(struct ifnet *, struct iopoll_ctx *);
222 static void iopoll_add_sysctl(struct sysctl_ctx_list *,
223 struct sysctl_oid_list *, struct iopoll_ctx *, int);
224 static void sysctl_burstmax_handler(netmsg_t);
225 static int sysctl_burstmax(SYSCTL_HANDLER_ARGS);
226 static void sysctl_eachburst_handler(netmsg_t);
227 static int sysctl_eachburst(SYSCTL_HANDLER_ARGS);
230 * Common functions
232 static void poll_comm_init(int);
233 static void poll_comm_start(int);
234 static void poll_comm_adjust_pollhz(struct poll_comm *);
235 static void poll_comm_systimer0(systimer_t, int, struct intrframe *);
236 static void poll_comm_systimer(systimer_t, int, struct intrframe *);
237 static void sysctl_pollhz_handler(netmsg_t);
238 static void sysctl_stfrac_handler(netmsg_t);
239 static void sysctl_txfrac_handler(netmsg_t);
240 static int sysctl_pollhz(SYSCTL_HANDLER_ARGS);
241 static int sysctl_stfrac(SYSCTL_HANDLER_ARGS);
242 static int sysctl_txfrac(SYSCTL_HANDLER_ARGS);
243 static int sysctl_compat_npoll_stfrac(SYSCTL_HANDLER_ARGS);
244 static int sysctl_compat_npoll_cpuid(SYSCTL_HANDLER_ARGS);
246 static struct stpoll_ctx stpoll_context;
247 static struct poll_comm *poll_common[MAXCPU];
248 static struct iopoll_ctx *rxpoll_context[MAXCPU];
249 static struct iopoll_ctx *txpoll_context[MAXCPU];
251 SYSCTL_NODE(_net, OID_AUTO, ifpoll, CTLFLAG_RW, 0,
252 "Network device polling parameters");
254 static int iopoll_burst_max = IOPOLL_BURST_MAX;
255 static int iopoll_each_burst = IOPOLL_EACH_BURST;
256 static int iopoll_user_frac = IOPOLL_USER_FRAC;
258 static int ifpoll_pollhz = IFPOLL_FREQ_DEFAULT;
259 static int ifpoll_stfrac = IFPOLL_STFRAC_DEFAULT;
260 static int ifpoll_txfrac = IFPOLL_TXFRAC_DEFAULT;
262 TUNABLE_INT("net.ifpoll.burst_max", &iopoll_burst_max);
263 TUNABLE_INT("net.ifpoll.each_burst", &iopoll_each_burst);
264 TUNABLE_INT("net.ifpoll.user_frac", &iopoll_user_frac);
265 TUNABLE_INT("net.ifpoll.pollhz", &ifpoll_pollhz);
266 TUNABLE_INT("net.ifpoll.status_frac", &ifpoll_stfrac);
267 TUNABLE_INT("net.ifpoll.tx_frac", &ifpoll_txfrac);
269 #if !defined(KTR_IF_POLL)
270 #define KTR_IF_POLL KTR_ALL
271 #endif
272 KTR_INFO_MASTER(if_poll);
273 KTR_INFO(KTR_IF_POLL, if_poll, rx_start, 0, "rx start");
274 KTR_INFO(KTR_IF_POLL, if_poll, rx_end, 1, "rx end");
275 KTR_INFO(KTR_IF_POLL, if_poll, tx_start, 2, "tx start");
276 KTR_INFO(KTR_IF_POLL, if_poll, tx_end, 3, "tx end");
277 KTR_INFO(KTR_IF_POLL, if_poll, rx_mstart, 4, "rx more start");
278 KTR_INFO(KTR_IF_POLL, if_poll, rx_mend, 5, "rx more end");
279 KTR_INFO(KTR_IF_POLL, if_poll, tx_mstart, 6, "tx more start");
280 KTR_INFO(KTR_IF_POLL, if_poll, tx_mend, 7, "tx more end");
281 KTR_INFO(KTR_IF_POLL, if_poll, ioclock_start, 8, "ioclock start");
282 KTR_INFO(KTR_IF_POLL, if_poll, ioclock_end, 9, "ioclock end");
283 #define logpoll(name) KTR_LOG(if_poll_ ## name)
285 #define IFPOLL_FREQ_ADJ(comm) (((comm)->poll_cpuid * 3) % 50)
287 static __inline int
288 poll_comm_pollhz_div(const struct poll_comm *comm, int pollhz)
290 return pollhz + IFPOLL_FREQ_ADJ(comm);
293 static __inline int
294 poll_comm_pollhz_conv(const struct poll_comm *comm, int pollhz)
296 return pollhz - IFPOLL_FREQ_ADJ(comm);
299 static __inline void
300 ifpoll_sendmsg_oncpu(netmsg_t msg)
302 if (msg->lmsg.ms_flags & MSGF_DONE)
303 lwkt_sendmsg_oncpu(netisr_cpuport(mycpuid), &msg->lmsg);
306 static __inline void
307 sched_stpoll(struct stpoll_ctx *st_ctx)
309 ifpoll_sendmsg_oncpu((netmsg_t)&st_ctx->poll_netmsg);
312 static __inline void
313 sched_iopoll(struct iopoll_ctx *io_ctx)
315 ifpoll_sendmsg_oncpu((netmsg_t)&io_ctx->poll_netmsg);
318 static __inline void
319 sched_iopollmore(struct iopoll_ctx *io_ctx)
321 ifpoll_sendmsg_oncpu((netmsg_t)&io_ctx->poll_more_netmsg);
325 * Initialize per-cpu polling(4) context. Called from kern_clock.c:
327 void
328 ifpoll_init_pcpu(int cpuid)
330 if (cpuid >= ncpus2)
331 return;
333 poll_comm_init(cpuid);
335 if (cpuid == 0)
336 stpoll_init();
337 iopoll_init(cpuid);
339 poll_comm_start(cpuid);
343 ifpoll_register(struct ifnet *ifp)
345 struct ifpoll_info *info;
346 struct netmsg_base nmsg;
347 int error;
349 if (ifp->if_npoll == NULL) {
350 /* Device does not support polling */
351 return EOPNOTSUPP;
354 info = kmalloc(sizeof(*info), M_TEMP, M_WAITOK | M_ZERO);
357 * Attempt to register. Interlock with IFF_NPOLLING.
360 ifnet_serialize_all(ifp);
362 if (ifp->if_flags & IFF_NPOLLING) {
363 /* Already polling */
364 ifnet_deserialize_all(ifp);
365 kfree(info, M_TEMP);
366 return EBUSY;
369 info->ifpi_ifp = ifp;
371 ifp->if_flags |= IFF_NPOLLING;
372 ifp->if_npoll(ifp, info);
374 ifnet_deserialize_all(ifp);
376 netmsg_init(&nmsg, NULL, &curthread->td_msgport,
377 0, ifpoll_register_handler);
378 nmsg.lmsg.u.ms_resultp = info;
380 error = lwkt_domsg(netisr_cpuport(0), &nmsg.lmsg, 0);
381 if (error) {
382 if (!ifpoll_deregister(ifp)) {
383 if_printf(ifp, "ifpoll_register: "
384 "ifpoll_deregister failed!\n");
388 kfree(info, M_TEMP);
389 return error;
393 ifpoll_deregister(struct ifnet *ifp)
395 struct netmsg_base nmsg;
396 int error;
398 if (ifp->if_npoll == NULL)
399 return EOPNOTSUPP;
401 ifnet_serialize_all(ifp);
403 if ((ifp->if_flags & IFF_NPOLLING) == 0) {
404 ifnet_deserialize_all(ifp);
405 return EINVAL;
407 ifp->if_flags &= ~IFF_NPOLLING;
409 ifnet_deserialize_all(ifp);
411 netmsg_init(&nmsg, NULL, &curthread->td_msgport,
412 0, ifpoll_deregister_handler);
413 nmsg.lmsg.u.ms_resultp = ifp;
415 error = lwkt_domsg(netisr_cpuport(0), &nmsg.lmsg, 0);
416 if (!error) {
417 ifnet_serialize_all(ifp);
418 ifp->if_npoll(ifp, NULL);
419 ifnet_deserialize_all(ifp);
421 return error;
424 static void
425 ifpoll_register_handler(netmsg_t nmsg)
427 const struct ifpoll_info *info = nmsg->lmsg.u.ms_resultp;
428 int cpuid = mycpuid, nextcpu;
429 int error;
431 KKASSERT(cpuid < ncpus2);
432 KKASSERT(&curthread->td_msgport == netisr_cpuport(cpuid));
434 if (cpuid == 0) {
435 error = stpoll_register(info->ifpi_ifp, &info->ifpi_status);
436 if (error)
437 goto failed;
440 error = iopoll_register(info->ifpi_ifp, rxpoll_context[cpuid],
441 &info->ifpi_rx[cpuid]);
442 if (error)
443 goto failed;
445 error = iopoll_register(info->ifpi_ifp, txpoll_context[cpuid],
446 &info->ifpi_tx[cpuid]);
447 if (error)
448 goto failed;
450 /* Adjust polling frequency, after all registration is done */
451 poll_comm_adjust_pollhz(poll_common[cpuid]);
453 nextcpu = cpuid + 1;
454 if (nextcpu < ncpus2)
455 lwkt_forwardmsg(netisr_cpuport(nextcpu), &nmsg->lmsg);
456 else
457 lwkt_replymsg(&nmsg->lmsg, 0);
458 return;
459 failed:
460 lwkt_replymsg(&nmsg->lmsg, error);
463 static void
464 ifpoll_deregister_handler(netmsg_t nmsg)
466 struct ifnet *ifp = nmsg->lmsg.u.ms_resultp;
467 int cpuid = mycpuid, nextcpu;
469 KKASSERT(cpuid < ncpus2);
470 KKASSERT(&curthread->td_msgport == netisr_cpuport(cpuid));
472 /* Ignore errors */
473 if (cpuid == 0)
474 stpoll_deregister(ifp);
475 iopoll_deregister(ifp, rxpoll_context[cpuid]);
476 iopoll_deregister(ifp, txpoll_context[cpuid]);
478 /* Adjust polling frequency, after all deregistration is done */
479 poll_comm_adjust_pollhz(poll_common[cpuid]);
481 nextcpu = cpuid + 1;
482 if (nextcpu < ncpus2)
483 lwkt_forwardmsg(netisr_cpuport(nextcpu), &nmsg->lmsg);
484 else
485 lwkt_replymsg(&nmsg->lmsg, 0);
488 static void
489 stpoll_init(void)
491 struct stpoll_ctx *st_ctx = &stpoll_context;
492 const struct poll_comm *comm = poll_common[0];
494 sysctl_ctx_init(&st_ctx->poll_sysctl_ctx);
495 st_ctx->poll_sysctl_tree = SYSCTL_ADD_NODE(&st_ctx->poll_sysctl_ctx,
496 SYSCTL_CHILDREN(comm->sysctl_tree),
497 OID_AUTO, "status", CTLFLAG_RD, 0, "");
499 SYSCTL_ADD_UINT(&st_ctx->poll_sysctl_ctx,
500 SYSCTL_CHILDREN(st_ctx->poll_sysctl_tree),
501 OID_AUTO, "handlers", CTLFLAG_RD,
502 &st_ctx->poll_handlers, 0,
503 "Number of registered status poll handlers");
505 netmsg_init(&st_ctx->poll_netmsg, NULL, &netisr_adone_rport,
506 0, stpoll_handler);
510 * stpoll_handler is scheduled by sched_stpoll when appropriate, typically
511 * once per polling systimer tick.
513 static void
514 stpoll_handler(netmsg_t msg)
516 struct stpoll_ctx *st_ctx = &stpoll_context;
517 struct thread *td = curthread;
518 int i;
520 ASSERT_IS_NETISR(td, 0);
522 crit_enter_quick(td);
524 /* Reply ASAP */
525 lwkt_replymsg(&msg->lmsg, 0);
527 if (st_ctx->poll_handlers == 0) {
528 crit_exit_quick(td);
529 return;
532 for (i = 0; i < st_ctx->poll_handlers; ++i) {
533 const struct stpoll_rec *rec = &st_ctx->pr[i];
534 struct ifnet *ifp = rec->ifp;
536 if (!lwkt_serialize_try(rec->serializer))
537 continue;
539 if ((ifp->if_flags & (IFF_RUNNING | IFF_NPOLLING)) ==
540 (IFF_RUNNING | IFF_NPOLLING))
541 rec->status_func(ifp);
543 lwkt_serialize_exit(rec->serializer);
546 crit_exit_quick(td);
550 * Hook from status poll systimer. Tries to schedule an status poll.
551 * NOTE: Caller should hold critical section.
553 static void
554 stpoll_clock(struct stpoll_ctx *st_ctx)
556 KKASSERT(mycpuid == 0);
558 if (st_ctx->poll_handlers == 0)
559 return;
560 sched_stpoll(st_ctx);
563 static int
564 stpoll_register(struct ifnet *ifp, const struct ifpoll_status *st_rec)
566 struct stpoll_ctx *st_ctx = &stpoll_context;
567 int error;
569 ASSERT_IN_NETISR(0);
571 if (st_rec->status_func == NULL)
572 return 0;
575 * Check if there is room.
577 if (st_ctx->poll_handlers >= IFPOLL_LIST_LEN) {
579 * List full, cannot register more entries.
580 * This should never happen; if it does, it is probably a
581 * broken driver trying to register multiple times. Checking
582 * this at runtime is expensive, and won't solve the problem
583 * anyways, so just report a few times and then give up.
585 static int verbose = 10; /* XXX */
587 if (verbose > 0) {
588 kprintf("status poll handlers list full, "
589 "maybe a broken driver ?\n");
590 verbose--;
592 error = ENOENT;
593 } else {
594 struct stpoll_rec *rec = &st_ctx->pr[st_ctx->poll_handlers];
596 rec->ifp = ifp;
597 rec->serializer = st_rec->serializer;
598 rec->status_func = st_rec->status_func;
600 st_ctx->poll_handlers++;
601 error = 0;
603 return error;
606 static int
607 stpoll_deregister(struct ifnet *ifp)
609 struct stpoll_ctx *st_ctx = &stpoll_context;
610 int i, error;
612 ASSERT_IN_NETISR(0);
614 for (i = 0; i < st_ctx->poll_handlers; ++i) {
615 if (st_ctx->pr[i].ifp == ifp) /* Found it */
616 break;
618 if (i == st_ctx->poll_handlers) {
619 error = ENOENT;
620 } else {
621 st_ctx->poll_handlers--;
622 if (i < st_ctx->poll_handlers) {
623 /* Last entry replaces this one. */
624 st_ctx->pr[i] = st_ctx->pr[st_ctx->poll_handlers];
626 error = 0;
628 return error;
631 static __inline void
632 iopoll_reset_state(struct iopoll_ctx *io_ctx)
634 crit_enter();
635 io_ctx->poll_burst = io_ctx->poll_each_burst;
636 io_ctx->pending_polls = 0;
637 io_ctx->residual_burst = 0;
638 io_ctx->phase = 0;
639 io_ctx->kern_frac = 0;
640 bzero(&io_ctx->poll_start_t, sizeof(io_ctx->poll_start_t));
641 bzero(&io_ctx->prev_t, sizeof(io_ctx->prev_t));
642 crit_exit();
645 static void
646 iopoll_init(int cpuid)
648 KKASSERT(cpuid < ncpus2);
650 rxpoll_context[cpuid] = iopoll_ctx_create(cpuid, IFPOLL_RX);
651 txpoll_context[cpuid] = iopoll_ctx_create(cpuid, IFPOLL_TX);
654 static struct iopoll_ctx *
655 iopoll_ctx_create(int cpuid, int poll_type)
657 struct poll_comm *comm;
658 struct iopoll_ctx *io_ctx;
659 const char *poll_type_str;
660 netisr_fn_t handler, more_handler;
662 KKASSERT(poll_type == IFPOLL_RX || poll_type == IFPOLL_TX);
665 * Make sure that tunables are in sane state
667 if (iopoll_burst_max < MIN_IOPOLL_BURST_MAX)
668 iopoll_burst_max = MIN_IOPOLL_BURST_MAX;
669 else if (iopoll_burst_max > MAX_IOPOLL_BURST_MAX)
670 iopoll_burst_max = MAX_IOPOLL_BURST_MAX;
672 if (iopoll_each_burst > iopoll_burst_max)
673 iopoll_each_burst = iopoll_burst_max;
675 comm = poll_common[cpuid];
678 * Create the per-cpu polling context
680 io_ctx = kmalloc_cachealign(sizeof(*io_ctx), M_DEVBUF,
681 M_WAITOK | M_ZERO);
683 io_ctx->poll_each_burst = iopoll_each_burst;
684 io_ctx->poll_burst_max = iopoll_burst_max;
685 io_ctx->user_frac = iopoll_user_frac;
686 if (poll_type == IFPOLL_RX)
687 io_ctx->pollhz = comm->pollhz;
688 else
689 io_ctx->pollhz = comm->pollhz / (comm->poll_txfrac + 1);
690 io_ctx->poll_cpuid = cpuid;
691 iopoll_reset_state(io_ctx);
693 if (poll_type == IFPOLL_RX) {
694 handler = rxpoll_handler;
695 more_handler = rxpollmore_handler;
696 } else {
697 handler = txpoll_handler;
698 more_handler = txpollmore_handler;
701 netmsg_init(&io_ctx->poll_netmsg, NULL, &netisr_adone_rport,
702 0, handler);
703 io_ctx->poll_netmsg.lmsg.u.ms_resultp = io_ctx;
705 netmsg_init(&io_ctx->poll_more_netmsg, NULL, &netisr_adone_rport,
706 0, more_handler);
707 io_ctx->poll_more_netmsg.lmsg.u.ms_resultp = io_ctx;
710 * Initialize per-cpu sysctl nodes
712 if (poll_type == IFPOLL_RX)
713 poll_type_str = "rx";
714 else
715 poll_type_str = "tx";
717 sysctl_ctx_init(&io_ctx->poll_sysctl_ctx);
718 io_ctx->poll_sysctl_tree = SYSCTL_ADD_NODE(&io_ctx->poll_sysctl_ctx,
719 SYSCTL_CHILDREN(comm->sysctl_tree),
720 OID_AUTO, poll_type_str, CTLFLAG_RD, 0, "");
721 iopoll_add_sysctl(&io_ctx->poll_sysctl_ctx,
722 SYSCTL_CHILDREN(io_ctx->poll_sysctl_tree), io_ctx, poll_type);
724 return io_ctx;
728 * Hook from iopoll systimer. Tries to schedule an iopoll, but keeps
729 * track of lost ticks due to the previous handler taking too long.
730 * Normally, this should not happen, because polling handler should
731 * run for a short time. However, in some cases (e.g. when there are
732 * changes in link status etc.) the drivers take a very long time
733 * (even in the order of milliseconds) to reset and reconfigure the
734 * device, causing apparent lost polls.
736 * The first part of the code is just for debugging purposes, and tries
737 * to count how often hardclock ticks are shorter than they should,
738 * meaning either stray interrupts or delayed events.
740 * WARNING! called from fastint or IPI, the MP lock might not be held.
741 * NOTE: Caller should hold critical section.
743 static void
744 iopoll_clock(struct iopoll_ctx *io_ctx)
746 union microtime_pcpu t;
747 int delta;
749 KKASSERT(mycpuid == io_ctx->poll_cpuid);
751 if (io_ctx->poll_handlers == 0)
752 return;
754 logpoll(ioclock_start);
756 microtime_pcpu_get(&t);
757 delta = microtime_pcpu_diff(&io_ctx->prev_t, &t);
758 if (delta * io_ctx->pollhz < 500000)
759 io_ctx->short_ticks++;
760 else
761 io_ctx->prev_t = t;
763 if (io_ctx->pending_polls > 100) {
765 * Too much, assume it has stalled (not always true
766 * see comment above).
768 io_ctx->stalled++;
769 io_ctx->pending_polls = 0;
770 io_ctx->phase = 0;
773 if (io_ctx->phase <= 2) {
774 if (io_ctx->phase != 0)
775 io_ctx->suspect++;
776 io_ctx->phase = 1;
777 sched_iopoll(io_ctx);
778 io_ctx->phase = 2;
780 if (io_ctx->pending_polls++ > 0)
781 io_ctx->lost_polls++;
783 logpoll(ioclock_end);
787 * rxpoll_handler and txpoll_handler are scheduled by sched_iopoll when
788 * appropriate, typically once per polling systimer tick.
790 * Note that the message is replied immediately in order to allow a new
791 * ISR to be scheduled in the handler.
793 static void
794 rxpoll_handler(netmsg_t msg)
796 struct iopoll_ctx *io_ctx;
797 struct thread *td = curthread;
798 int i, cycles;
800 logpoll(rx_start);
802 io_ctx = msg->lmsg.u.ms_resultp;
803 KKASSERT(&td->td_msgport == netisr_cpuport(io_ctx->poll_cpuid));
805 crit_enter_quick(td);
807 /* Reply ASAP */
808 lwkt_replymsg(&msg->lmsg, 0);
810 if (io_ctx->poll_handlers == 0) {
811 crit_exit_quick(td);
812 logpoll(rx_end);
813 return;
816 io_ctx->phase = 3;
817 if (io_ctx->residual_burst == 0) {
818 /* First call in this tick */
819 microtime_pcpu_get(&io_ctx->poll_start_t);
820 io_ctx->residual_burst = io_ctx->poll_burst;
822 cycles = (io_ctx->residual_burst < io_ctx->poll_each_burst) ?
823 io_ctx->residual_burst : io_ctx->poll_each_burst;
824 io_ctx->residual_burst -= cycles;
826 for (i = 0; i < io_ctx->poll_handlers; i++) {
827 const struct iopoll_rec *rec = &io_ctx->pr[i];
828 struct ifnet *ifp = rec->ifp;
830 if (!lwkt_serialize_try(rec->serializer))
831 continue;
833 if ((ifp->if_flags & (IFF_RUNNING | IFF_NPOLLING)) ==
834 (IFF_RUNNING | IFF_NPOLLING))
835 rec->poll_func(ifp, rec->arg, cycles);
837 lwkt_serialize_exit(rec->serializer);
841 * Do a quick exit/enter to catch any higher-priority
842 * interrupt sources.
844 crit_exit_quick(td);
845 crit_enter_quick(td);
847 sched_iopollmore(io_ctx);
848 io_ctx->phase = 4;
850 crit_exit_quick(td);
852 logpoll(rx_end);
855 static void
856 txpoll_handler(netmsg_t msg)
858 struct iopoll_ctx *io_ctx;
859 struct thread *td = curthread;
860 int i;
862 logpoll(tx_start);
864 io_ctx = msg->lmsg.u.ms_resultp;
865 KKASSERT(&td->td_msgport == netisr_cpuport(io_ctx->poll_cpuid));
867 crit_enter_quick(td);
869 /* Reply ASAP */
870 lwkt_replymsg(&msg->lmsg, 0);
872 if (io_ctx->poll_handlers == 0) {
873 crit_exit_quick(td);
874 logpoll(tx_end);
875 return;
878 io_ctx->phase = 3;
880 for (i = 0; i < io_ctx->poll_handlers; i++) {
881 const struct iopoll_rec *rec = &io_ctx->pr[i];
882 struct ifnet *ifp = rec->ifp;
884 if (!lwkt_serialize_try(rec->serializer))
885 continue;
887 if ((ifp->if_flags & (IFF_RUNNING | IFF_NPOLLING)) ==
888 (IFF_RUNNING | IFF_NPOLLING))
889 rec->poll_func(ifp, rec->arg, -1);
891 lwkt_serialize_exit(rec->serializer);
895 * Do a quick exit/enter to catch any higher-priority
896 * interrupt sources.
898 crit_exit_quick(td);
899 crit_enter_quick(td);
901 sched_iopollmore(io_ctx);
902 io_ctx->phase = 4;
904 crit_exit_quick(td);
906 logpoll(tx_end);
910 * rxpollmore_handler and txpollmore_handler are called after other netisr's,
911 * possibly scheduling another rxpoll_handler or txpoll_handler call, or
912 * adapting the burst size for the next cycle.
914 * It is very bad to fetch large bursts of packets from a single card at once,
915 * because the burst could take a long time to be completely processed leading
916 * to unfairness. To reduce the problem, and also to account better for time
917 * spent in network-related processing, we split the burst in smaller chunks
918 * of fixed size, giving control to the other netisr's between chunks. This
919 * helps in improving the fairness, reducing livelock and accounting for the
920 * work performed in low level handling.
922 static void
923 rxpollmore_handler(netmsg_t msg)
925 struct thread *td = curthread;
926 struct iopoll_ctx *io_ctx;
927 union microtime_pcpu t;
928 int kern_load;
929 uint32_t pending_polls;
931 logpoll(rx_mstart);
933 io_ctx = msg->lmsg.u.ms_resultp;
934 KKASSERT(&td->td_msgport == netisr_cpuport(io_ctx->poll_cpuid));
936 crit_enter_quick(td);
938 /* Replay ASAP */
939 lwkt_replymsg(&msg->lmsg, 0);
941 if (io_ctx->poll_handlers == 0) {
942 crit_exit_quick(td);
943 logpoll(rx_mend);
944 return;
947 io_ctx->phase = 5;
948 if (io_ctx->residual_burst > 0) {
949 sched_iopoll(io_ctx);
950 crit_exit_quick(td);
951 /* Will run immediately on return, followed by netisrs */
952 logpoll(rx_mend);
953 return;
956 /* Here we can account time spent in iopoll's in this tick */
957 microtime_pcpu_get(&t);
958 kern_load = microtime_pcpu_diff(&io_ctx->poll_start_t, &t);
959 kern_load = (kern_load * io_ctx->pollhz) / 10000; /* 0..100 */
960 io_ctx->kern_frac = kern_load;
962 if (kern_load > (100 - io_ctx->user_frac)) {
963 /* Try decrease ticks */
964 if (io_ctx->poll_burst > 1)
965 io_ctx->poll_burst--;
966 } else {
967 if (io_ctx->poll_burst < io_ctx->poll_burst_max)
968 io_ctx->poll_burst++;
971 io_ctx->pending_polls--;
972 pending_polls = io_ctx->pending_polls;
974 if (pending_polls == 0) {
975 /* We are done */
976 io_ctx->phase = 0;
977 } else {
979 * Last cycle was long and caused us to miss one or more
980 * hardclock ticks. Restart processing again, but slightly
981 * reduce the burst size to prevent that this happens again.
983 io_ctx->poll_burst -= (io_ctx->poll_burst / 8);
984 if (io_ctx->poll_burst < 1)
985 io_ctx->poll_burst = 1;
986 sched_iopoll(io_ctx);
987 io_ctx->phase = 6;
990 crit_exit_quick(td);
992 logpoll(rx_mend);
995 static void
996 txpollmore_handler(netmsg_t msg)
998 struct thread *td = curthread;
999 struct iopoll_ctx *io_ctx;
1000 uint32_t pending_polls;
1002 logpoll(tx_mstart);
1004 io_ctx = msg->lmsg.u.ms_resultp;
1005 KKASSERT(&td->td_msgport == netisr_cpuport(io_ctx->poll_cpuid));
1007 crit_enter_quick(td);
1009 /* Replay ASAP */
1010 lwkt_replymsg(&msg->lmsg, 0);
1012 if (io_ctx->poll_handlers == 0) {
1013 crit_exit_quick(td);
1014 logpoll(tx_mend);
1015 return;
1018 io_ctx->phase = 5;
1020 io_ctx->pending_polls--;
1021 pending_polls = io_ctx->pending_polls;
1023 if (pending_polls == 0) {
1024 /* We are done */
1025 io_ctx->phase = 0;
1026 } else {
1028 * Last cycle was long and caused us to miss one or more
1029 * hardclock ticks. Restart processing again.
1031 sched_iopoll(io_ctx);
1032 io_ctx->phase = 6;
1035 crit_exit_quick(td);
1037 logpoll(tx_mend);
1040 static void
1041 iopoll_add_sysctl(struct sysctl_ctx_list *ctx, struct sysctl_oid_list *parent,
1042 struct iopoll_ctx *io_ctx, int poll_type)
1044 if (poll_type == IFPOLL_RX) {
1045 SYSCTL_ADD_PROC(ctx, parent, OID_AUTO, "burst_max",
1046 CTLTYPE_UINT | CTLFLAG_RW, io_ctx, 0, sysctl_burstmax,
1047 "IU", "Max Polling burst size");
1049 SYSCTL_ADD_PROC(ctx, parent, OID_AUTO, "each_burst",
1050 CTLTYPE_UINT | CTLFLAG_RW, io_ctx, 0, sysctl_eachburst,
1051 "IU", "Max size of each burst");
1053 SYSCTL_ADD_UINT(ctx, parent, OID_AUTO, "burst", CTLFLAG_RD,
1054 &io_ctx->poll_burst, 0, "Current polling burst size");
1056 SYSCTL_ADD_UINT(ctx, parent, OID_AUTO, "user_frac", CTLFLAG_RW,
1057 &io_ctx->user_frac, 0, "Desired user fraction of cpu time");
1059 SYSCTL_ADD_UINT(ctx, parent, OID_AUTO, "kern_frac", CTLFLAG_RD,
1060 &io_ctx->kern_frac, 0, "Kernel fraction of cpu time");
1062 SYSCTL_ADD_INT(ctx, parent, OID_AUTO, "residual_burst", CTLFLAG_RD,
1063 &io_ctx->residual_burst, 0,
1064 "# of residual cycles in burst");
1067 SYSCTL_ADD_UINT(ctx, parent, OID_AUTO, "phase", CTLFLAG_RD,
1068 &io_ctx->phase, 0, "Polling phase");
1070 SYSCTL_ADD_ULONG(ctx, parent, OID_AUTO, "suspect", CTLFLAG_RW,
1071 &io_ctx->suspect, "Suspected events");
1073 SYSCTL_ADD_ULONG(ctx, parent, OID_AUTO, "stalled", CTLFLAG_RW,
1074 &io_ctx->stalled, "Potential stalls");
1076 SYSCTL_ADD_ULONG(ctx, parent, OID_AUTO, "short_ticks", CTLFLAG_RW,
1077 &io_ctx->short_ticks,
1078 "Hardclock ticks shorter than they should be");
1080 SYSCTL_ADD_ULONG(ctx, parent, OID_AUTO, "lost_polls", CTLFLAG_RW,
1081 &io_ctx->lost_polls,
1082 "How many times we would have lost a poll tick");
1084 SYSCTL_ADD_UINT(ctx, parent, OID_AUTO, "pending_polls", CTLFLAG_RD,
1085 &io_ctx->pending_polls, 0, "Do we need to poll again");
1087 SYSCTL_ADD_UINT(ctx, parent, OID_AUTO, "handlers", CTLFLAG_RD,
1088 &io_ctx->poll_handlers, 0, "Number of registered poll handlers");
1091 static void
1092 sysctl_burstmax_handler(netmsg_t nmsg)
1094 struct iopoll_sysctl_netmsg *msg = (struct iopoll_sysctl_netmsg *)nmsg;
1095 struct iopoll_ctx *io_ctx;
1097 io_ctx = msg->ctx;
1098 KKASSERT(&curthread->td_msgport == netisr_cpuport(io_ctx->poll_cpuid));
1100 io_ctx->poll_burst_max = nmsg->lmsg.u.ms_result;
1101 if (io_ctx->poll_each_burst > io_ctx->poll_burst_max)
1102 io_ctx->poll_each_burst = io_ctx->poll_burst_max;
1103 if (io_ctx->poll_burst > io_ctx->poll_burst_max)
1104 io_ctx->poll_burst = io_ctx->poll_burst_max;
1105 if (io_ctx->residual_burst > io_ctx->poll_burst_max)
1106 io_ctx->residual_burst = io_ctx->poll_burst_max;
1108 lwkt_replymsg(&nmsg->lmsg, 0);
1111 static int
1112 sysctl_burstmax(SYSCTL_HANDLER_ARGS)
1114 struct iopoll_ctx *io_ctx = arg1;
1115 struct iopoll_sysctl_netmsg msg;
1116 uint32_t burst_max;
1117 int error;
1119 burst_max = io_ctx->poll_burst_max;
1120 error = sysctl_handle_int(oidp, &burst_max, 0, req);
1121 if (error || req->newptr == NULL)
1122 return error;
1123 if (burst_max < MIN_IOPOLL_BURST_MAX)
1124 burst_max = MIN_IOPOLL_BURST_MAX;
1125 else if (burst_max > MAX_IOPOLL_BURST_MAX)
1126 burst_max = MAX_IOPOLL_BURST_MAX;
1128 netmsg_init(&msg.base, NULL, &curthread->td_msgport,
1129 0, sysctl_burstmax_handler);
1130 msg.base.lmsg.u.ms_result = burst_max;
1131 msg.ctx = io_ctx;
1133 return lwkt_domsg(netisr_cpuport(io_ctx->poll_cpuid),
1134 &msg.base.lmsg, 0);
1137 static void
1138 sysctl_eachburst_handler(netmsg_t nmsg)
1140 struct iopoll_sysctl_netmsg *msg = (struct iopoll_sysctl_netmsg *)nmsg;
1141 struct iopoll_ctx *io_ctx;
1142 uint32_t each_burst;
1144 io_ctx = msg->ctx;
1145 KKASSERT(&curthread->td_msgport == netisr_cpuport(io_ctx->poll_cpuid));
1147 each_burst = nmsg->lmsg.u.ms_result;
1148 if (each_burst > io_ctx->poll_burst_max)
1149 each_burst = io_ctx->poll_burst_max;
1150 else if (each_burst < 1)
1151 each_burst = 1;
1152 io_ctx->poll_each_burst = each_burst;
1154 lwkt_replymsg(&nmsg->lmsg, 0);
1157 static int
1158 sysctl_eachburst(SYSCTL_HANDLER_ARGS)
1160 struct iopoll_ctx *io_ctx = arg1;
1161 struct iopoll_sysctl_netmsg msg;
1162 uint32_t each_burst;
1163 int error;
1165 each_burst = io_ctx->poll_each_burst;
1166 error = sysctl_handle_int(oidp, &each_burst, 0, req);
1167 if (error || req->newptr == NULL)
1168 return error;
1170 netmsg_init(&msg.base, NULL, &curthread->td_msgport,
1171 0, sysctl_eachburst_handler);
1172 msg.base.lmsg.u.ms_result = each_burst;
1173 msg.ctx = io_ctx;
1175 return lwkt_domsg(netisr_cpuport(io_ctx->poll_cpuid),
1176 &msg.base.lmsg, 0);
1179 static int
1180 iopoll_register(struct ifnet *ifp, struct iopoll_ctx *io_ctx,
1181 const struct ifpoll_io *io_rec)
1183 int error;
1185 KKASSERT(&curthread->td_msgport == netisr_cpuport(io_ctx->poll_cpuid));
1187 if (io_rec->poll_func == NULL)
1188 return 0;
1191 * Check if there is room.
1193 if (io_ctx->poll_handlers >= IFPOLL_LIST_LEN) {
1195 * List full, cannot register more entries.
1196 * This should never happen; if it does, it is probably a
1197 * broken driver trying to register multiple times. Checking
1198 * this at runtime is expensive, and won't solve the problem
1199 * anyways, so just report a few times and then give up.
1201 static int verbose = 10; /* XXX */
1202 if (verbose > 0) {
1203 kprintf("io poll handlers list full, "
1204 "maybe a broken driver ?\n");
1205 verbose--;
1207 error = ENOENT;
1208 } else {
1209 struct iopoll_rec *rec = &io_ctx->pr[io_ctx->poll_handlers];
1211 rec->ifp = ifp;
1212 rec->serializer = io_rec->serializer;
1213 rec->arg = io_rec->arg;
1214 rec->poll_func = io_rec->poll_func;
1216 io_ctx->poll_handlers++;
1217 error = 0;
1219 return error;
1222 static int
1223 iopoll_deregister(struct ifnet *ifp, struct iopoll_ctx *io_ctx)
1225 int i, error;
1227 KKASSERT(&curthread->td_msgport == netisr_cpuport(io_ctx->poll_cpuid));
1229 for (i = 0; i < io_ctx->poll_handlers; ++i) {
1230 if (io_ctx->pr[i].ifp == ifp) /* Found it */
1231 break;
1233 if (i == io_ctx->poll_handlers) {
1234 error = ENOENT;
1235 } else {
1236 io_ctx->poll_handlers--;
1237 if (i < io_ctx->poll_handlers) {
1238 /* Last entry replaces this one. */
1239 io_ctx->pr[i] = io_ctx->pr[io_ctx->poll_handlers];
1242 if (io_ctx->poll_handlers == 0)
1243 iopoll_reset_state(io_ctx);
1244 error = 0;
1246 return error;
1249 static void
1250 poll_comm_init(int cpuid)
1252 struct poll_comm *comm;
1253 char cpuid_str[16];
1255 comm = kmalloc_cachealign(sizeof(*comm), M_DEVBUF, M_WAITOK | M_ZERO);
1257 if (ifpoll_stfrac < 1)
1258 ifpoll_stfrac = IFPOLL_STFRAC_DEFAULT;
1259 if (ifpoll_txfrac < 1)
1260 ifpoll_txfrac = IFPOLL_TXFRAC_DEFAULT;
1262 comm->poll_cpuid = cpuid;
1263 comm->pollhz = poll_comm_pollhz_div(comm, ifpoll_pollhz);
1264 comm->poll_stfrac = ifpoll_stfrac - 1;
1265 comm->poll_txfrac = ifpoll_txfrac - 1;
1267 ksnprintf(cpuid_str, sizeof(cpuid_str), "%d", cpuid);
1269 sysctl_ctx_init(&comm->sysctl_ctx);
1270 comm->sysctl_tree = SYSCTL_ADD_NODE(&comm->sysctl_ctx,
1271 SYSCTL_STATIC_CHILDREN(_net_ifpoll),
1272 OID_AUTO, cpuid_str, CTLFLAG_RD, 0, "");
1274 SYSCTL_ADD_PROC(&comm->sysctl_ctx, SYSCTL_CHILDREN(comm->sysctl_tree),
1275 OID_AUTO, "pollhz", CTLTYPE_INT | CTLFLAG_RW,
1276 comm, 0, sysctl_pollhz,
1277 "I", "Device polling frequency");
1279 if (cpuid == 0) {
1280 SYSCTL_ADD_PROC(&comm->sysctl_ctx,
1281 SYSCTL_CHILDREN(comm->sysctl_tree),
1282 OID_AUTO, "status_frac",
1283 CTLTYPE_INT | CTLFLAG_RW,
1284 comm, 0, sysctl_stfrac,
1285 "I", "# of cycles before status is polled");
1287 SYSCTL_ADD_PROC(&comm->sysctl_ctx, SYSCTL_CHILDREN(comm->sysctl_tree),
1288 OID_AUTO, "tx_frac", CTLTYPE_INT | CTLFLAG_RW,
1289 comm, 0, sysctl_txfrac,
1290 "I", "# of cycles before TX is polled");
1292 poll_common[cpuid] = comm;
1295 static void
1296 poll_comm_start(int cpuid)
1298 struct poll_comm *comm = poll_common[cpuid];
1299 systimer_func_t func;
1302 * Initialize systimer
1304 if (cpuid == 0)
1305 func = poll_comm_systimer0;
1306 else
1307 func = poll_comm_systimer;
1308 systimer_init_periodic_nq(&comm->pollclock, func, comm, 1);
1311 static void
1312 _poll_comm_systimer(struct poll_comm *comm)
1314 iopoll_clock(rxpoll_context[comm->poll_cpuid]);
1315 if (comm->txfrac_count-- == 0) {
1316 comm->txfrac_count = comm->poll_txfrac;
1317 iopoll_clock(txpoll_context[comm->poll_cpuid]);
1321 static void
1322 poll_comm_systimer0(systimer_t info, int in_ipi __unused,
1323 struct intrframe *frame __unused)
1325 struct poll_comm *comm = info->data;
1326 globaldata_t gd = mycpu;
1328 KKASSERT(comm->poll_cpuid == gd->gd_cpuid && gd->gd_cpuid == 0);
1330 crit_enter_gd(gd);
1332 if (comm->stfrac_count-- == 0) {
1333 comm->stfrac_count = comm->poll_stfrac;
1334 stpoll_clock(&stpoll_context);
1336 _poll_comm_systimer(comm);
1338 crit_exit_gd(gd);
1341 static void
1342 poll_comm_systimer(systimer_t info, int in_ipi __unused,
1343 struct intrframe *frame __unused)
1345 struct poll_comm *comm = info->data;
1346 globaldata_t gd = mycpu;
1348 KKASSERT(comm->poll_cpuid == gd->gd_cpuid && gd->gd_cpuid != 0);
1350 crit_enter_gd(gd);
1351 _poll_comm_systimer(comm);
1352 crit_exit_gd(gd);
1355 static void
1356 poll_comm_adjust_pollhz(struct poll_comm *comm)
1358 uint32_t handlers;
1359 int pollhz = 1;
1361 KKASSERT(&curthread->td_msgport == netisr_cpuport(comm->poll_cpuid));
1364 * If there is no polling handler registered, set systimer
1365 * frequency to the lowest value. Polling systimer frequency
1366 * will be adjusted to the requested value, once there are
1367 * registered handlers.
1369 handlers = rxpoll_context[mycpuid]->poll_handlers +
1370 txpoll_context[mycpuid]->poll_handlers;
1371 if (comm->poll_cpuid == 0)
1372 handlers += stpoll_context.poll_handlers;
1373 if (handlers)
1374 pollhz = comm->pollhz;
1375 systimer_adjust_periodic(&comm->pollclock, pollhz);
1378 static int
1379 sysctl_pollhz(SYSCTL_HANDLER_ARGS)
1381 struct poll_comm *comm = arg1;
1382 struct netmsg_base nmsg;
1383 int error, phz;
1385 phz = poll_comm_pollhz_conv(comm, comm->pollhz);
1386 error = sysctl_handle_int(oidp, &phz, 0, req);
1387 if (error || req->newptr == NULL)
1388 return error;
1389 if (phz <= 0)
1390 return EINVAL;
1391 else if (phz > IFPOLL_FREQ_MAX)
1392 phz = IFPOLL_FREQ_MAX;
1394 netmsg_init(&nmsg, NULL, &curthread->td_msgport,
1395 0, sysctl_pollhz_handler);
1396 nmsg.lmsg.u.ms_result = phz;
1398 return lwkt_domsg(netisr_cpuport(comm->poll_cpuid), &nmsg.lmsg, 0);
1401 static void
1402 sysctl_pollhz_handler(netmsg_t nmsg)
1404 struct poll_comm *comm = poll_common[mycpuid];
1406 KKASSERT(&curthread->td_msgport == netisr_cpuport(comm->poll_cpuid));
1408 /* Save polling frequency */
1409 comm->pollhz = poll_comm_pollhz_div(comm, nmsg->lmsg.u.ms_result);
1412 * Adjust cached pollhz
1414 rxpoll_context[mycpuid]->pollhz = comm->pollhz;
1415 txpoll_context[mycpuid]->pollhz =
1416 comm->pollhz / (comm->poll_txfrac + 1);
1419 * Adjust polling frequency
1421 poll_comm_adjust_pollhz(comm);
1423 lwkt_replymsg(&nmsg->lmsg, 0);
1426 static int
1427 sysctl_stfrac(SYSCTL_HANDLER_ARGS)
1429 struct poll_comm *comm = arg1;
1430 struct netmsg_base nmsg;
1431 int error, stfrac;
1433 KKASSERT(comm->poll_cpuid == 0);
1435 stfrac = comm->poll_stfrac + 1;
1436 error = sysctl_handle_int(oidp, &stfrac, 0, req);
1437 if (error || req->newptr == NULL)
1438 return error;
1439 if (stfrac < 1)
1440 return EINVAL;
1442 netmsg_init(&nmsg, NULL, &curthread->td_msgport,
1443 0, sysctl_stfrac_handler);
1444 nmsg.lmsg.u.ms_result = stfrac - 1;
1446 return lwkt_domsg(netisr_cpuport(comm->poll_cpuid), &nmsg.lmsg, 0);
1449 static void
1450 sysctl_stfrac_handler(netmsg_t nmsg)
1452 struct poll_comm *comm = poll_common[mycpuid];
1453 int stfrac = nmsg->lmsg.u.ms_result;
1455 KKASSERT(&curthread->td_msgport == netisr_cpuport(comm->poll_cpuid));
1457 crit_enter();
1458 comm->poll_stfrac = stfrac;
1459 if (comm->stfrac_count > comm->poll_stfrac)
1460 comm->stfrac_count = comm->poll_stfrac;
1461 crit_exit();
1463 lwkt_replymsg(&nmsg->lmsg, 0);
1466 static int
1467 sysctl_txfrac(SYSCTL_HANDLER_ARGS)
1469 struct poll_comm *comm = arg1;
1470 struct netmsg_base nmsg;
1471 int error, txfrac;
1473 txfrac = comm->poll_txfrac + 1;
1474 error = sysctl_handle_int(oidp, &txfrac, 0, req);
1475 if (error || req->newptr == NULL)
1476 return error;
1477 if (txfrac < 1)
1478 return EINVAL;
1480 netmsg_init(&nmsg, NULL, &curthread->td_msgport,
1481 0, sysctl_txfrac_handler);
1482 nmsg.lmsg.u.ms_result = txfrac - 1;
1484 return lwkt_domsg(netisr_cpuport(comm->poll_cpuid), &nmsg.lmsg, 0);
1487 static void
1488 sysctl_txfrac_handler(netmsg_t nmsg)
1490 struct poll_comm *comm = poll_common[mycpuid];
1491 int txfrac = nmsg->lmsg.u.ms_result;
1493 KKASSERT(&curthread->td_msgport == netisr_cpuport(comm->poll_cpuid));
1495 crit_enter();
1496 comm->poll_txfrac = txfrac;
1497 if (comm->txfrac_count > comm->poll_txfrac)
1498 comm->txfrac_count = comm->poll_txfrac;
1499 crit_exit();
1501 lwkt_replymsg(&nmsg->lmsg, 0);
1504 void
1505 ifpoll_compat_setup(struct ifpoll_compat *cp,
1506 struct sysctl_ctx_list *sysctl_ctx,
1507 struct sysctl_oid *sysctl_tree,
1508 int unit, struct lwkt_serialize *slz)
1510 cp->ifpc_stcount = 0;
1511 cp->ifpc_stfrac = ((poll_common[0]->poll_stfrac + 1) *
1512 howmany(IOPOLL_BURST_MAX, IOPOLL_EACH_BURST)) - 1;
1514 cp->ifpc_cpuid = unit % ncpus2;
1515 cp->ifpc_serializer = slz;
1517 if (sysctl_ctx != NULL && sysctl_tree != NULL) {
1518 SYSCTL_ADD_PROC(sysctl_ctx, SYSCTL_CHILDREN(sysctl_tree),
1519 OID_AUTO, "npoll_stfrac", CTLTYPE_INT | CTLFLAG_RW,
1520 cp, 0, sysctl_compat_npoll_stfrac, "I",
1521 "polling status frac");
1522 SYSCTL_ADD_PROC(sysctl_ctx, SYSCTL_CHILDREN(sysctl_tree),
1523 OID_AUTO, "npoll_cpuid", CTLTYPE_INT | CTLFLAG_RW,
1524 cp, 0, sysctl_compat_npoll_cpuid, "I",
1525 "polling cpuid");
1529 static int
1530 sysctl_compat_npoll_stfrac(SYSCTL_HANDLER_ARGS)
1532 struct ifpoll_compat *cp = arg1;
1533 int error = 0, stfrac;
1535 lwkt_serialize_enter(cp->ifpc_serializer);
1537 stfrac = cp->ifpc_stfrac + 1;
1538 error = sysctl_handle_int(oidp, &stfrac, 0, req);
1539 if (!error && req->newptr != NULL) {
1540 if (stfrac < 1) {
1541 error = EINVAL;
1542 } else {
1543 cp->ifpc_stfrac = stfrac - 1;
1544 if (cp->ifpc_stcount > cp->ifpc_stfrac)
1545 cp->ifpc_stcount = cp->ifpc_stfrac;
1549 lwkt_serialize_exit(cp->ifpc_serializer);
1550 return error;
1553 static int
1554 sysctl_compat_npoll_cpuid(SYSCTL_HANDLER_ARGS)
1556 struct ifpoll_compat *cp = arg1;
1557 int error = 0, cpuid;
1559 lwkt_serialize_enter(cp->ifpc_serializer);
1561 cpuid = cp->ifpc_cpuid;
1562 error = sysctl_handle_int(oidp, &cpuid, 0, req);
1563 if (!error && req->newptr != NULL) {
1564 if (cpuid < 0 || cpuid >= ncpus2)
1565 error = EINVAL;
1566 else
1567 cp->ifpc_cpuid = cpuid;
1570 lwkt_serialize_exit(cp->ifpc_serializer);
1571 return error;