2 * Copyright (c) 2001-2002 Luigi Rizzo
4 * Supported by: the Xorp Project (www.xorp.org)
6 * Redistribution and use in source and binary forms, with or without
7 * modification, are permitted provided that the following conditions
9 * 1. Redistributions of source code must retain the above copyright
10 * notice, this list of conditions and the following disclaimer.
11 * 2. Redistributions in binary form must reproduce the above copyright
12 * notice, this list of conditions and the following disclaimer in the
13 * documentation and/or other materials provided with the distribution.
15 * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND
16 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
17 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
18 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE
19 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
20 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
27 * $FreeBSD: src/sys/kern/kern_poll.c,v 1.2.2.4 2002/06/27 23:26:33 luigi Exp $
30 #include "opt_ifpoll.h"
32 #include <sys/param.h>
33 #include <sys/kernel.h>
35 #include <sys/malloc.h>
36 #include <sys/serialize.h>
37 #include <sys/socket.h>
38 #include <sys/sysctl.h>
39 #include <sys/microtime_pcpu.h>
41 #include <sys/thread2.h>
42 #include <sys/msgport2.h>
45 #include <net/if_poll.h>
46 #include <net/netmsg2.h>
47 #include <net/netisr2.h>
50 * Polling support for network device drivers.
52 * Drivers which support this feature try to register one status polling
53 * handler and several TX/RX polling handlers with the polling code.
54 * If interface's if_npoll is called with non-NULL second argument, then
55 * a register operation is requested, else a deregister operation is
56 * requested. If the requested operation is "register", driver should
57 * setup the ifpoll_info passed in accoding its own needs:
58 * ifpoll_info.ifpi_status.status_func == NULL
59 * No status polling handler will be installed on CPU(0)
60 * ifpoll_info.ifpi_rx[n].poll_func == NULL
61 * No RX polling handler will be installed on CPU(n)
62 * ifpoll_info.ifpi_tx[n].poll_func == NULL
63 * No TX polling handler will be installed on CPU(n)
65 * RX is polled at the specified polling frequency (net.ifpoll.X.pollhz).
66 * TX and status polling could be done at lower frequency than RX frequency
67 * (net.ifpoll.0.status_frac and net.ifpoll.X.tx_frac). To avoid systimer
68 * staggering at high frequency, RX systimer gives TX and status polling a
71 * All of the registered polling handlers are called only if the interface
72 * is marked as 'IFF_RUNNING and IFF_NPOLLING'. However, the interface's
73 * register and deregister function (ifnet.if_npoll) will be called even
74 * if interface is not marked with 'IFF_RUNNING'.
76 * If registration is successful, the driver must disable interrupts,
77 * and further I/O is performed through the TX/RX polling handler, which
78 * are invoked (at least once per clock tick) with 3 arguments: the "arg"
79 * passed at register time, a struct ifnet pointer, and a "count" limit.
80 * The registered serializer will be held before calling the related
83 * The count limit specifies how much work the handler can do during the
84 * call -- typically this is the number of packets to be received, or
85 * transmitted, etc. (drivers are free to interpret this number, as long
86 * as the max time spent in the function grows roughly linearly with the
89 * A second variable controls the sharing of CPU between polling/kernel
90 * network processing, and other activities (typically userlevel tasks):
91 * net.ifpoll.X.{rx,tx}.user_frac (between 0 and 100, default 50) sets the
92 * share of CPU allocated to user tasks. CPU is allocated proportionally
93 * to the shares, by dynamically adjusting the "count" (poll_burst).
95 * Other parameters can should be left to their default values.
96 * The following constraints hold
98 * 1 <= poll_burst <= poll_burst_max
99 * 1 <= poll_each_burst <= poll_burst_max
100 * MIN_POLL_BURST_MAX <= poll_burst_max <= MAX_POLL_BURST_MAX
103 #define IFPOLL_LIST_LEN 128
104 #define IFPOLL_FREQ_MAX 30000
106 #define MIN_IOPOLL_BURST_MAX 10
107 #define MAX_IOPOLL_BURST_MAX 5000
108 #define IOPOLL_BURST_MAX 250 /* good for 1000Mbit net and HZ=6000 */
110 #define IOPOLL_EACH_BURST 50
111 #define IOPOLL_USER_FRAC 50
113 #define IFPOLL_FREQ_DEFAULT 6000
115 #define IFPOLL_TXFRAC_DEFAULT 1 /* 1/1 of the pollhz */
116 #define IFPOLL_STFRAC_DEFAULT 120 /* 1/120 of the pollhz */
118 #define IFPOLL_RX 0x1
119 #define IFPOLL_TX 0x2
122 struct lwkt_serialize
*serializer
;
125 ifpoll_iofn_t poll_func
;
129 union microtime_pcpu prev_t
;
130 u_long short_ticks
; /* statistics */
131 u_long lost_polls
; /* statistics */
132 u_long suspect
; /* statistics */
133 u_long stalled
; /* statistics */
134 uint32_t pending_polls
; /* state */
136 struct netmsg_base poll_netmsg
;
137 struct netmsg_base poll_more_netmsg
;
141 uint32_t phase
; /* state */
142 int residual_burst
; /* state */
143 uint32_t poll_each_burst
; /* tunable */
144 union microtime_pcpu poll_start_t
; /* state */
146 uint32_t poll_burst
; /* state */
147 uint32_t poll_burst_max
; /* tunable */
148 uint32_t user_frac
; /* tunable */
149 uint32_t kern_frac
; /* state */
151 uint32_t poll_handlers
; /* next free entry in pr[]. */
152 struct iopoll_rec pr
[IFPOLL_LIST_LEN
];
154 struct sysctl_ctx_list poll_sysctl_ctx
;
155 struct sysctl_oid
*poll_sysctl_tree
;
159 struct systimer pollclock
;
162 int stfrac_count
; /* state */
163 int poll_stfrac
; /* tunable */
165 int txfrac_count
; /* state */
166 int poll_txfrac
; /* tunable */
168 int pollhz
; /* tunable */
170 struct sysctl_ctx_list sysctl_ctx
;
171 struct sysctl_oid
*sysctl_tree
;
175 struct lwkt_serialize
*serializer
;
177 ifpoll_stfn_t status_func
;
181 struct netmsg_base poll_netmsg
;
183 uint32_t poll_handlers
; /* next free entry in pr[]. */
184 struct stpoll_rec pr
[IFPOLL_LIST_LEN
];
186 struct sysctl_ctx_list poll_sysctl_ctx
;
187 struct sysctl_oid
*poll_sysctl_tree
;
190 struct iopoll_sysctl_netmsg
{
191 struct netmsg_base base
;
192 struct iopoll_ctx
*ctx
;
195 static void ifpoll_init_pcpu(int);
196 static void ifpoll_register_handler(netmsg_t
);
197 static void ifpoll_deregister_handler(netmsg_t
);
202 static void stpoll_init(void);
203 static void stpoll_handler(netmsg_t
);
204 static void stpoll_clock(struct stpoll_ctx
*);
205 static int stpoll_register(struct ifnet
*, const struct ifpoll_status
*);
206 static int stpoll_deregister(struct ifnet
*);
211 static struct iopoll_ctx
*iopoll_ctx_create(int, int);
212 static void iopoll_init(int);
213 static void rxpoll_handler(netmsg_t
);
214 static void txpoll_handler(netmsg_t
);
215 static void rxpollmore_handler(netmsg_t
);
216 static void txpollmore_handler(netmsg_t
);
217 static void iopoll_clock(struct iopoll_ctx
*);
218 static int iopoll_register(struct ifnet
*, struct iopoll_ctx
*,
219 const struct ifpoll_io
*);
220 static int iopoll_deregister(struct ifnet
*, struct iopoll_ctx
*);
222 static void iopoll_add_sysctl(struct sysctl_ctx_list
*,
223 struct sysctl_oid_list
*, struct iopoll_ctx
*, int);
224 static void sysctl_burstmax_handler(netmsg_t
);
225 static int sysctl_burstmax(SYSCTL_HANDLER_ARGS
);
226 static void sysctl_eachburst_handler(netmsg_t
);
227 static int sysctl_eachburst(SYSCTL_HANDLER_ARGS
);
232 static void poll_comm_init(int);
233 static void poll_comm_start(int);
234 static void poll_comm_adjust_pollhz(struct poll_comm
*);
235 static void poll_comm_systimer0(systimer_t
, int, struct intrframe
*);
236 static void poll_comm_systimer(systimer_t
, int, struct intrframe
*);
237 static void sysctl_pollhz_handler(netmsg_t
);
238 static void sysctl_stfrac_handler(netmsg_t
);
239 static void sysctl_txfrac_handler(netmsg_t
);
240 static int sysctl_pollhz(SYSCTL_HANDLER_ARGS
);
241 static int sysctl_stfrac(SYSCTL_HANDLER_ARGS
);
242 static int sysctl_txfrac(SYSCTL_HANDLER_ARGS
);
243 static int sysctl_compat_npoll_stfrac(SYSCTL_HANDLER_ARGS
);
244 static int sysctl_compat_npoll_cpuid(SYSCTL_HANDLER_ARGS
);
246 static struct stpoll_ctx stpoll_context
;
247 static struct poll_comm
*poll_common
[MAXCPU
];
248 static struct iopoll_ctx
*rxpoll_context
[MAXCPU
];
249 static struct iopoll_ctx
*txpoll_context
[MAXCPU
];
251 SYSCTL_NODE(_net
, OID_AUTO
, ifpoll
, CTLFLAG_RW
, 0,
252 "Network device polling parameters");
254 static int iopoll_burst_max
= IOPOLL_BURST_MAX
;
255 static int iopoll_each_burst
= IOPOLL_EACH_BURST
;
256 static int iopoll_user_frac
= IOPOLL_USER_FRAC
;
258 static int ifpoll_pollhz
= IFPOLL_FREQ_DEFAULT
;
259 static int ifpoll_stfrac
= IFPOLL_STFRAC_DEFAULT
;
260 static int ifpoll_txfrac
= IFPOLL_TXFRAC_DEFAULT
;
262 TUNABLE_INT("net.ifpoll.burst_max", &iopoll_burst_max
);
263 TUNABLE_INT("net.ifpoll.each_burst", &iopoll_each_burst
);
264 TUNABLE_INT("net.ifpoll.user_frac", &iopoll_user_frac
);
265 TUNABLE_INT("net.ifpoll.pollhz", &ifpoll_pollhz
);
266 TUNABLE_INT("net.ifpoll.status_frac", &ifpoll_stfrac
);
267 TUNABLE_INT("net.ifpoll.tx_frac", &ifpoll_txfrac
);
269 #if !defined(KTR_IF_POLL)
270 #define KTR_IF_POLL KTR_ALL
272 KTR_INFO_MASTER(if_poll
);
273 KTR_INFO(KTR_IF_POLL
, if_poll
, rx_start
, 0, "rx start");
274 KTR_INFO(KTR_IF_POLL
, if_poll
, rx_end
, 1, "rx end");
275 KTR_INFO(KTR_IF_POLL
, if_poll
, tx_start
, 2, "tx start");
276 KTR_INFO(KTR_IF_POLL
, if_poll
, tx_end
, 3, "tx end");
277 KTR_INFO(KTR_IF_POLL
, if_poll
, rx_mstart
, 4, "rx more start");
278 KTR_INFO(KTR_IF_POLL
, if_poll
, rx_mend
, 5, "rx more end");
279 KTR_INFO(KTR_IF_POLL
, if_poll
, tx_mstart
, 6, "tx more start");
280 KTR_INFO(KTR_IF_POLL
, if_poll
, tx_mend
, 7, "tx more end");
281 KTR_INFO(KTR_IF_POLL
, if_poll
, ioclock_start
, 8, "ioclock start");
282 KTR_INFO(KTR_IF_POLL
, if_poll
, ioclock_end
, 9, "ioclock end");
283 #define logpoll(name) KTR_LOG(if_poll_ ## name)
285 #define IFPOLL_FREQ_ADJ(comm) (((comm)->poll_cpuid * 3) % 50)
288 poll_comm_pollhz_div(const struct poll_comm
*comm
, int pollhz
)
290 return pollhz
+ IFPOLL_FREQ_ADJ(comm
);
294 poll_comm_pollhz_conv(const struct poll_comm
*comm
, int pollhz
)
296 return pollhz
- IFPOLL_FREQ_ADJ(comm
);
300 ifpoll_sendmsg_oncpu(netmsg_t msg
)
302 if (msg
->lmsg
.ms_flags
& MSGF_DONE
)
303 netisr_sendmsg_oncpu(&msg
->base
);
307 sched_stpoll(struct stpoll_ctx
*st_ctx
)
309 ifpoll_sendmsg_oncpu((netmsg_t
)&st_ctx
->poll_netmsg
);
313 sched_iopoll(struct iopoll_ctx
*io_ctx
)
315 ifpoll_sendmsg_oncpu((netmsg_t
)&io_ctx
->poll_netmsg
);
319 sched_iopollmore(struct iopoll_ctx
*io_ctx
)
321 ifpoll_sendmsg_oncpu((netmsg_t
)&io_ctx
->poll_more_netmsg
);
325 * Initialize per-cpu polling(4) context.
328 ifpoll_init_pcpu(int cpuid
)
331 poll_comm_init(cpuid
);
337 poll_comm_start(cpuid
);
341 ifpoll_init_handler(netmsg_t msg
)
345 ifpoll_init_pcpu(cpu
);
346 netisr_forwardmsg(&msg
->base
, cpu
+ 1);
350 ifpoll_sysinit(void *dummy __unused
)
352 struct netmsg_base msg
;
354 netmsg_init(&msg
, NULL
, &curthread
->td_msgport
, 0, ifpoll_init_handler
);
355 netisr_domsg(&msg
, 0);
357 SYSINIT(ifpoll
, SI_SUB_PRE_DRIVERS
, SI_ORDER_ANY
, ifpoll_sysinit
, NULL
);
360 ifpoll_register(struct ifnet
*ifp
)
362 struct ifpoll_info
*info
;
363 struct netmsg_base nmsg
;
366 if (ifp
->if_npoll
== NULL
) {
367 /* Device does not support polling */
371 info
= kmalloc(sizeof(*info
), M_TEMP
, M_WAITOK
| M_ZERO
);
374 * Attempt to register. Interlock with IFF_NPOLLING.
377 ifnet_serialize_all(ifp
);
379 if (ifp
->if_flags
& IFF_NPOLLING
) {
380 /* Already polling */
381 ifnet_deserialize_all(ifp
);
386 info
->ifpi_ifp
= ifp
;
388 ifp
->if_flags
|= IFF_NPOLLING
;
389 ifp
->if_npoll(ifp
, info
);
391 ifnet_deserialize_all(ifp
);
393 netmsg_init(&nmsg
, NULL
, &curthread
->td_msgport
,
394 0, ifpoll_register_handler
);
395 nmsg
.lmsg
.u
.ms_resultp
= info
;
397 error
= netisr_domsg(&nmsg
, 0);
399 if (!ifpoll_deregister(ifp
)) {
400 if_printf(ifp
, "ifpoll_register: "
401 "ifpoll_deregister failed!\n");
410 ifpoll_deregister(struct ifnet
*ifp
)
412 struct netmsg_base nmsg
;
415 if (ifp
->if_npoll
== NULL
)
418 ifnet_serialize_all(ifp
);
420 if ((ifp
->if_flags
& IFF_NPOLLING
) == 0) {
421 ifnet_deserialize_all(ifp
);
424 ifp
->if_flags
&= ~IFF_NPOLLING
;
426 ifnet_deserialize_all(ifp
);
428 netmsg_init(&nmsg
, NULL
, &curthread
->td_msgport
,
429 0, ifpoll_deregister_handler
);
430 nmsg
.lmsg
.u
.ms_resultp
= ifp
;
432 error
= netisr_domsg(&nmsg
, 0);
434 ifnet_serialize_all(ifp
);
435 ifp
->if_npoll(ifp
, NULL
);
436 ifnet_deserialize_all(ifp
);
442 ifpoll_register_handler(netmsg_t nmsg
)
444 const struct ifpoll_info
*info
= nmsg
->lmsg
.u
.ms_resultp
;
448 KKASSERT(cpuid
< netisr_ncpus
);
449 KKASSERT(&curthread
->td_msgport
== netisr_cpuport(cpuid
));
452 error
= stpoll_register(info
->ifpi_ifp
, &info
->ifpi_status
);
457 error
= iopoll_register(info
->ifpi_ifp
, rxpoll_context
[cpuid
],
458 &info
->ifpi_rx
[cpuid
]);
462 error
= iopoll_register(info
->ifpi_ifp
, txpoll_context
[cpuid
],
463 &info
->ifpi_tx
[cpuid
]);
467 /* Adjust polling frequency, after all registration is done */
468 poll_comm_adjust_pollhz(poll_common
[cpuid
]);
470 netisr_forwardmsg(&nmsg
->base
, cpuid
+ 1);
473 netisr_replymsg(&nmsg
->base
, error
);
477 ifpoll_deregister_handler(netmsg_t nmsg
)
479 struct ifnet
*ifp
= nmsg
->lmsg
.u
.ms_resultp
;
482 KKASSERT(cpuid
< netisr_ncpus
);
483 KKASSERT(&curthread
->td_msgport
== netisr_cpuport(cpuid
));
487 stpoll_deregister(ifp
);
488 iopoll_deregister(ifp
, rxpoll_context
[cpuid
]);
489 iopoll_deregister(ifp
, txpoll_context
[cpuid
]);
491 /* Adjust polling frequency, after all deregistration is done */
492 poll_comm_adjust_pollhz(poll_common
[cpuid
]);
494 netisr_forwardmsg(&nmsg
->base
, cpuid
+ 1);
500 struct stpoll_ctx
*st_ctx
= &stpoll_context
;
501 const struct poll_comm
*comm
= poll_common
[0];
503 sysctl_ctx_init(&st_ctx
->poll_sysctl_ctx
);
504 st_ctx
->poll_sysctl_tree
= SYSCTL_ADD_NODE(&st_ctx
->poll_sysctl_ctx
,
505 SYSCTL_CHILDREN(comm
->sysctl_tree
),
506 OID_AUTO
, "status", CTLFLAG_RD
, 0, "");
508 SYSCTL_ADD_UINT(&st_ctx
->poll_sysctl_ctx
,
509 SYSCTL_CHILDREN(st_ctx
->poll_sysctl_tree
),
510 OID_AUTO
, "handlers", CTLFLAG_RD
,
511 &st_ctx
->poll_handlers
, 0,
512 "Number of registered status poll handlers");
514 netmsg_init(&st_ctx
->poll_netmsg
, NULL
, &netisr_adone_rport
,
519 * stpoll_handler is scheduled by sched_stpoll when appropriate, typically
520 * once per polling systimer tick.
523 stpoll_handler(netmsg_t msg
)
525 struct stpoll_ctx
*st_ctx
= &stpoll_context
;
526 struct thread
*td
= curthread
;
531 crit_enter_quick(td
);
534 netisr_replymsg(&msg
->base
, 0);
536 if (st_ctx
->poll_handlers
== 0) {
541 for (i
= 0; i
< st_ctx
->poll_handlers
; ++i
) {
542 const struct stpoll_rec
*rec
= &st_ctx
->pr
[i
];
543 struct ifnet
*ifp
= rec
->ifp
;
545 if (!lwkt_serialize_try(rec
->serializer
))
548 if ((ifp
->if_flags
& (IFF_RUNNING
| IFF_NPOLLING
)) ==
549 (IFF_RUNNING
| IFF_NPOLLING
))
550 rec
->status_func(ifp
);
552 lwkt_serialize_exit(rec
->serializer
);
559 * Hook from status poll systimer. Tries to schedule an status poll.
560 * NOTE: Caller should hold critical section.
563 stpoll_clock(struct stpoll_ctx
*st_ctx
)
565 KKASSERT(mycpuid
== 0);
567 if (st_ctx
->poll_handlers
== 0)
569 sched_stpoll(st_ctx
);
573 stpoll_register(struct ifnet
*ifp
, const struct ifpoll_status
*st_rec
)
575 struct stpoll_ctx
*st_ctx
= &stpoll_context
;
580 if (st_rec
->status_func
== NULL
)
584 * Check if there is room.
586 if (st_ctx
->poll_handlers
>= IFPOLL_LIST_LEN
) {
588 * List full, cannot register more entries.
589 * This should never happen; if it does, it is probably a
590 * broken driver trying to register multiple times. Checking
591 * this at runtime is expensive, and won't solve the problem
592 * anyways, so just report a few times and then give up.
594 static int verbose
= 10; /* XXX */
597 kprintf("status poll handlers list full, "
598 "maybe a broken driver ?\n");
603 struct stpoll_rec
*rec
= &st_ctx
->pr
[st_ctx
->poll_handlers
];
606 rec
->serializer
= st_rec
->serializer
;
607 rec
->status_func
= st_rec
->status_func
;
609 st_ctx
->poll_handlers
++;
616 stpoll_deregister(struct ifnet
*ifp
)
618 struct stpoll_ctx
*st_ctx
= &stpoll_context
;
623 for (i
= 0; i
< st_ctx
->poll_handlers
; ++i
) {
624 if (st_ctx
->pr
[i
].ifp
== ifp
) /* Found it */
627 if (i
== st_ctx
->poll_handlers
) {
630 st_ctx
->poll_handlers
--;
631 if (i
< st_ctx
->poll_handlers
) {
632 /* Last entry replaces this one. */
633 st_ctx
->pr
[i
] = st_ctx
->pr
[st_ctx
->poll_handlers
];
641 iopoll_reset_state(struct iopoll_ctx
*io_ctx
)
644 io_ctx
->poll_burst
= io_ctx
->poll_each_burst
;
645 io_ctx
->pending_polls
= 0;
646 io_ctx
->residual_burst
= 0;
648 io_ctx
->kern_frac
= 0;
649 bzero(&io_ctx
->poll_start_t
, sizeof(io_ctx
->poll_start_t
));
650 bzero(&io_ctx
->prev_t
, sizeof(io_ctx
->prev_t
));
655 iopoll_init(int cpuid
)
657 KKASSERT(cpuid
< netisr_ncpus
);
659 rxpoll_context
[cpuid
] = iopoll_ctx_create(cpuid
, IFPOLL_RX
);
660 txpoll_context
[cpuid
] = iopoll_ctx_create(cpuid
, IFPOLL_TX
);
663 static struct iopoll_ctx
*
664 iopoll_ctx_create(int cpuid
, int poll_type
)
666 struct poll_comm
*comm
;
667 struct iopoll_ctx
*io_ctx
;
668 const char *poll_type_str
;
669 netisr_fn_t handler
, more_handler
;
671 KKASSERT(poll_type
== IFPOLL_RX
|| poll_type
== IFPOLL_TX
);
674 * Make sure that tunables are in sane state
676 if (iopoll_burst_max
< MIN_IOPOLL_BURST_MAX
)
677 iopoll_burst_max
= MIN_IOPOLL_BURST_MAX
;
678 else if (iopoll_burst_max
> MAX_IOPOLL_BURST_MAX
)
679 iopoll_burst_max
= MAX_IOPOLL_BURST_MAX
;
681 if (iopoll_each_burst
> iopoll_burst_max
)
682 iopoll_each_burst
= iopoll_burst_max
;
684 comm
= poll_common
[cpuid
];
687 * Create the per-cpu polling context
689 io_ctx
= kmalloc_cachealign(sizeof(*io_ctx
), M_DEVBUF
,
692 io_ctx
->poll_each_burst
= iopoll_each_burst
;
693 io_ctx
->poll_burst_max
= iopoll_burst_max
;
694 io_ctx
->user_frac
= iopoll_user_frac
;
695 if (poll_type
== IFPOLL_RX
)
696 io_ctx
->pollhz
= comm
->pollhz
;
698 io_ctx
->pollhz
= comm
->pollhz
/ (comm
->poll_txfrac
+ 1);
699 io_ctx
->poll_cpuid
= cpuid
;
700 iopoll_reset_state(io_ctx
);
702 if (poll_type
== IFPOLL_RX
) {
703 handler
= rxpoll_handler
;
704 more_handler
= rxpollmore_handler
;
706 handler
= txpoll_handler
;
707 more_handler
= txpollmore_handler
;
710 netmsg_init(&io_ctx
->poll_netmsg
, NULL
, &netisr_adone_rport
,
712 io_ctx
->poll_netmsg
.lmsg
.u
.ms_resultp
= io_ctx
;
714 netmsg_init(&io_ctx
->poll_more_netmsg
, NULL
, &netisr_adone_rport
,
716 io_ctx
->poll_more_netmsg
.lmsg
.u
.ms_resultp
= io_ctx
;
719 * Initialize per-cpu sysctl nodes
721 if (poll_type
== IFPOLL_RX
)
722 poll_type_str
= "rx";
724 poll_type_str
= "tx";
726 sysctl_ctx_init(&io_ctx
->poll_sysctl_ctx
);
727 io_ctx
->poll_sysctl_tree
= SYSCTL_ADD_NODE(&io_ctx
->poll_sysctl_ctx
,
728 SYSCTL_CHILDREN(comm
->sysctl_tree
),
729 OID_AUTO
, poll_type_str
, CTLFLAG_RD
, 0, "");
730 iopoll_add_sysctl(&io_ctx
->poll_sysctl_ctx
,
731 SYSCTL_CHILDREN(io_ctx
->poll_sysctl_tree
), io_ctx
, poll_type
);
737 * Hook from iopoll systimer. Tries to schedule an iopoll, but keeps
738 * track of lost ticks due to the previous handler taking too long.
739 * Normally, this should not happen, because polling handler should
740 * run for a short time. However, in some cases (e.g. when there are
741 * changes in link status etc.) the drivers take a very long time
742 * (even in the order of milliseconds) to reset and reconfigure the
743 * device, causing apparent lost polls.
745 * The first part of the code is just for debugging purposes, and tries
746 * to count how often hardclock ticks are shorter than they should,
747 * meaning either stray interrupts or delayed events.
749 * WARNING! called from fastint or IPI, the MP lock might not be held.
750 * NOTE: Caller should hold critical section.
753 iopoll_clock(struct iopoll_ctx
*io_ctx
)
755 union microtime_pcpu t
;
758 KKASSERT(mycpuid
== io_ctx
->poll_cpuid
);
760 if (io_ctx
->poll_handlers
== 0)
763 logpoll(ioclock_start
);
765 microtime_pcpu_get(&t
);
766 delta
= microtime_pcpu_diff(&io_ctx
->prev_t
, &t
);
767 if (delta
* io_ctx
->pollhz
< 500000)
768 io_ctx
->short_ticks
++;
772 if (io_ctx
->pending_polls
> 100) {
774 * Too much, assume it has stalled (not always true
775 * see comment above).
778 io_ctx
->pending_polls
= 0;
782 if (io_ctx
->phase
<= 2) {
783 if (io_ctx
->phase
!= 0)
786 sched_iopoll(io_ctx
);
789 if (io_ctx
->pending_polls
++ > 0)
790 io_ctx
->lost_polls
++;
792 logpoll(ioclock_end
);
796 * rxpoll_handler and txpoll_handler are scheduled by sched_iopoll when
797 * appropriate, typically once per polling systimer tick.
799 * Note that the message is replied immediately in order to allow a new
800 * ISR to be scheduled in the handler.
803 rxpoll_handler(netmsg_t msg
)
805 struct iopoll_ctx
*io_ctx
;
806 struct thread
*td
= curthread
;
811 io_ctx
= msg
->lmsg
.u
.ms_resultp
;
812 KKASSERT(&td
->td_msgport
== netisr_cpuport(io_ctx
->poll_cpuid
));
814 crit_enter_quick(td
);
817 netisr_replymsg(&msg
->base
, 0);
819 if (io_ctx
->poll_handlers
== 0) {
826 if (io_ctx
->residual_burst
== 0) {
827 /* First call in this tick */
828 microtime_pcpu_get(&io_ctx
->poll_start_t
);
829 io_ctx
->residual_burst
= io_ctx
->poll_burst
;
831 cycles
= (io_ctx
->residual_burst
< io_ctx
->poll_each_burst
) ?
832 io_ctx
->residual_burst
: io_ctx
->poll_each_burst
;
833 io_ctx
->residual_burst
-= cycles
;
835 for (i
= 0; i
< io_ctx
->poll_handlers
; i
++) {
836 const struct iopoll_rec
*rec
= &io_ctx
->pr
[i
];
837 struct ifnet
*ifp
= rec
->ifp
;
839 if (!lwkt_serialize_try(rec
->serializer
))
842 if ((ifp
->if_flags
& (IFF_RUNNING
| IFF_NPOLLING
)) ==
843 (IFF_RUNNING
| IFF_NPOLLING
))
844 rec
->poll_func(ifp
, rec
->arg
, cycles
);
846 lwkt_serialize_exit(rec
->serializer
);
850 * Do a quick exit/enter to catch any higher-priority
854 crit_enter_quick(td
);
856 sched_iopollmore(io_ctx
);
865 txpoll_handler(netmsg_t msg
)
867 struct iopoll_ctx
*io_ctx
;
868 struct thread
*td
= curthread
;
873 io_ctx
= msg
->lmsg
.u
.ms_resultp
;
874 KKASSERT(&td
->td_msgport
== netisr_cpuport(io_ctx
->poll_cpuid
));
876 crit_enter_quick(td
);
879 netisr_replymsg(&msg
->base
, 0);
881 if (io_ctx
->poll_handlers
== 0) {
889 for (i
= 0; i
< io_ctx
->poll_handlers
; i
++) {
890 const struct iopoll_rec
*rec
= &io_ctx
->pr
[i
];
891 struct ifnet
*ifp
= rec
->ifp
;
893 if (!lwkt_serialize_try(rec
->serializer
))
896 if ((ifp
->if_flags
& (IFF_RUNNING
| IFF_NPOLLING
)) ==
897 (IFF_RUNNING
| IFF_NPOLLING
))
898 rec
->poll_func(ifp
, rec
->arg
, -1);
900 lwkt_serialize_exit(rec
->serializer
);
904 * Do a quick exit/enter to catch any higher-priority
908 crit_enter_quick(td
);
910 sched_iopollmore(io_ctx
);
919 * rxpollmore_handler and txpollmore_handler are called after other netisr's,
920 * possibly scheduling another rxpoll_handler or txpoll_handler call, or
921 * adapting the burst size for the next cycle.
923 * It is very bad to fetch large bursts of packets from a single card at once,
924 * because the burst could take a long time to be completely processed leading
925 * to unfairness. To reduce the problem, and also to account better for time
926 * spent in network-related processing, we split the burst in smaller chunks
927 * of fixed size, giving control to the other netisr's between chunks. This
928 * helps in improving the fairness, reducing livelock and accounting for the
929 * work performed in low level handling.
932 rxpollmore_handler(netmsg_t msg
)
934 struct thread
*td
= curthread
;
935 struct iopoll_ctx
*io_ctx
;
936 union microtime_pcpu t
;
938 uint32_t pending_polls
;
942 io_ctx
= msg
->lmsg
.u
.ms_resultp
;
943 KKASSERT(&td
->td_msgport
== netisr_cpuport(io_ctx
->poll_cpuid
));
945 crit_enter_quick(td
);
948 netisr_replymsg(&msg
->base
, 0);
950 if (io_ctx
->poll_handlers
== 0) {
957 if (io_ctx
->residual_burst
> 0) {
958 sched_iopoll(io_ctx
);
960 /* Will run immediately on return, followed by netisrs */
965 /* Here we can account time spent in iopoll's in this tick */
966 microtime_pcpu_get(&t
);
967 kern_load
= microtime_pcpu_diff(&io_ctx
->poll_start_t
, &t
);
968 kern_load
= (kern_load
* io_ctx
->pollhz
) / 10000; /* 0..100 */
969 io_ctx
->kern_frac
= kern_load
;
971 if (kern_load
> (100 - io_ctx
->user_frac
)) {
972 /* Try decrease ticks */
973 if (io_ctx
->poll_burst
> 1)
974 io_ctx
->poll_burst
--;
976 if (io_ctx
->poll_burst
< io_ctx
->poll_burst_max
)
977 io_ctx
->poll_burst
++;
980 io_ctx
->pending_polls
--;
981 pending_polls
= io_ctx
->pending_polls
;
983 if (pending_polls
== 0) {
988 * Last cycle was long and caused us to miss one or more
989 * hardclock ticks. Restart processing again, but slightly
990 * reduce the burst size to prevent that this happens again.
992 io_ctx
->poll_burst
-= (io_ctx
->poll_burst
/ 8);
993 if (io_ctx
->poll_burst
< 1)
994 io_ctx
->poll_burst
= 1;
995 sched_iopoll(io_ctx
);
1005 txpollmore_handler(netmsg_t msg
)
1007 struct thread
*td
= curthread
;
1008 struct iopoll_ctx
*io_ctx
;
1009 uint32_t pending_polls
;
1013 io_ctx
= msg
->lmsg
.u
.ms_resultp
;
1014 KKASSERT(&td
->td_msgport
== netisr_cpuport(io_ctx
->poll_cpuid
));
1016 crit_enter_quick(td
);
1019 netisr_replymsg(&msg
->base
, 0);
1021 if (io_ctx
->poll_handlers
== 0) {
1022 crit_exit_quick(td
);
1029 io_ctx
->pending_polls
--;
1030 pending_polls
= io_ctx
->pending_polls
;
1032 if (pending_polls
== 0) {
1037 * Last cycle was long and caused us to miss one or more
1038 * hardclock ticks. Restart processing again.
1040 sched_iopoll(io_ctx
);
1044 crit_exit_quick(td
);
1050 iopoll_add_sysctl(struct sysctl_ctx_list
*ctx
, struct sysctl_oid_list
*parent
,
1051 struct iopoll_ctx
*io_ctx
, int poll_type
)
1053 if (poll_type
== IFPOLL_RX
) {
1054 SYSCTL_ADD_PROC(ctx
, parent
, OID_AUTO
, "burst_max",
1055 CTLTYPE_UINT
| CTLFLAG_RW
, io_ctx
, 0, sysctl_burstmax
,
1056 "IU", "Max Polling burst size");
1058 SYSCTL_ADD_PROC(ctx
, parent
, OID_AUTO
, "each_burst",
1059 CTLTYPE_UINT
| CTLFLAG_RW
, io_ctx
, 0, sysctl_eachburst
,
1060 "IU", "Max size of each burst");
1062 SYSCTL_ADD_UINT(ctx
, parent
, OID_AUTO
, "burst", CTLFLAG_RD
,
1063 &io_ctx
->poll_burst
, 0, "Current polling burst size");
1065 SYSCTL_ADD_UINT(ctx
, parent
, OID_AUTO
, "user_frac", CTLFLAG_RW
,
1066 &io_ctx
->user_frac
, 0, "Desired user fraction of cpu time");
1068 SYSCTL_ADD_UINT(ctx
, parent
, OID_AUTO
, "kern_frac", CTLFLAG_RD
,
1069 &io_ctx
->kern_frac
, 0, "Kernel fraction of cpu time");
1071 SYSCTL_ADD_INT(ctx
, parent
, OID_AUTO
, "residual_burst", CTLFLAG_RD
,
1072 &io_ctx
->residual_burst
, 0,
1073 "# of residual cycles in burst");
1076 SYSCTL_ADD_UINT(ctx
, parent
, OID_AUTO
, "phase", CTLFLAG_RD
,
1077 &io_ctx
->phase
, 0, "Polling phase");
1079 SYSCTL_ADD_ULONG(ctx
, parent
, OID_AUTO
, "suspect", CTLFLAG_RW
,
1080 &io_ctx
->suspect
, "Suspected events");
1082 SYSCTL_ADD_ULONG(ctx
, parent
, OID_AUTO
, "stalled", CTLFLAG_RW
,
1083 &io_ctx
->stalled
, "Potential stalls");
1085 SYSCTL_ADD_ULONG(ctx
, parent
, OID_AUTO
, "short_ticks", CTLFLAG_RW
,
1086 &io_ctx
->short_ticks
,
1087 "Hardclock ticks shorter than they should be");
1089 SYSCTL_ADD_ULONG(ctx
, parent
, OID_AUTO
, "lost_polls", CTLFLAG_RW
,
1090 &io_ctx
->lost_polls
,
1091 "How many times we would have lost a poll tick");
1093 SYSCTL_ADD_UINT(ctx
, parent
, OID_AUTO
, "pending_polls", CTLFLAG_RD
,
1094 &io_ctx
->pending_polls
, 0, "Do we need to poll again");
1096 SYSCTL_ADD_UINT(ctx
, parent
, OID_AUTO
, "handlers", CTLFLAG_RD
,
1097 &io_ctx
->poll_handlers
, 0, "Number of registered poll handlers");
1101 sysctl_burstmax_handler(netmsg_t nmsg
)
1103 struct iopoll_sysctl_netmsg
*msg
= (struct iopoll_sysctl_netmsg
*)nmsg
;
1104 struct iopoll_ctx
*io_ctx
;
1107 KKASSERT(&curthread
->td_msgport
== netisr_cpuport(io_ctx
->poll_cpuid
));
1109 io_ctx
->poll_burst_max
= nmsg
->lmsg
.u
.ms_result
;
1110 if (io_ctx
->poll_each_burst
> io_ctx
->poll_burst_max
)
1111 io_ctx
->poll_each_burst
= io_ctx
->poll_burst_max
;
1112 if (io_ctx
->poll_burst
> io_ctx
->poll_burst_max
)
1113 io_ctx
->poll_burst
= io_ctx
->poll_burst_max
;
1114 if (io_ctx
->residual_burst
> io_ctx
->poll_burst_max
)
1115 io_ctx
->residual_burst
= io_ctx
->poll_burst_max
;
1117 netisr_replymsg(&nmsg
->base
, 0);
1121 sysctl_burstmax(SYSCTL_HANDLER_ARGS
)
1123 struct iopoll_ctx
*io_ctx
= arg1
;
1124 struct iopoll_sysctl_netmsg msg
;
1128 burst_max
= io_ctx
->poll_burst_max
;
1129 error
= sysctl_handle_int(oidp
, &burst_max
, 0, req
);
1130 if (error
|| req
->newptr
== NULL
)
1132 if (burst_max
< MIN_IOPOLL_BURST_MAX
)
1133 burst_max
= MIN_IOPOLL_BURST_MAX
;
1134 else if (burst_max
> MAX_IOPOLL_BURST_MAX
)
1135 burst_max
= MAX_IOPOLL_BURST_MAX
;
1137 netmsg_init(&msg
.base
, NULL
, &curthread
->td_msgport
,
1138 0, sysctl_burstmax_handler
);
1139 msg
.base
.lmsg
.u
.ms_result
= burst_max
;
1142 return netisr_domsg(&msg
.base
, io_ctx
->poll_cpuid
);
1146 sysctl_eachburst_handler(netmsg_t nmsg
)
1148 struct iopoll_sysctl_netmsg
*msg
= (struct iopoll_sysctl_netmsg
*)nmsg
;
1149 struct iopoll_ctx
*io_ctx
;
1150 uint32_t each_burst
;
1153 KKASSERT(&curthread
->td_msgport
== netisr_cpuport(io_ctx
->poll_cpuid
));
1155 each_burst
= nmsg
->lmsg
.u
.ms_result
;
1156 if (each_burst
> io_ctx
->poll_burst_max
)
1157 each_burst
= io_ctx
->poll_burst_max
;
1158 else if (each_burst
< 1)
1160 io_ctx
->poll_each_burst
= each_burst
;
1162 netisr_replymsg(&nmsg
->base
, 0);
1166 sysctl_eachburst(SYSCTL_HANDLER_ARGS
)
1168 struct iopoll_ctx
*io_ctx
= arg1
;
1169 struct iopoll_sysctl_netmsg msg
;
1170 uint32_t each_burst
;
1173 each_burst
= io_ctx
->poll_each_burst
;
1174 error
= sysctl_handle_int(oidp
, &each_burst
, 0, req
);
1175 if (error
|| req
->newptr
== NULL
)
1178 netmsg_init(&msg
.base
, NULL
, &curthread
->td_msgport
,
1179 0, sysctl_eachburst_handler
);
1180 msg
.base
.lmsg
.u
.ms_result
= each_burst
;
1183 return netisr_domsg(&msg
.base
, io_ctx
->poll_cpuid
);
1187 iopoll_register(struct ifnet
*ifp
, struct iopoll_ctx
*io_ctx
,
1188 const struct ifpoll_io
*io_rec
)
1192 KKASSERT(&curthread
->td_msgport
== netisr_cpuport(io_ctx
->poll_cpuid
));
1194 if (io_rec
->poll_func
== NULL
)
1198 * Check if there is room.
1200 if (io_ctx
->poll_handlers
>= IFPOLL_LIST_LEN
) {
1202 * List full, cannot register more entries.
1203 * This should never happen; if it does, it is probably a
1204 * broken driver trying to register multiple times. Checking
1205 * this at runtime is expensive, and won't solve the problem
1206 * anyways, so just report a few times and then give up.
1208 static int verbose
= 10; /* XXX */
1210 kprintf("io poll handlers list full, "
1211 "maybe a broken driver ?\n");
1216 struct iopoll_rec
*rec
= &io_ctx
->pr
[io_ctx
->poll_handlers
];
1219 rec
->serializer
= io_rec
->serializer
;
1220 rec
->arg
= io_rec
->arg
;
1221 rec
->poll_func
= io_rec
->poll_func
;
1223 io_ctx
->poll_handlers
++;
1230 iopoll_deregister(struct ifnet
*ifp
, struct iopoll_ctx
*io_ctx
)
1234 KKASSERT(&curthread
->td_msgport
== netisr_cpuport(io_ctx
->poll_cpuid
));
1236 for (i
= 0; i
< io_ctx
->poll_handlers
; ++i
) {
1237 if (io_ctx
->pr
[i
].ifp
== ifp
) /* Found it */
1240 if (i
== io_ctx
->poll_handlers
) {
1243 io_ctx
->poll_handlers
--;
1244 if (i
< io_ctx
->poll_handlers
) {
1245 /* Last entry replaces this one. */
1246 io_ctx
->pr
[i
] = io_ctx
->pr
[io_ctx
->poll_handlers
];
1249 if (io_ctx
->poll_handlers
== 0)
1250 iopoll_reset_state(io_ctx
);
1257 poll_comm_init(int cpuid
)
1259 struct poll_comm
*comm
;
1262 comm
= kmalloc_cachealign(sizeof(*comm
), M_DEVBUF
, M_WAITOK
| M_ZERO
);
1264 if (ifpoll_stfrac
< 1)
1265 ifpoll_stfrac
= IFPOLL_STFRAC_DEFAULT
;
1266 if (ifpoll_txfrac
< 1)
1267 ifpoll_txfrac
= IFPOLL_TXFRAC_DEFAULT
;
1269 comm
->poll_cpuid
= cpuid
;
1270 comm
->pollhz
= poll_comm_pollhz_div(comm
, ifpoll_pollhz
);
1271 comm
->poll_stfrac
= ifpoll_stfrac
- 1;
1272 comm
->poll_txfrac
= ifpoll_txfrac
- 1;
1274 ksnprintf(cpuid_str
, sizeof(cpuid_str
), "%d", cpuid
);
1276 sysctl_ctx_init(&comm
->sysctl_ctx
);
1277 comm
->sysctl_tree
= SYSCTL_ADD_NODE(&comm
->sysctl_ctx
,
1278 SYSCTL_STATIC_CHILDREN(_net_ifpoll
),
1279 OID_AUTO
, cpuid_str
, CTLFLAG_RD
, 0, "");
1281 SYSCTL_ADD_PROC(&comm
->sysctl_ctx
, SYSCTL_CHILDREN(comm
->sysctl_tree
),
1282 OID_AUTO
, "pollhz", CTLTYPE_INT
| CTLFLAG_RW
,
1283 comm
, 0, sysctl_pollhz
,
1284 "I", "Device polling frequency");
1287 SYSCTL_ADD_PROC(&comm
->sysctl_ctx
,
1288 SYSCTL_CHILDREN(comm
->sysctl_tree
),
1289 OID_AUTO
, "status_frac",
1290 CTLTYPE_INT
| CTLFLAG_RW
,
1291 comm
, 0, sysctl_stfrac
,
1292 "I", "# of cycles before status is polled");
1294 SYSCTL_ADD_PROC(&comm
->sysctl_ctx
, SYSCTL_CHILDREN(comm
->sysctl_tree
),
1295 OID_AUTO
, "tx_frac", CTLTYPE_INT
| CTLFLAG_RW
,
1296 comm
, 0, sysctl_txfrac
,
1297 "I", "# of cycles before TX is polled");
1299 poll_common
[cpuid
] = comm
;
1303 poll_comm_start(int cpuid
)
1305 struct poll_comm
*comm
= poll_common
[cpuid
];
1306 systimer_func_t func
;
1309 * Initialize systimer
1312 func
= poll_comm_systimer0
;
1314 func
= poll_comm_systimer
;
1315 systimer_init_periodic_nq(&comm
->pollclock
, func
, comm
, 1);
1319 _poll_comm_systimer(struct poll_comm
*comm
)
1321 iopoll_clock(rxpoll_context
[comm
->poll_cpuid
]);
1322 if (comm
->txfrac_count
-- == 0) {
1323 comm
->txfrac_count
= comm
->poll_txfrac
;
1324 iopoll_clock(txpoll_context
[comm
->poll_cpuid
]);
1329 poll_comm_systimer0(systimer_t info
, int in_ipi __unused
,
1330 struct intrframe
*frame __unused
)
1332 struct poll_comm
*comm
= info
->data
;
1333 globaldata_t gd
= mycpu
;
1335 KKASSERT(comm
->poll_cpuid
== gd
->gd_cpuid
&& gd
->gd_cpuid
== 0);
1339 if (comm
->stfrac_count
-- == 0) {
1340 comm
->stfrac_count
= comm
->poll_stfrac
;
1341 stpoll_clock(&stpoll_context
);
1343 _poll_comm_systimer(comm
);
1349 poll_comm_systimer(systimer_t info
, int in_ipi __unused
,
1350 struct intrframe
*frame __unused
)
1352 struct poll_comm
*comm
= info
->data
;
1353 globaldata_t gd
= mycpu
;
1355 KKASSERT(comm
->poll_cpuid
== gd
->gd_cpuid
&& gd
->gd_cpuid
!= 0);
1358 _poll_comm_systimer(comm
);
1363 poll_comm_adjust_pollhz(struct poll_comm
*comm
)
1368 KKASSERT(&curthread
->td_msgport
== netisr_cpuport(comm
->poll_cpuid
));
1371 * If there is no polling handler registered, set systimer
1372 * frequency to the lowest value. Polling systimer frequency
1373 * will be adjusted to the requested value, once there are
1374 * registered handlers.
1376 handlers
= rxpoll_context
[mycpuid
]->poll_handlers
+
1377 txpoll_context
[mycpuid
]->poll_handlers
;
1378 if (comm
->poll_cpuid
== 0)
1379 handlers
+= stpoll_context
.poll_handlers
;
1381 pollhz
= comm
->pollhz
;
1382 systimer_adjust_periodic(&comm
->pollclock
, pollhz
);
1386 sysctl_pollhz(SYSCTL_HANDLER_ARGS
)
1388 struct poll_comm
*comm
= arg1
;
1389 struct netmsg_base nmsg
;
1392 phz
= poll_comm_pollhz_conv(comm
, comm
->pollhz
);
1393 error
= sysctl_handle_int(oidp
, &phz
, 0, req
);
1394 if (error
|| req
->newptr
== NULL
)
1398 else if (phz
> IFPOLL_FREQ_MAX
)
1399 phz
= IFPOLL_FREQ_MAX
;
1401 netmsg_init(&nmsg
, NULL
, &curthread
->td_msgport
,
1402 0, sysctl_pollhz_handler
);
1403 nmsg
.lmsg
.u
.ms_result
= phz
;
1405 return netisr_domsg(&nmsg
, comm
->poll_cpuid
);
1409 sysctl_pollhz_handler(netmsg_t nmsg
)
1411 struct poll_comm
*comm
= poll_common
[mycpuid
];
1413 KKASSERT(&curthread
->td_msgport
== netisr_cpuport(comm
->poll_cpuid
));
1415 /* Save polling frequency */
1416 comm
->pollhz
= poll_comm_pollhz_div(comm
, nmsg
->lmsg
.u
.ms_result
);
1419 * Adjust cached pollhz
1421 rxpoll_context
[mycpuid
]->pollhz
= comm
->pollhz
;
1422 txpoll_context
[mycpuid
]->pollhz
=
1423 comm
->pollhz
/ (comm
->poll_txfrac
+ 1);
1426 * Adjust polling frequency
1428 poll_comm_adjust_pollhz(comm
);
1430 netisr_replymsg(&nmsg
->base
, 0);
1434 sysctl_stfrac(SYSCTL_HANDLER_ARGS
)
1436 struct poll_comm
*comm
= arg1
;
1437 struct netmsg_base nmsg
;
1440 KKASSERT(comm
->poll_cpuid
== 0);
1442 stfrac
= comm
->poll_stfrac
+ 1;
1443 error
= sysctl_handle_int(oidp
, &stfrac
, 0, req
);
1444 if (error
|| req
->newptr
== NULL
)
1449 netmsg_init(&nmsg
, NULL
, &curthread
->td_msgport
,
1450 0, sysctl_stfrac_handler
);
1451 nmsg
.lmsg
.u
.ms_result
= stfrac
- 1;
1453 return netisr_domsg(&nmsg
, comm
->poll_cpuid
);
1457 sysctl_stfrac_handler(netmsg_t nmsg
)
1459 struct poll_comm
*comm
= poll_common
[mycpuid
];
1460 int stfrac
= nmsg
->lmsg
.u
.ms_result
;
1462 KKASSERT(&curthread
->td_msgport
== netisr_cpuport(comm
->poll_cpuid
));
1465 comm
->poll_stfrac
= stfrac
;
1466 if (comm
->stfrac_count
> comm
->poll_stfrac
)
1467 comm
->stfrac_count
= comm
->poll_stfrac
;
1470 netisr_replymsg(&nmsg
->base
, 0);
1474 sysctl_txfrac(SYSCTL_HANDLER_ARGS
)
1476 struct poll_comm
*comm
= arg1
;
1477 struct netmsg_base nmsg
;
1480 txfrac
= comm
->poll_txfrac
+ 1;
1481 error
= sysctl_handle_int(oidp
, &txfrac
, 0, req
);
1482 if (error
|| req
->newptr
== NULL
)
1487 netmsg_init(&nmsg
, NULL
, &curthread
->td_msgport
,
1488 0, sysctl_txfrac_handler
);
1489 nmsg
.lmsg
.u
.ms_result
= txfrac
- 1;
1491 return netisr_domsg(&nmsg
, comm
->poll_cpuid
);
1495 sysctl_txfrac_handler(netmsg_t nmsg
)
1497 struct poll_comm
*comm
= poll_common
[mycpuid
];
1498 int txfrac
= nmsg
->lmsg
.u
.ms_result
;
1500 KKASSERT(&curthread
->td_msgport
== netisr_cpuport(comm
->poll_cpuid
));
1503 comm
->poll_txfrac
= txfrac
;
1504 if (comm
->txfrac_count
> comm
->poll_txfrac
)
1505 comm
->txfrac_count
= comm
->poll_txfrac
;
1508 netisr_replymsg(&nmsg
->base
, 0);
1512 ifpoll_compat_setup(struct ifpoll_compat
*cp
,
1513 struct sysctl_ctx_list
*sysctl_ctx
,
1514 struct sysctl_oid
*sysctl_tree
,
1515 int unit
, struct lwkt_serialize
*slz
)
1517 cp
->ifpc_stcount
= 0;
1518 cp
->ifpc_stfrac
= ((poll_common
[0]->poll_stfrac
+ 1) *
1519 howmany(IOPOLL_BURST_MAX
, IOPOLL_EACH_BURST
)) - 1;
1521 cp
->ifpc_cpuid
= unit
% netisr_ncpus
;
1522 cp
->ifpc_serializer
= slz
;
1524 if (sysctl_ctx
!= NULL
&& sysctl_tree
!= NULL
) {
1525 SYSCTL_ADD_PROC(sysctl_ctx
, SYSCTL_CHILDREN(sysctl_tree
),
1526 OID_AUTO
, "npoll_stfrac", CTLTYPE_INT
| CTLFLAG_RW
,
1527 cp
, 0, sysctl_compat_npoll_stfrac
, "I",
1528 "polling status frac");
1529 SYSCTL_ADD_PROC(sysctl_ctx
, SYSCTL_CHILDREN(sysctl_tree
),
1530 OID_AUTO
, "npoll_cpuid", CTLTYPE_INT
| CTLFLAG_RW
,
1531 cp
, 0, sysctl_compat_npoll_cpuid
, "I",
1537 sysctl_compat_npoll_stfrac(SYSCTL_HANDLER_ARGS
)
1539 struct ifpoll_compat
*cp
= arg1
;
1540 int error
= 0, stfrac
;
1542 lwkt_serialize_enter(cp
->ifpc_serializer
);
1544 stfrac
= cp
->ifpc_stfrac
+ 1;
1545 error
= sysctl_handle_int(oidp
, &stfrac
, 0, req
);
1546 if (!error
&& req
->newptr
!= NULL
) {
1550 cp
->ifpc_stfrac
= stfrac
- 1;
1551 if (cp
->ifpc_stcount
> cp
->ifpc_stfrac
)
1552 cp
->ifpc_stcount
= cp
->ifpc_stfrac
;
1556 lwkt_serialize_exit(cp
->ifpc_serializer
);
1561 sysctl_compat_npoll_cpuid(SYSCTL_HANDLER_ARGS
)
1563 struct ifpoll_compat
*cp
= arg1
;
1564 int error
= 0, cpuid
;
1566 lwkt_serialize_enter(cp
->ifpc_serializer
);
1568 cpuid
= cp
->ifpc_cpuid
;
1569 error
= sysctl_handle_int(oidp
, &cpuid
, 0, req
);
1570 if (!error
&& req
->newptr
!= NULL
) {
1571 if (cpuid
< 0 || cpuid
>= netisr_ncpus
)
1574 cp
->ifpc_cpuid
= cpuid
;
1577 lwkt_serialize_exit(cp
->ifpc_serializer
);