2 * Copyright (c) 2001-2002 Luigi Rizzo
4 * Supported by: the Xorp Project (www.xorp.org)
6 * Redistribution and use in source and binary forms, with or without
7 * modification, are permitted provided that the following conditions
9 * 1. Redistributions of source code must retain the above copyright
10 * notice, this list of conditions and the following disclaimer.
11 * 2. Redistributions in binary form must reproduce the above copyright
12 * notice, this list of conditions and the following disclaimer in the
13 * documentation and/or other materials provided with the distribution.
15 * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND
16 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
17 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
18 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE
19 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
20 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
27 * $FreeBSD: src/sys/kern/kern_poll.c,v 1.2.2.4 2002/06/27 23:26:33 luigi Exp $
30 #include "opt_ifpoll.h"
32 #include <sys/param.h>
33 #include <sys/kernel.h>
35 #include <sys/malloc.h>
36 #include <sys/serialize.h>
37 #include <sys/socket.h>
38 #include <sys/sysctl.h>
39 #include <sys/microtime_pcpu.h>
41 #include <sys/thread2.h>
42 #include <sys/msgport2.h>
45 #include <net/if_poll.h>
46 #include <net/netmsg2.h>
47 #include <net/netisr2.h>
50 * Polling support for network device drivers.
52 * Drivers which support this feature try to register one status polling
53 * handler and several TX/RX polling handlers with the polling code.
54 * If interface's if_npoll is called with non-NULL second argument, then
55 * a register operation is requested, else a deregister operation is
56 * requested. If the requested operation is "register", driver should
57 * setup the ifpoll_info passed in accoding its own needs:
58 * ifpoll_info.ifpi_status.status_func == NULL
59 * No status polling handler will be installed on CPU(0)
60 * ifpoll_info.ifpi_rx[n].poll_func == NULL
61 * No RX polling handler will be installed on CPU(n)
62 * ifpoll_info.ifpi_tx[n].poll_func == NULL
63 * No TX polling handler will be installed on CPU(n)
65 * RX is polled at the specified polling frequency (net.ifpoll.X.pollhz).
66 * TX and status polling could be done at lower frequency than RX frequency
67 * (net.ifpoll.0.status_frac and net.ifpoll.X.tx_frac). To avoid systimer
68 * staggering at high frequency, RX systimer gives TX and status polling a
71 * All of the registered polling handlers are called only if the interface
72 * is marked as 'IFF_RUNNING and IFF_NPOLLING'. However, the interface's
73 * register and deregister function (ifnet.if_npoll) will be called even
74 * if interface is not marked with 'IFF_RUNNING'.
76 * If registration is successful, the driver must disable interrupts,
77 * and further I/O is performed through the TX/RX polling handler, which
78 * are invoked (at least once per clock tick) with 3 arguments: the "arg"
79 * passed at register time, a struct ifnet pointer, and a "count" limit.
80 * The registered serializer will be held before calling the related
83 * The count limit specifies how much work the handler can do during the
84 * call -- typically this is the number of packets to be received, or
85 * transmitted, etc. (drivers are free to interpret this number, as long
86 * as the max time spent in the function grows roughly linearly with the
89 * A second variable controls the sharing of CPU between polling/kernel
90 * network processing, and other activities (typically userlevel tasks):
91 * net.ifpoll.X.{rx,tx}.user_frac (between 0 and 100, default 50) sets the
92 * share of CPU allocated to user tasks. CPU is allocated proportionally
93 * to the shares, by dynamically adjusting the "count" (poll_burst).
95 * Other parameters can should be left to their default values.
96 * The following constraints hold
98 * 1 <= poll_burst <= poll_burst_max
99 * 1 <= poll_each_burst <= poll_burst_max
100 * MIN_POLL_BURST_MAX <= poll_burst_max <= MAX_POLL_BURST_MAX
103 #define IFPOLL_LIST_LEN 128
104 #define IFPOLL_FREQ_MAX 30000
106 #define MIN_IOPOLL_BURST_MAX 10
107 #define MAX_IOPOLL_BURST_MAX 5000
108 #define IOPOLL_BURST_MAX 250 /* good for 1000Mbit net and HZ=6000 */
110 #define IOPOLL_EACH_BURST 50
111 #define IOPOLL_USER_FRAC 50
113 #define IFPOLL_FREQ_DEFAULT 6000
115 #define IFPOLL_TXFRAC_DEFAULT 1 /* 1/1 of the pollhz */
116 #define IFPOLL_STFRAC_DEFAULT 120 /* 1/120 of the pollhz */
118 #define IFPOLL_RX 0x1
119 #define IFPOLL_TX 0x2
122 struct lwkt_serialize
*serializer
;
125 ifpoll_iofn_t poll_func
;
129 union microtime_pcpu prev_t
;
130 u_long short_ticks
; /* statistics */
131 u_long lost_polls
; /* statistics */
132 u_long suspect
; /* statistics */
133 u_long stalled
; /* statistics */
134 uint32_t pending_polls
; /* state */
136 struct netmsg_base poll_netmsg
;
137 struct netmsg_base poll_more_netmsg
;
141 uint32_t phase
; /* state */
142 int residual_burst
; /* state */
143 uint32_t poll_each_burst
; /* tunable */
144 union microtime_pcpu poll_start_t
; /* state */
146 uint32_t poll_burst
; /* state */
147 uint32_t poll_burst_max
; /* tunable */
148 uint32_t user_frac
; /* tunable */
149 uint32_t kern_frac
; /* state */
151 uint32_t poll_handlers
; /* next free entry in pr[]. */
152 struct iopoll_rec pr
[IFPOLL_LIST_LEN
];
154 struct sysctl_ctx_list poll_sysctl_ctx
;
155 struct sysctl_oid
*poll_sysctl_tree
;
159 struct systimer pollclock
;
162 int stfrac_count
; /* state */
163 int poll_stfrac
; /* tunable */
165 int txfrac_count
; /* state */
166 int poll_txfrac
; /* tunable */
168 int pollhz
; /* tunable */
170 struct sysctl_ctx_list sysctl_ctx
;
171 struct sysctl_oid
*sysctl_tree
;
175 struct lwkt_serialize
*serializer
;
177 ifpoll_stfn_t status_func
;
181 struct netmsg_base poll_netmsg
;
183 uint32_t poll_handlers
; /* next free entry in pr[]. */
184 struct stpoll_rec pr
[IFPOLL_LIST_LEN
];
186 struct sysctl_ctx_list poll_sysctl_ctx
;
187 struct sysctl_oid
*poll_sysctl_tree
;
190 struct iopoll_sysctl_netmsg
{
191 struct netmsg_base base
;
192 struct iopoll_ctx
*ctx
;
195 void ifpoll_init_pcpu(int);
196 static void ifpoll_register_handler(netmsg_t
);
197 static void ifpoll_deregister_handler(netmsg_t
);
202 static void stpoll_init(void);
203 static void stpoll_handler(netmsg_t
);
204 static void stpoll_clock(struct stpoll_ctx
*);
205 static int stpoll_register(struct ifnet
*, const struct ifpoll_status
*);
206 static int stpoll_deregister(struct ifnet
*);
211 static struct iopoll_ctx
*iopoll_ctx_create(int, int);
212 static void iopoll_init(int);
213 static void rxpoll_handler(netmsg_t
);
214 static void txpoll_handler(netmsg_t
);
215 static void rxpollmore_handler(netmsg_t
);
216 static void txpollmore_handler(netmsg_t
);
217 static void iopoll_clock(struct iopoll_ctx
*);
218 static int iopoll_register(struct ifnet
*, struct iopoll_ctx
*,
219 const struct ifpoll_io
*);
220 static int iopoll_deregister(struct ifnet
*, struct iopoll_ctx
*);
222 static void iopoll_add_sysctl(struct sysctl_ctx_list
*,
223 struct sysctl_oid_list
*, struct iopoll_ctx
*, int);
224 static void sysctl_burstmax_handler(netmsg_t
);
225 static int sysctl_burstmax(SYSCTL_HANDLER_ARGS
);
226 static void sysctl_eachburst_handler(netmsg_t
);
227 static int sysctl_eachburst(SYSCTL_HANDLER_ARGS
);
232 static void poll_comm_init(int);
233 static void poll_comm_start(int);
234 static void poll_comm_adjust_pollhz(struct poll_comm
*);
235 static void poll_comm_systimer0(systimer_t
, int, struct intrframe
*);
236 static void poll_comm_systimer(systimer_t
, int, struct intrframe
*);
237 static void sysctl_pollhz_handler(netmsg_t
);
238 static void sysctl_stfrac_handler(netmsg_t
);
239 static void sysctl_txfrac_handler(netmsg_t
);
240 static int sysctl_pollhz(SYSCTL_HANDLER_ARGS
);
241 static int sysctl_stfrac(SYSCTL_HANDLER_ARGS
);
242 static int sysctl_txfrac(SYSCTL_HANDLER_ARGS
);
243 static int sysctl_compat_npoll_stfrac(SYSCTL_HANDLER_ARGS
);
244 static int sysctl_compat_npoll_cpuid(SYSCTL_HANDLER_ARGS
);
246 static struct stpoll_ctx stpoll_context
;
247 static struct poll_comm
*poll_common
[MAXCPU
];
248 static struct iopoll_ctx
*rxpoll_context
[MAXCPU
];
249 static struct iopoll_ctx
*txpoll_context
[MAXCPU
];
251 SYSCTL_NODE(_net
, OID_AUTO
, ifpoll
, CTLFLAG_RW
, 0,
252 "Network device polling parameters");
254 static int iopoll_burst_max
= IOPOLL_BURST_MAX
;
255 static int iopoll_each_burst
= IOPOLL_EACH_BURST
;
256 static int iopoll_user_frac
= IOPOLL_USER_FRAC
;
258 static int ifpoll_pollhz
= IFPOLL_FREQ_DEFAULT
;
259 static int ifpoll_stfrac
= IFPOLL_STFRAC_DEFAULT
;
260 static int ifpoll_txfrac
= IFPOLL_TXFRAC_DEFAULT
;
262 TUNABLE_INT("net.ifpoll.burst_max", &iopoll_burst_max
);
263 TUNABLE_INT("net.ifpoll.each_burst", &iopoll_each_burst
);
264 TUNABLE_INT("net.ifpoll.user_frac", &iopoll_user_frac
);
265 TUNABLE_INT("net.ifpoll.pollhz", &ifpoll_pollhz
);
266 TUNABLE_INT("net.ifpoll.status_frac", &ifpoll_stfrac
);
267 TUNABLE_INT("net.ifpoll.tx_frac", &ifpoll_txfrac
);
269 #if !defined(KTR_IF_POLL)
270 #define KTR_IF_POLL KTR_ALL
272 KTR_INFO_MASTER(if_poll
);
273 KTR_INFO(KTR_IF_POLL
, if_poll
, rx_start
, 0, "rx start");
274 KTR_INFO(KTR_IF_POLL
, if_poll
, rx_end
, 1, "rx end");
275 KTR_INFO(KTR_IF_POLL
, if_poll
, tx_start
, 2, "tx start");
276 KTR_INFO(KTR_IF_POLL
, if_poll
, tx_end
, 3, "tx end");
277 KTR_INFO(KTR_IF_POLL
, if_poll
, rx_mstart
, 4, "rx more start");
278 KTR_INFO(KTR_IF_POLL
, if_poll
, rx_mend
, 5, "rx more end");
279 KTR_INFO(KTR_IF_POLL
, if_poll
, tx_mstart
, 6, "tx more start");
280 KTR_INFO(KTR_IF_POLL
, if_poll
, tx_mend
, 7, "tx more end");
281 KTR_INFO(KTR_IF_POLL
, if_poll
, ioclock_start
, 8, "ioclock start");
282 KTR_INFO(KTR_IF_POLL
, if_poll
, ioclock_end
, 9, "ioclock end");
283 #define logpoll(name) KTR_LOG(if_poll_ ## name)
285 #define IFPOLL_FREQ_ADJ(comm) (((comm)->poll_cpuid * 3) % 50)
288 poll_comm_pollhz_div(const struct poll_comm
*comm
, int pollhz
)
290 return pollhz
+ IFPOLL_FREQ_ADJ(comm
);
294 poll_comm_pollhz_conv(const struct poll_comm
*comm
, int pollhz
)
296 return pollhz
- IFPOLL_FREQ_ADJ(comm
);
300 ifpoll_sendmsg_oncpu(netmsg_t msg
)
302 if (msg
->lmsg
.ms_flags
& MSGF_DONE
)
303 lwkt_sendmsg_oncpu(netisr_cpuport(mycpuid
), &msg
->lmsg
);
307 sched_stpoll(struct stpoll_ctx
*st_ctx
)
309 ifpoll_sendmsg_oncpu((netmsg_t
)&st_ctx
->poll_netmsg
);
313 sched_iopoll(struct iopoll_ctx
*io_ctx
)
315 ifpoll_sendmsg_oncpu((netmsg_t
)&io_ctx
->poll_netmsg
);
319 sched_iopollmore(struct iopoll_ctx
*io_ctx
)
321 ifpoll_sendmsg_oncpu((netmsg_t
)&io_ctx
->poll_more_netmsg
);
325 * Initialize per-cpu polling(4) context. Called from kern_clock.c:
328 ifpoll_init_pcpu(int cpuid
)
333 poll_comm_init(cpuid
);
339 poll_comm_start(cpuid
);
343 ifpoll_register(struct ifnet
*ifp
)
345 struct ifpoll_info
*info
;
346 struct netmsg_base nmsg
;
349 if (ifp
->if_npoll
== NULL
) {
350 /* Device does not support polling */
354 info
= kmalloc(sizeof(*info
), M_TEMP
, M_WAITOK
| M_ZERO
);
357 * Attempt to register. Interlock with IFF_NPOLLING.
360 ifnet_serialize_all(ifp
);
362 if (ifp
->if_flags
& IFF_NPOLLING
) {
363 /* Already polling */
364 ifnet_deserialize_all(ifp
);
369 info
->ifpi_ifp
= ifp
;
371 ifp
->if_flags
|= IFF_NPOLLING
;
372 ifp
->if_npoll(ifp
, info
);
374 ifnet_deserialize_all(ifp
);
376 netmsg_init(&nmsg
, NULL
, &curthread
->td_msgport
,
377 0, ifpoll_register_handler
);
378 nmsg
.lmsg
.u
.ms_resultp
= info
;
380 error
= lwkt_domsg(netisr_cpuport(0), &nmsg
.lmsg
, 0);
382 if (!ifpoll_deregister(ifp
)) {
383 if_printf(ifp
, "ifpoll_register: "
384 "ifpoll_deregister failed!\n");
393 ifpoll_deregister(struct ifnet
*ifp
)
395 struct netmsg_base nmsg
;
398 if (ifp
->if_npoll
== NULL
)
401 ifnet_serialize_all(ifp
);
403 if ((ifp
->if_flags
& IFF_NPOLLING
) == 0) {
404 ifnet_deserialize_all(ifp
);
407 ifp
->if_flags
&= ~IFF_NPOLLING
;
409 ifnet_deserialize_all(ifp
);
411 netmsg_init(&nmsg
, NULL
, &curthread
->td_msgport
,
412 0, ifpoll_deregister_handler
);
413 nmsg
.lmsg
.u
.ms_resultp
= ifp
;
415 error
= lwkt_domsg(netisr_cpuport(0), &nmsg
.lmsg
, 0);
417 ifnet_serialize_all(ifp
);
418 ifp
->if_npoll(ifp
, NULL
);
419 ifnet_deserialize_all(ifp
);
425 ifpoll_register_handler(netmsg_t nmsg
)
427 const struct ifpoll_info
*info
= nmsg
->lmsg
.u
.ms_resultp
;
428 int cpuid
= mycpuid
, nextcpu
;
431 KKASSERT(cpuid
< ncpus2
);
432 KKASSERT(&curthread
->td_msgport
== netisr_cpuport(cpuid
));
435 error
= stpoll_register(info
->ifpi_ifp
, &info
->ifpi_status
);
440 error
= iopoll_register(info
->ifpi_ifp
, rxpoll_context
[cpuid
],
441 &info
->ifpi_rx
[cpuid
]);
445 error
= iopoll_register(info
->ifpi_ifp
, txpoll_context
[cpuid
],
446 &info
->ifpi_tx
[cpuid
]);
450 /* Adjust polling frequency, after all registration is done */
451 poll_comm_adjust_pollhz(poll_common
[cpuid
]);
454 if (nextcpu
< ncpus2
)
455 lwkt_forwardmsg(netisr_cpuport(nextcpu
), &nmsg
->lmsg
);
457 lwkt_replymsg(&nmsg
->lmsg
, 0);
460 lwkt_replymsg(&nmsg
->lmsg
, error
);
464 ifpoll_deregister_handler(netmsg_t nmsg
)
466 struct ifnet
*ifp
= nmsg
->lmsg
.u
.ms_resultp
;
467 int cpuid
= mycpuid
, nextcpu
;
469 KKASSERT(cpuid
< ncpus2
);
470 KKASSERT(&curthread
->td_msgport
== netisr_cpuport(cpuid
));
474 stpoll_deregister(ifp
);
475 iopoll_deregister(ifp
, rxpoll_context
[cpuid
]);
476 iopoll_deregister(ifp
, txpoll_context
[cpuid
]);
478 /* Adjust polling frequency, after all deregistration is done */
479 poll_comm_adjust_pollhz(poll_common
[cpuid
]);
482 if (nextcpu
< ncpus2
)
483 lwkt_forwardmsg(netisr_cpuport(nextcpu
), &nmsg
->lmsg
);
485 lwkt_replymsg(&nmsg
->lmsg
, 0);
491 struct stpoll_ctx
*st_ctx
= &stpoll_context
;
492 const struct poll_comm
*comm
= poll_common
[0];
494 sysctl_ctx_init(&st_ctx
->poll_sysctl_ctx
);
495 st_ctx
->poll_sysctl_tree
= SYSCTL_ADD_NODE(&st_ctx
->poll_sysctl_ctx
,
496 SYSCTL_CHILDREN(comm
->sysctl_tree
),
497 OID_AUTO
, "status", CTLFLAG_RD
, 0, "");
499 SYSCTL_ADD_UINT(&st_ctx
->poll_sysctl_ctx
,
500 SYSCTL_CHILDREN(st_ctx
->poll_sysctl_tree
),
501 OID_AUTO
, "handlers", CTLFLAG_RD
,
502 &st_ctx
->poll_handlers
, 0,
503 "Number of registered status poll handlers");
505 netmsg_init(&st_ctx
->poll_netmsg
, NULL
, &netisr_adone_rport
,
510 * stpoll_handler is scheduled by sched_stpoll when appropriate, typically
511 * once per polling systimer tick.
514 stpoll_handler(netmsg_t msg
)
516 struct stpoll_ctx
*st_ctx
= &stpoll_context
;
517 struct thread
*td
= curthread
;
520 ASSERT_IS_NETISR(td
, 0);
522 crit_enter_quick(td
);
525 lwkt_replymsg(&msg
->lmsg
, 0);
527 if (st_ctx
->poll_handlers
== 0) {
532 for (i
= 0; i
< st_ctx
->poll_handlers
; ++i
) {
533 const struct stpoll_rec
*rec
= &st_ctx
->pr
[i
];
534 struct ifnet
*ifp
= rec
->ifp
;
536 if (!lwkt_serialize_try(rec
->serializer
))
539 if ((ifp
->if_flags
& (IFF_RUNNING
| IFF_NPOLLING
)) ==
540 (IFF_RUNNING
| IFF_NPOLLING
))
541 rec
->status_func(ifp
);
543 lwkt_serialize_exit(rec
->serializer
);
550 * Hook from status poll systimer. Tries to schedule an status poll.
551 * NOTE: Caller should hold critical section.
554 stpoll_clock(struct stpoll_ctx
*st_ctx
)
556 KKASSERT(mycpuid
== 0);
558 if (st_ctx
->poll_handlers
== 0)
560 sched_stpoll(st_ctx
);
564 stpoll_register(struct ifnet
*ifp
, const struct ifpoll_status
*st_rec
)
566 struct stpoll_ctx
*st_ctx
= &stpoll_context
;
571 if (st_rec
->status_func
== NULL
)
575 * Check if there is room.
577 if (st_ctx
->poll_handlers
>= IFPOLL_LIST_LEN
) {
579 * List full, cannot register more entries.
580 * This should never happen; if it does, it is probably a
581 * broken driver trying to register multiple times. Checking
582 * this at runtime is expensive, and won't solve the problem
583 * anyways, so just report a few times and then give up.
585 static int verbose
= 10; /* XXX */
588 kprintf("status poll handlers list full, "
589 "maybe a broken driver ?\n");
594 struct stpoll_rec
*rec
= &st_ctx
->pr
[st_ctx
->poll_handlers
];
597 rec
->serializer
= st_rec
->serializer
;
598 rec
->status_func
= st_rec
->status_func
;
600 st_ctx
->poll_handlers
++;
607 stpoll_deregister(struct ifnet
*ifp
)
609 struct stpoll_ctx
*st_ctx
= &stpoll_context
;
614 for (i
= 0; i
< st_ctx
->poll_handlers
; ++i
) {
615 if (st_ctx
->pr
[i
].ifp
== ifp
) /* Found it */
618 if (i
== st_ctx
->poll_handlers
) {
621 st_ctx
->poll_handlers
--;
622 if (i
< st_ctx
->poll_handlers
) {
623 /* Last entry replaces this one. */
624 st_ctx
->pr
[i
] = st_ctx
->pr
[st_ctx
->poll_handlers
];
632 iopoll_reset_state(struct iopoll_ctx
*io_ctx
)
635 io_ctx
->poll_burst
= io_ctx
->poll_each_burst
;
636 io_ctx
->pending_polls
= 0;
637 io_ctx
->residual_burst
= 0;
639 io_ctx
->kern_frac
= 0;
640 bzero(&io_ctx
->poll_start_t
, sizeof(io_ctx
->poll_start_t
));
641 bzero(&io_ctx
->prev_t
, sizeof(io_ctx
->prev_t
));
646 iopoll_init(int cpuid
)
648 KKASSERT(cpuid
< ncpus2
);
650 rxpoll_context
[cpuid
] = iopoll_ctx_create(cpuid
, IFPOLL_RX
);
651 txpoll_context
[cpuid
] = iopoll_ctx_create(cpuid
, IFPOLL_TX
);
654 static struct iopoll_ctx
*
655 iopoll_ctx_create(int cpuid
, int poll_type
)
657 struct poll_comm
*comm
;
658 struct iopoll_ctx
*io_ctx
;
659 const char *poll_type_str
;
660 netisr_fn_t handler
, more_handler
;
662 KKASSERT(poll_type
== IFPOLL_RX
|| poll_type
== IFPOLL_TX
);
665 * Make sure that tunables are in sane state
667 if (iopoll_burst_max
< MIN_IOPOLL_BURST_MAX
)
668 iopoll_burst_max
= MIN_IOPOLL_BURST_MAX
;
669 else if (iopoll_burst_max
> MAX_IOPOLL_BURST_MAX
)
670 iopoll_burst_max
= MAX_IOPOLL_BURST_MAX
;
672 if (iopoll_each_burst
> iopoll_burst_max
)
673 iopoll_each_burst
= iopoll_burst_max
;
675 comm
= poll_common
[cpuid
];
678 * Create the per-cpu polling context
680 io_ctx
= kmalloc_cachealign(sizeof(*io_ctx
), M_DEVBUF
,
683 io_ctx
->poll_each_burst
= iopoll_each_burst
;
684 io_ctx
->poll_burst_max
= iopoll_burst_max
;
685 io_ctx
->user_frac
= iopoll_user_frac
;
686 if (poll_type
== IFPOLL_RX
)
687 io_ctx
->pollhz
= comm
->pollhz
;
689 io_ctx
->pollhz
= comm
->pollhz
/ (comm
->poll_txfrac
+ 1);
690 io_ctx
->poll_cpuid
= cpuid
;
691 iopoll_reset_state(io_ctx
);
693 if (poll_type
== IFPOLL_RX
) {
694 handler
= rxpoll_handler
;
695 more_handler
= rxpollmore_handler
;
697 handler
= txpoll_handler
;
698 more_handler
= txpollmore_handler
;
701 netmsg_init(&io_ctx
->poll_netmsg
, NULL
, &netisr_adone_rport
,
703 io_ctx
->poll_netmsg
.lmsg
.u
.ms_resultp
= io_ctx
;
705 netmsg_init(&io_ctx
->poll_more_netmsg
, NULL
, &netisr_adone_rport
,
707 io_ctx
->poll_more_netmsg
.lmsg
.u
.ms_resultp
= io_ctx
;
710 * Initialize per-cpu sysctl nodes
712 if (poll_type
== IFPOLL_RX
)
713 poll_type_str
= "rx";
715 poll_type_str
= "tx";
717 sysctl_ctx_init(&io_ctx
->poll_sysctl_ctx
);
718 io_ctx
->poll_sysctl_tree
= SYSCTL_ADD_NODE(&io_ctx
->poll_sysctl_ctx
,
719 SYSCTL_CHILDREN(comm
->sysctl_tree
),
720 OID_AUTO
, poll_type_str
, CTLFLAG_RD
, 0, "");
721 iopoll_add_sysctl(&io_ctx
->poll_sysctl_ctx
,
722 SYSCTL_CHILDREN(io_ctx
->poll_sysctl_tree
), io_ctx
, poll_type
);
728 * Hook from iopoll systimer. Tries to schedule an iopoll, but keeps
729 * track of lost ticks due to the previous handler taking too long.
730 * Normally, this should not happen, because polling handler should
731 * run for a short time. However, in some cases (e.g. when there are
732 * changes in link status etc.) the drivers take a very long time
733 * (even in the order of milliseconds) to reset and reconfigure the
734 * device, causing apparent lost polls.
736 * The first part of the code is just for debugging purposes, and tries
737 * to count how often hardclock ticks are shorter than they should,
738 * meaning either stray interrupts or delayed events.
740 * WARNING! called from fastint or IPI, the MP lock might not be held.
741 * NOTE: Caller should hold critical section.
744 iopoll_clock(struct iopoll_ctx
*io_ctx
)
746 union microtime_pcpu t
;
749 KKASSERT(mycpuid
== io_ctx
->poll_cpuid
);
751 if (io_ctx
->poll_handlers
== 0)
754 logpoll(ioclock_start
);
756 microtime_pcpu_get(&t
);
757 delta
= microtime_pcpu_diff(&io_ctx
->prev_t
, &t
);
758 if (delta
* io_ctx
->pollhz
< 500000)
759 io_ctx
->short_ticks
++;
763 if (io_ctx
->pending_polls
> 100) {
765 * Too much, assume it has stalled (not always true
766 * see comment above).
769 io_ctx
->pending_polls
= 0;
773 if (io_ctx
->phase
<= 2) {
774 if (io_ctx
->phase
!= 0)
777 sched_iopoll(io_ctx
);
780 if (io_ctx
->pending_polls
++ > 0)
781 io_ctx
->lost_polls
++;
783 logpoll(ioclock_end
);
787 * rxpoll_handler and txpoll_handler are scheduled by sched_iopoll when
788 * appropriate, typically once per polling systimer tick.
790 * Note that the message is replied immediately in order to allow a new
791 * ISR to be scheduled in the handler.
794 rxpoll_handler(netmsg_t msg
)
796 struct iopoll_ctx
*io_ctx
;
797 struct thread
*td
= curthread
;
802 io_ctx
= msg
->lmsg
.u
.ms_resultp
;
803 KKASSERT(&td
->td_msgport
== netisr_cpuport(io_ctx
->poll_cpuid
));
805 crit_enter_quick(td
);
808 lwkt_replymsg(&msg
->lmsg
, 0);
810 if (io_ctx
->poll_handlers
== 0) {
817 if (io_ctx
->residual_burst
== 0) {
818 /* First call in this tick */
819 microtime_pcpu_get(&io_ctx
->poll_start_t
);
820 io_ctx
->residual_burst
= io_ctx
->poll_burst
;
822 cycles
= (io_ctx
->residual_burst
< io_ctx
->poll_each_burst
) ?
823 io_ctx
->residual_burst
: io_ctx
->poll_each_burst
;
824 io_ctx
->residual_burst
-= cycles
;
826 for (i
= 0; i
< io_ctx
->poll_handlers
; i
++) {
827 const struct iopoll_rec
*rec
= &io_ctx
->pr
[i
];
828 struct ifnet
*ifp
= rec
->ifp
;
830 if (!lwkt_serialize_try(rec
->serializer
))
833 if ((ifp
->if_flags
& (IFF_RUNNING
| IFF_NPOLLING
)) ==
834 (IFF_RUNNING
| IFF_NPOLLING
))
835 rec
->poll_func(ifp
, rec
->arg
, cycles
);
837 lwkt_serialize_exit(rec
->serializer
);
841 * Do a quick exit/enter to catch any higher-priority
845 crit_enter_quick(td
);
847 sched_iopollmore(io_ctx
);
856 txpoll_handler(netmsg_t msg
)
858 struct iopoll_ctx
*io_ctx
;
859 struct thread
*td
= curthread
;
864 io_ctx
= msg
->lmsg
.u
.ms_resultp
;
865 KKASSERT(&td
->td_msgport
== netisr_cpuport(io_ctx
->poll_cpuid
));
867 crit_enter_quick(td
);
870 lwkt_replymsg(&msg
->lmsg
, 0);
872 if (io_ctx
->poll_handlers
== 0) {
880 for (i
= 0; i
< io_ctx
->poll_handlers
; i
++) {
881 const struct iopoll_rec
*rec
= &io_ctx
->pr
[i
];
882 struct ifnet
*ifp
= rec
->ifp
;
884 if (!lwkt_serialize_try(rec
->serializer
))
887 if ((ifp
->if_flags
& (IFF_RUNNING
| IFF_NPOLLING
)) ==
888 (IFF_RUNNING
| IFF_NPOLLING
))
889 rec
->poll_func(ifp
, rec
->arg
, -1);
891 lwkt_serialize_exit(rec
->serializer
);
895 * Do a quick exit/enter to catch any higher-priority
899 crit_enter_quick(td
);
901 sched_iopollmore(io_ctx
);
910 * rxpollmore_handler and txpollmore_handler are called after other netisr's,
911 * possibly scheduling another rxpoll_handler or txpoll_handler call, or
912 * adapting the burst size for the next cycle.
914 * It is very bad to fetch large bursts of packets from a single card at once,
915 * because the burst could take a long time to be completely processed leading
916 * to unfairness. To reduce the problem, and also to account better for time
917 * spent in network-related processing, we split the burst in smaller chunks
918 * of fixed size, giving control to the other netisr's between chunks. This
919 * helps in improving the fairness, reducing livelock and accounting for the
920 * work performed in low level handling.
923 rxpollmore_handler(netmsg_t msg
)
925 struct thread
*td
= curthread
;
926 struct iopoll_ctx
*io_ctx
;
927 union microtime_pcpu t
;
929 uint32_t pending_polls
;
933 io_ctx
= msg
->lmsg
.u
.ms_resultp
;
934 KKASSERT(&td
->td_msgport
== netisr_cpuport(io_ctx
->poll_cpuid
));
936 crit_enter_quick(td
);
939 lwkt_replymsg(&msg
->lmsg
, 0);
941 if (io_ctx
->poll_handlers
== 0) {
948 if (io_ctx
->residual_burst
> 0) {
949 sched_iopoll(io_ctx
);
951 /* Will run immediately on return, followed by netisrs */
956 /* Here we can account time spent in iopoll's in this tick */
957 microtime_pcpu_get(&t
);
958 kern_load
= microtime_pcpu_diff(&io_ctx
->poll_start_t
, &t
);
959 kern_load
= (kern_load
* io_ctx
->pollhz
) / 10000; /* 0..100 */
960 io_ctx
->kern_frac
= kern_load
;
962 if (kern_load
> (100 - io_ctx
->user_frac
)) {
963 /* Try decrease ticks */
964 if (io_ctx
->poll_burst
> 1)
965 io_ctx
->poll_burst
--;
967 if (io_ctx
->poll_burst
< io_ctx
->poll_burst_max
)
968 io_ctx
->poll_burst
++;
971 io_ctx
->pending_polls
--;
972 pending_polls
= io_ctx
->pending_polls
;
974 if (pending_polls
== 0) {
979 * Last cycle was long and caused us to miss one or more
980 * hardclock ticks. Restart processing again, but slightly
981 * reduce the burst size to prevent that this happens again.
983 io_ctx
->poll_burst
-= (io_ctx
->poll_burst
/ 8);
984 if (io_ctx
->poll_burst
< 1)
985 io_ctx
->poll_burst
= 1;
986 sched_iopoll(io_ctx
);
996 txpollmore_handler(netmsg_t msg
)
998 struct thread
*td
= curthread
;
999 struct iopoll_ctx
*io_ctx
;
1000 uint32_t pending_polls
;
1004 io_ctx
= msg
->lmsg
.u
.ms_resultp
;
1005 KKASSERT(&td
->td_msgport
== netisr_cpuport(io_ctx
->poll_cpuid
));
1007 crit_enter_quick(td
);
1010 lwkt_replymsg(&msg
->lmsg
, 0);
1012 if (io_ctx
->poll_handlers
== 0) {
1013 crit_exit_quick(td
);
1020 io_ctx
->pending_polls
--;
1021 pending_polls
= io_ctx
->pending_polls
;
1023 if (pending_polls
== 0) {
1028 * Last cycle was long and caused us to miss one or more
1029 * hardclock ticks. Restart processing again.
1031 sched_iopoll(io_ctx
);
1035 crit_exit_quick(td
);
1041 iopoll_add_sysctl(struct sysctl_ctx_list
*ctx
, struct sysctl_oid_list
*parent
,
1042 struct iopoll_ctx
*io_ctx
, int poll_type
)
1044 if (poll_type
== IFPOLL_RX
) {
1045 SYSCTL_ADD_PROC(ctx
, parent
, OID_AUTO
, "burst_max",
1046 CTLTYPE_UINT
| CTLFLAG_RW
, io_ctx
, 0, sysctl_burstmax
,
1047 "IU", "Max Polling burst size");
1049 SYSCTL_ADD_PROC(ctx
, parent
, OID_AUTO
, "each_burst",
1050 CTLTYPE_UINT
| CTLFLAG_RW
, io_ctx
, 0, sysctl_eachburst
,
1051 "IU", "Max size of each burst");
1053 SYSCTL_ADD_UINT(ctx
, parent
, OID_AUTO
, "burst", CTLFLAG_RD
,
1054 &io_ctx
->poll_burst
, 0, "Current polling burst size");
1056 SYSCTL_ADD_UINT(ctx
, parent
, OID_AUTO
, "user_frac", CTLFLAG_RW
,
1057 &io_ctx
->user_frac
, 0, "Desired user fraction of cpu time");
1059 SYSCTL_ADD_UINT(ctx
, parent
, OID_AUTO
, "kern_frac", CTLFLAG_RD
,
1060 &io_ctx
->kern_frac
, 0, "Kernel fraction of cpu time");
1062 SYSCTL_ADD_INT(ctx
, parent
, OID_AUTO
, "residual_burst", CTLFLAG_RD
,
1063 &io_ctx
->residual_burst
, 0,
1064 "# of residual cycles in burst");
1067 SYSCTL_ADD_UINT(ctx
, parent
, OID_AUTO
, "phase", CTLFLAG_RD
,
1068 &io_ctx
->phase
, 0, "Polling phase");
1070 SYSCTL_ADD_ULONG(ctx
, parent
, OID_AUTO
, "suspect", CTLFLAG_RW
,
1071 &io_ctx
->suspect
, "Suspected events");
1073 SYSCTL_ADD_ULONG(ctx
, parent
, OID_AUTO
, "stalled", CTLFLAG_RW
,
1074 &io_ctx
->stalled
, "Potential stalls");
1076 SYSCTL_ADD_ULONG(ctx
, parent
, OID_AUTO
, "short_ticks", CTLFLAG_RW
,
1077 &io_ctx
->short_ticks
,
1078 "Hardclock ticks shorter than they should be");
1080 SYSCTL_ADD_ULONG(ctx
, parent
, OID_AUTO
, "lost_polls", CTLFLAG_RW
,
1081 &io_ctx
->lost_polls
,
1082 "How many times we would have lost a poll tick");
1084 SYSCTL_ADD_UINT(ctx
, parent
, OID_AUTO
, "pending_polls", CTLFLAG_RD
,
1085 &io_ctx
->pending_polls
, 0, "Do we need to poll again");
1087 SYSCTL_ADD_UINT(ctx
, parent
, OID_AUTO
, "handlers", CTLFLAG_RD
,
1088 &io_ctx
->poll_handlers
, 0, "Number of registered poll handlers");
1092 sysctl_burstmax_handler(netmsg_t nmsg
)
1094 struct iopoll_sysctl_netmsg
*msg
= (struct iopoll_sysctl_netmsg
*)nmsg
;
1095 struct iopoll_ctx
*io_ctx
;
1098 KKASSERT(&curthread
->td_msgport
== netisr_cpuport(io_ctx
->poll_cpuid
));
1100 io_ctx
->poll_burst_max
= nmsg
->lmsg
.u
.ms_result
;
1101 if (io_ctx
->poll_each_burst
> io_ctx
->poll_burst_max
)
1102 io_ctx
->poll_each_burst
= io_ctx
->poll_burst_max
;
1103 if (io_ctx
->poll_burst
> io_ctx
->poll_burst_max
)
1104 io_ctx
->poll_burst
= io_ctx
->poll_burst_max
;
1105 if (io_ctx
->residual_burst
> io_ctx
->poll_burst_max
)
1106 io_ctx
->residual_burst
= io_ctx
->poll_burst_max
;
1108 lwkt_replymsg(&nmsg
->lmsg
, 0);
1112 sysctl_burstmax(SYSCTL_HANDLER_ARGS
)
1114 struct iopoll_ctx
*io_ctx
= arg1
;
1115 struct iopoll_sysctl_netmsg msg
;
1119 burst_max
= io_ctx
->poll_burst_max
;
1120 error
= sysctl_handle_int(oidp
, &burst_max
, 0, req
);
1121 if (error
|| req
->newptr
== NULL
)
1123 if (burst_max
< MIN_IOPOLL_BURST_MAX
)
1124 burst_max
= MIN_IOPOLL_BURST_MAX
;
1125 else if (burst_max
> MAX_IOPOLL_BURST_MAX
)
1126 burst_max
= MAX_IOPOLL_BURST_MAX
;
1128 netmsg_init(&msg
.base
, NULL
, &curthread
->td_msgport
,
1129 0, sysctl_burstmax_handler
);
1130 msg
.base
.lmsg
.u
.ms_result
= burst_max
;
1133 return lwkt_domsg(netisr_cpuport(io_ctx
->poll_cpuid
),
1138 sysctl_eachburst_handler(netmsg_t nmsg
)
1140 struct iopoll_sysctl_netmsg
*msg
= (struct iopoll_sysctl_netmsg
*)nmsg
;
1141 struct iopoll_ctx
*io_ctx
;
1142 uint32_t each_burst
;
1145 KKASSERT(&curthread
->td_msgport
== netisr_cpuport(io_ctx
->poll_cpuid
));
1147 each_burst
= nmsg
->lmsg
.u
.ms_result
;
1148 if (each_burst
> io_ctx
->poll_burst_max
)
1149 each_burst
= io_ctx
->poll_burst_max
;
1150 else if (each_burst
< 1)
1152 io_ctx
->poll_each_burst
= each_burst
;
1154 lwkt_replymsg(&nmsg
->lmsg
, 0);
1158 sysctl_eachburst(SYSCTL_HANDLER_ARGS
)
1160 struct iopoll_ctx
*io_ctx
= arg1
;
1161 struct iopoll_sysctl_netmsg msg
;
1162 uint32_t each_burst
;
1165 each_burst
= io_ctx
->poll_each_burst
;
1166 error
= sysctl_handle_int(oidp
, &each_burst
, 0, req
);
1167 if (error
|| req
->newptr
== NULL
)
1170 netmsg_init(&msg
.base
, NULL
, &curthread
->td_msgport
,
1171 0, sysctl_eachburst_handler
);
1172 msg
.base
.lmsg
.u
.ms_result
= each_burst
;
1175 return lwkt_domsg(netisr_cpuport(io_ctx
->poll_cpuid
),
1180 iopoll_register(struct ifnet
*ifp
, struct iopoll_ctx
*io_ctx
,
1181 const struct ifpoll_io
*io_rec
)
1185 KKASSERT(&curthread
->td_msgport
== netisr_cpuport(io_ctx
->poll_cpuid
));
1187 if (io_rec
->poll_func
== NULL
)
1191 * Check if there is room.
1193 if (io_ctx
->poll_handlers
>= IFPOLL_LIST_LEN
) {
1195 * List full, cannot register more entries.
1196 * This should never happen; if it does, it is probably a
1197 * broken driver trying to register multiple times. Checking
1198 * this at runtime is expensive, and won't solve the problem
1199 * anyways, so just report a few times and then give up.
1201 static int verbose
= 10; /* XXX */
1203 kprintf("io poll handlers list full, "
1204 "maybe a broken driver ?\n");
1209 struct iopoll_rec
*rec
= &io_ctx
->pr
[io_ctx
->poll_handlers
];
1212 rec
->serializer
= io_rec
->serializer
;
1213 rec
->arg
= io_rec
->arg
;
1214 rec
->poll_func
= io_rec
->poll_func
;
1216 io_ctx
->poll_handlers
++;
1223 iopoll_deregister(struct ifnet
*ifp
, struct iopoll_ctx
*io_ctx
)
1227 KKASSERT(&curthread
->td_msgport
== netisr_cpuport(io_ctx
->poll_cpuid
));
1229 for (i
= 0; i
< io_ctx
->poll_handlers
; ++i
) {
1230 if (io_ctx
->pr
[i
].ifp
== ifp
) /* Found it */
1233 if (i
== io_ctx
->poll_handlers
) {
1236 io_ctx
->poll_handlers
--;
1237 if (i
< io_ctx
->poll_handlers
) {
1238 /* Last entry replaces this one. */
1239 io_ctx
->pr
[i
] = io_ctx
->pr
[io_ctx
->poll_handlers
];
1242 if (io_ctx
->poll_handlers
== 0)
1243 iopoll_reset_state(io_ctx
);
1250 poll_comm_init(int cpuid
)
1252 struct poll_comm
*comm
;
1255 comm
= kmalloc_cachealign(sizeof(*comm
), M_DEVBUF
, M_WAITOK
| M_ZERO
);
1257 if (ifpoll_stfrac
< 1)
1258 ifpoll_stfrac
= IFPOLL_STFRAC_DEFAULT
;
1259 if (ifpoll_txfrac
< 1)
1260 ifpoll_txfrac
= IFPOLL_TXFRAC_DEFAULT
;
1262 comm
->poll_cpuid
= cpuid
;
1263 comm
->pollhz
= poll_comm_pollhz_div(comm
, ifpoll_pollhz
);
1264 comm
->poll_stfrac
= ifpoll_stfrac
- 1;
1265 comm
->poll_txfrac
= ifpoll_txfrac
- 1;
1267 ksnprintf(cpuid_str
, sizeof(cpuid_str
), "%d", cpuid
);
1269 sysctl_ctx_init(&comm
->sysctl_ctx
);
1270 comm
->sysctl_tree
= SYSCTL_ADD_NODE(&comm
->sysctl_ctx
,
1271 SYSCTL_STATIC_CHILDREN(_net_ifpoll
),
1272 OID_AUTO
, cpuid_str
, CTLFLAG_RD
, 0, "");
1274 SYSCTL_ADD_PROC(&comm
->sysctl_ctx
, SYSCTL_CHILDREN(comm
->sysctl_tree
),
1275 OID_AUTO
, "pollhz", CTLTYPE_INT
| CTLFLAG_RW
,
1276 comm
, 0, sysctl_pollhz
,
1277 "I", "Device polling frequency");
1280 SYSCTL_ADD_PROC(&comm
->sysctl_ctx
,
1281 SYSCTL_CHILDREN(comm
->sysctl_tree
),
1282 OID_AUTO
, "status_frac",
1283 CTLTYPE_INT
| CTLFLAG_RW
,
1284 comm
, 0, sysctl_stfrac
,
1285 "I", "# of cycles before status is polled");
1287 SYSCTL_ADD_PROC(&comm
->sysctl_ctx
, SYSCTL_CHILDREN(comm
->sysctl_tree
),
1288 OID_AUTO
, "tx_frac", CTLTYPE_INT
| CTLFLAG_RW
,
1289 comm
, 0, sysctl_txfrac
,
1290 "I", "# of cycles before TX is polled");
1292 poll_common
[cpuid
] = comm
;
1296 poll_comm_start(int cpuid
)
1298 struct poll_comm
*comm
= poll_common
[cpuid
];
1299 systimer_func_t func
;
1302 * Initialize systimer
1305 func
= poll_comm_systimer0
;
1307 func
= poll_comm_systimer
;
1308 systimer_init_periodic_nq(&comm
->pollclock
, func
, comm
, 1);
1312 _poll_comm_systimer(struct poll_comm
*comm
)
1314 iopoll_clock(rxpoll_context
[comm
->poll_cpuid
]);
1315 if (comm
->txfrac_count
-- == 0) {
1316 comm
->txfrac_count
= comm
->poll_txfrac
;
1317 iopoll_clock(txpoll_context
[comm
->poll_cpuid
]);
1322 poll_comm_systimer0(systimer_t info
, int in_ipi __unused
,
1323 struct intrframe
*frame __unused
)
1325 struct poll_comm
*comm
= info
->data
;
1326 globaldata_t gd
= mycpu
;
1328 KKASSERT(comm
->poll_cpuid
== gd
->gd_cpuid
&& gd
->gd_cpuid
== 0);
1332 if (comm
->stfrac_count
-- == 0) {
1333 comm
->stfrac_count
= comm
->poll_stfrac
;
1334 stpoll_clock(&stpoll_context
);
1336 _poll_comm_systimer(comm
);
1342 poll_comm_systimer(systimer_t info
, int in_ipi __unused
,
1343 struct intrframe
*frame __unused
)
1345 struct poll_comm
*comm
= info
->data
;
1346 globaldata_t gd
= mycpu
;
1348 KKASSERT(comm
->poll_cpuid
== gd
->gd_cpuid
&& gd
->gd_cpuid
!= 0);
1351 _poll_comm_systimer(comm
);
1356 poll_comm_adjust_pollhz(struct poll_comm
*comm
)
1361 KKASSERT(&curthread
->td_msgport
== netisr_cpuport(comm
->poll_cpuid
));
1364 * If there is no polling handler registered, set systimer
1365 * frequency to the lowest value. Polling systimer frequency
1366 * will be adjusted to the requested value, once there are
1367 * registered handlers.
1369 handlers
= rxpoll_context
[mycpuid
]->poll_handlers
+
1370 txpoll_context
[mycpuid
]->poll_handlers
;
1371 if (comm
->poll_cpuid
== 0)
1372 handlers
+= stpoll_context
.poll_handlers
;
1374 pollhz
= comm
->pollhz
;
1375 systimer_adjust_periodic(&comm
->pollclock
, pollhz
);
1379 sysctl_pollhz(SYSCTL_HANDLER_ARGS
)
1381 struct poll_comm
*comm
= arg1
;
1382 struct netmsg_base nmsg
;
1385 phz
= poll_comm_pollhz_conv(comm
, comm
->pollhz
);
1386 error
= sysctl_handle_int(oidp
, &phz
, 0, req
);
1387 if (error
|| req
->newptr
== NULL
)
1391 else if (phz
> IFPOLL_FREQ_MAX
)
1392 phz
= IFPOLL_FREQ_MAX
;
1394 netmsg_init(&nmsg
, NULL
, &curthread
->td_msgport
,
1395 0, sysctl_pollhz_handler
);
1396 nmsg
.lmsg
.u
.ms_result
= phz
;
1398 return lwkt_domsg(netisr_cpuport(comm
->poll_cpuid
), &nmsg
.lmsg
, 0);
1402 sysctl_pollhz_handler(netmsg_t nmsg
)
1404 struct poll_comm
*comm
= poll_common
[mycpuid
];
1406 KKASSERT(&curthread
->td_msgport
== netisr_cpuport(comm
->poll_cpuid
));
1408 /* Save polling frequency */
1409 comm
->pollhz
= poll_comm_pollhz_div(comm
, nmsg
->lmsg
.u
.ms_result
);
1412 * Adjust cached pollhz
1414 rxpoll_context
[mycpuid
]->pollhz
= comm
->pollhz
;
1415 txpoll_context
[mycpuid
]->pollhz
=
1416 comm
->pollhz
/ (comm
->poll_txfrac
+ 1);
1419 * Adjust polling frequency
1421 poll_comm_adjust_pollhz(comm
);
1423 lwkt_replymsg(&nmsg
->lmsg
, 0);
1427 sysctl_stfrac(SYSCTL_HANDLER_ARGS
)
1429 struct poll_comm
*comm
= arg1
;
1430 struct netmsg_base nmsg
;
1433 KKASSERT(comm
->poll_cpuid
== 0);
1435 stfrac
= comm
->poll_stfrac
+ 1;
1436 error
= sysctl_handle_int(oidp
, &stfrac
, 0, req
);
1437 if (error
|| req
->newptr
== NULL
)
1442 netmsg_init(&nmsg
, NULL
, &curthread
->td_msgport
,
1443 0, sysctl_stfrac_handler
);
1444 nmsg
.lmsg
.u
.ms_result
= stfrac
- 1;
1446 return lwkt_domsg(netisr_cpuport(comm
->poll_cpuid
), &nmsg
.lmsg
, 0);
1450 sysctl_stfrac_handler(netmsg_t nmsg
)
1452 struct poll_comm
*comm
= poll_common
[mycpuid
];
1453 int stfrac
= nmsg
->lmsg
.u
.ms_result
;
1455 KKASSERT(&curthread
->td_msgport
== netisr_cpuport(comm
->poll_cpuid
));
1458 comm
->poll_stfrac
= stfrac
;
1459 if (comm
->stfrac_count
> comm
->poll_stfrac
)
1460 comm
->stfrac_count
= comm
->poll_stfrac
;
1463 lwkt_replymsg(&nmsg
->lmsg
, 0);
1467 sysctl_txfrac(SYSCTL_HANDLER_ARGS
)
1469 struct poll_comm
*comm
= arg1
;
1470 struct netmsg_base nmsg
;
1473 txfrac
= comm
->poll_txfrac
+ 1;
1474 error
= sysctl_handle_int(oidp
, &txfrac
, 0, req
);
1475 if (error
|| req
->newptr
== NULL
)
1480 netmsg_init(&nmsg
, NULL
, &curthread
->td_msgport
,
1481 0, sysctl_txfrac_handler
);
1482 nmsg
.lmsg
.u
.ms_result
= txfrac
- 1;
1484 return lwkt_domsg(netisr_cpuport(comm
->poll_cpuid
), &nmsg
.lmsg
, 0);
1488 sysctl_txfrac_handler(netmsg_t nmsg
)
1490 struct poll_comm
*comm
= poll_common
[mycpuid
];
1491 int txfrac
= nmsg
->lmsg
.u
.ms_result
;
1493 KKASSERT(&curthread
->td_msgport
== netisr_cpuport(comm
->poll_cpuid
));
1496 comm
->poll_txfrac
= txfrac
;
1497 if (comm
->txfrac_count
> comm
->poll_txfrac
)
1498 comm
->txfrac_count
= comm
->poll_txfrac
;
1501 lwkt_replymsg(&nmsg
->lmsg
, 0);
1505 ifpoll_compat_setup(struct ifpoll_compat
*cp
,
1506 struct sysctl_ctx_list
*sysctl_ctx
,
1507 struct sysctl_oid
*sysctl_tree
,
1508 int unit
, struct lwkt_serialize
*slz
)
1510 cp
->ifpc_stcount
= 0;
1511 cp
->ifpc_stfrac
= ((poll_common
[0]->poll_stfrac
+ 1) *
1512 howmany(IOPOLL_BURST_MAX
, IOPOLL_EACH_BURST
)) - 1;
1514 cp
->ifpc_cpuid
= unit
% ncpus2
;
1515 cp
->ifpc_serializer
= slz
;
1517 if (sysctl_ctx
!= NULL
&& sysctl_tree
!= NULL
) {
1518 SYSCTL_ADD_PROC(sysctl_ctx
, SYSCTL_CHILDREN(sysctl_tree
),
1519 OID_AUTO
, "npoll_stfrac", CTLTYPE_INT
| CTLFLAG_RW
,
1520 cp
, 0, sysctl_compat_npoll_stfrac
, "I",
1521 "polling status frac");
1522 SYSCTL_ADD_PROC(sysctl_ctx
, SYSCTL_CHILDREN(sysctl_tree
),
1523 OID_AUTO
, "npoll_cpuid", CTLTYPE_INT
| CTLFLAG_RW
,
1524 cp
, 0, sysctl_compat_npoll_cpuid
, "I",
1530 sysctl_compat_npoll_stfrac(SYSCTL_HANDLER_ARGS
)
1532 struct ifpoll_compat
*cp
= arg1
;
1533 int error
= 0, stfrac
;
1535 lwkt_serialize_enter(cp
->ifpc_serializer
);
1537 stfrac
= cp
->ifpc_stfrac
+ 1;
1538 error
= sysctl_handle_int(oidp
, &stfrac
, 0, req
);
1539 if (!error
&& req
->newptr
!= NULL
) {
1543 cp
->ifpc_stfrac
= stfrac
- 1;
1544 if (cp
->ifpc_stcount
> cp
->ifpc_stfrac
)
1545 cp
->ifpc_stcount
= cp
->ifpc_stfrac
;
1549 lwkt_serialize_exit(cp
->ifpc_serializer
);
1554 sysctl_compat_npoll_cpuid(SYSCTL_HANDLER_ARGS
)
1556 struct ifpoll_compat
*cp
= arg1
;
1557 int error
= 0, cpuid
;
1559 lwkt_serialize_enter(cp
->ifpc_serializer
);
1561 cpuid
= cp
->ifpc_cpuid
;
1562 error
= sysctl_handle_int(oidp
, &cpuid
, 0, req
);
1563 if (!error
&& req
->newptr
!= NULL
) {
1564 if (cpuid
< 0 || cpuid
>= ncpus2
)
1567 cp
->ifpc_cpuid
= cpuid
;
1570 lwkt_serialize_exit(cp
->ifpc_serializer
);