2 * Copyright (c) 2004 Jeffrey M. Hsu. All rights reserved.
3 * Copyright (c) 2004 The DragonFly Project. All rights reserved.
5 * This code is derived from software contributed to The DragonFly Project
8 * Redistribution and use in source and binary forms, with or without
9 * modification, are permitted provided that the following conditions
11 * 1. Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * 2. Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in the
15 * documentation and/or other materials provided with the distribution.
16 * 3. Neither the name of The DragonFly Project nor the names of its
17 * contributors may be used to endorse or promote products derived
18 * from this software without specific, prior written permission.
20 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
21 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
22 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
23 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
24 * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
25 * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
26 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
27 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
28 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
29 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
30 * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
35 * Copyright (c) 1982, 1986, 1991, 1993, 1995
36 * The Regents of the University of California. All rights reserved.
38 * Redistribution and use in source and binary forms, with or without
39 * modification, are permitted provided that the following conditions
41 * 1. Redistributions of source code must retain the above copyright
42 * notice, this list of conditions and the following disclaimer.
43 * 2. Redistributions in binary form must reproduce the above copyright
44 * notice, this list of conditions and the following disclaimer in the
45 * documentation and/or other materials provided with the distribution.
46 * 3. Neither the name of the University nor the names of its contributors
47 * may be used to endorse or promote products derived from this software
48 * without specific prior written permission.
50 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
51 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
52 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
53 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
54 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
55 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
56 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
57 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
58 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
59 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
62 * @(#)in_pcb.c 8.4 (Berkeley) 5/24/95
63 * $FreeBSD: src/sys/netinet/in_pcb.c,v 1.59.2.27 2004/01/02 04:06:42 ambrisko Exp $
66 #include "opt_ipsec.h"
67 #include "opt_inet6.h"
69 #include <sys/param.h>
70 #include <sys/systm.h>
71 #include <sys/malloc.h>
73 #include <sys/domain.h>
74 #include <sys/protosw.h>
75 #include <sys/socket.h>
76 #include <sys/socketvar.h>
80 #include <sys/kernel.h>
81 #include <sys/sysctl.h>
83 #include <sys/thread2.h>
84 #include <sys/socketvar2.h>
85 #include <sys/msgport2.h>
87 #include <machine/limits.h>
90 #include <net/if_types.h>
91 #include <net/route.h>
92 #include <net/netisr2.h>
93 #include <net/toeplitz2.h>
95 #include <netinet/in.h>
96 #include <netinet/in_pcb.h>
97 #include <netinet/in_var.h>
98 #include <netinet/ip_var.h>
100 #include <netinet/ip6.h>
101 #include <netinet6/ip6_var.h>
105 #include <netinet6/ipsec.h>
106 #include <netproto/key/key.h>
107 #include <netproto/ipsec/esp_var.h>
111 #if defined(IPSEC) || defined(IPSEC_ESP)
112 #error "Bad idea: don't compile with both IPSEC and FAST_IPSEC!"
115 #include <netproto/ipsec/ipsec.h>
116 #include <netproto/ipsec/key.h>
118 #endif /* FAST_IPSEC */
120 #define INP_LOCALGROUP_SIZMIN 8
121 #define INP_LOCALGROUP_SIZMAX 256
123 static struct inpcb
*in_pcblookup_local(struct inpcbporthead
*porthash
,
124 struct in_addr laddr
, u_int lport_arg
, int wild_okay
,
127 struct in_addr zeroin_addr
;
130 * These configure the range of local port addresses assigned to
131 * "unspecified" outgoing connections/packets/whatever.
133 int ipport_lowfirstauto
= IPPORT_RESERVED
- 1; /* 1023 */
134 int ipport_lowlastauto
= IPPORT_RESERVEDSTART
; /* 600 */
136 int ipport_firstauto
= IPPORT_RESERVED
; /* 1024 */
137 int ipport_lastauto
= IPPORT_USERRESERVED
; /* 5000 */
139 int ipport_hifirstauto
= IPPORT_HIFIRSTAUTO
; /* 49152 */
140 int ipport_hilastauto
= IPPORT_HILASTAUTO
; /* 65535 */
142 #define RANGECHK(var, min, max) \
143 if ((var) < (min)) { (var) = (min); } \
144 else if ((var) > (max)) { (var) = (max); }
146 int udpencap_enable
= 1; /* enabled by default */
147 int udpencap_port
= 4500; /* triggers decapsulation */
150 * Per-netisr inpcb markers.
151 * NOTE: they should only be used in netisrs.
153 static struct inpcb
*in_pcbmarkers
;
154 static struct inpcontainer
*in_pcbcontainer_markers
;
157 sysctl_net_ipport_check(SYSCTL_HANDLER_ARGS
)
161 error
= sysctl_handle_int(oidp
, oidp
->oid_arg1
, oidp
->oid_arg2
, req
);
163 RANGECHK(ipport_lowfirstauto
, 1, IPPORT_RESERVED
- 1);
164 RANGECHK(ipport_lowlastauto
, 1, IPPORT_RESERVED
- 1);
166 RANGECHK(ipport_firstauto
, IPPORT_RESERVED
, USHRT_MAX
);
167 RANGECHK(ipport_lastauto
, IPPORT_RESERVED
, USHRT_MAX
);
169 RANGECHK(ipport_hifirstauto
, IPPORT_RESERVED
, USHRT_MAX
);
170 RANGECHK(ipport_hilastauto
, IPPORT_RESERVED
, USHRT_MAX
);
177 SYSCTL_NODE(_net_inet_ip
, IPPROTO_IP
, portrange
, CTLFLAG_RW
, 0, "IP Ports");
179 SYSCTL_PROC(_net_inet_ip_portrange
, OID_AUTO
, lowfirst
, CTLTYPE_INT
|CTLFLAG_RW
,
180 &ipport_lowfirstauto
, 0, &sysctl_net_ipport_check
, "I", "");
181 SYSCTL_PROC(_net_inet_ip_portrange
, OID_AUTO
, lowlast
, CTLTYPE_INT
|CTLFLAG_RW
,
182 &ipport_lowlastauto
, 0, &sysctl_net_ipport_check
, "I", "");
183 SYSCTL_PROC(_net_inet_ip_portrange
, OID_AUTO
, first
, CTLTYPE_INT
|CTLFLAG_RW
,
184 &ipport_firstauto
, 0, &sysctl_net_ipport_check
, "I", "");
185 SYSCTL_PROC(_net_inet_ip_portrange
, OID_AUTO
, last
, CTLTYPE_INT
|CTLFLAG_RW
,
186 &ipport_lastauto
, 0, &sysctl_net_ipport_check
, "I", "");
187 SYSCTL_PROC(_net_inet_ip_portrange
, OID_AUTO
, hifirst
, CTLTYPE_INT
|CTLFLAG_RW
,
188 &ipport_hifirstauto
, 0, &sysctl_net_ipport_check
, "I", "");
189 SYSCTL_PROC(_net_inet_ip_portrange
, OID_AUTO
, hilast
, CTLTYPE_INT
|CTLFLAG_RW
,
190 &ipport_hilastauto
, 0, &sysctl_net_ipport_check
, "I", "");
192 static int ip_porthash_trycount
= 15;
193 SYSCTL_INT(_net_inet_ip
, OID_AUTO
, porthash_trycount
, CTLFLAG_RW
,
194 &ip_porthash_trycount
, 0,
195 "Number of tries to find local port matching hash of 4-tuple");
198 * in_pcb.c: manage the Protocol Control Blocks.
200 * NOTE: It is assumed that most of these functions will be called from
201 * a critical section. XXX - There are, unfortunately, a few exceptions
202 * to this rule that should be fixed.
204 * NOTE: The caller should initialize the cpu field to the cpu running the
205 * protocol stack associated with this inpcbinfo.
209 in_pcbinfo_init(struct inpcbinfo
*pcbinfo
, int cpu
, boolean_t shared
)
211 KASSERT(cpu
>= 0 && cpu
< netisr_ncpus
, ("invalid cpu%d", cpu
));
214 LIST_INIT(&pcbinfo
->pcblisthead
);
215 pcbinfo
->portsave
= kmalloc(sizeof(*pcbinfo
->portsave
), M_PCB
,
219 pcbinfo
->infotoken
= kmalloc(sizeof(struct lwkt_token
),
221 lwkt_token_init(pcbinfo
->infotoken
, "infotoken");
223 pcbinfo
->infotoken
= NULL
;
228 in_pcbportinfo_set(struct inpcbinfo
*pcbinfo
, struct inpcbportinfo
*portinfo
,
232 KASSERT(portinfo_cnt
> 0, ("invalid portinfo_cnt %d", portinfo_cnt
));
233 pcbinfo
->portinfo
= portinfo
;
234 pcbinfo
->portinfo_cnt
= portinfo_cnt
;
237 struct baddynamicports baddynamicports
;
240 * Check if the specified port is invalid for dynamic allocation.
243 in_baddynamic(u_int16_t port
, u_int16_t proto
)
247 return (DP_ISSET(baddynamicports
.tcp
, port
));
250 /* Cannot preset this as it is a sysctl */
251 if (port
== udpencap_port
)
254 return (DP_ISSET(baddynamicports
.udp
, port
));
261 in_pcbonlist(struct inpcb
*inp
)
263 struct inpcbinfo
*pcbinfo
= inp
->inp_pcbinfo
;
265 KASSERT(&curthread
->td_msgport
== netisr_cpuport(pcbinfo
->cpu
),
266 ("not in the correct netisr"));
267 KASSERT((inp
->inp_flags
& INP_ONLIST
) == 0, ("already on pcblist"));
268 inp
->inp_flags
|= INP_ONLIST
;
270 GET_PCBINFO_TOKEN(pcbinfo
);
271 LIST_INSERT_HEAD(&pcbinfo
->pcblisthead
, inp
, inp_list
);
272 pcbinfo
->ipi_count
++;
273 REL_PCBINFO_TOKEN(pcbinfo
);
277 in_pcbofflist(struct inpcb
*inp
)
279 struct inpcbinfo
*pcbinfo
= inp
->inp_pcbinfo
;
281 KASSERT(&curthread
->td_msgport
== netisr_cpuport(pcbinfo
->cpu
),
282 ("not in the correct netisr"));
283 KASSERT(inp
->inp_flags
& INP_ONLIST
, ("not on pcblist"));
284 inp
->inp_flags
&= ~INP_ONLIST
;
286 GET_PCBINFO_TOKEN(pcbinfo
);
287 LIST_REMOVE(inp
, inp_list
);
288 KASSERT(pcbinfo
->ipi_count
> 0,
289 ("invalid inpcb count %d", pcbinfo
->ipi_count
));
290 pcbinfo
->ipi_count
--;
291 REL_PCBINFO_TOKEN(pcbinfo
);
295 * Allocate a PCB and associate it with the socket.
298 in_pcballoc(struct socket
*so
, struct inpcbinfo
*pcbinfo
)
305 inp
= kmalloc(pcbinfo
->ipi_size
, M_PCB
, M_WAITOK
|M_ZERO
|M_NULLOK
);
308 inp
->inp_lgrpindex
= -1;
309 inp
->inp_gencnt
= ++pcbinfo
->ipi_gencnt
;
310 inp
->inp_pcbinfo
= pcbinfo
;
311 inp
->inp_socket
= so
;
313 error
= ipsec_init_policy(so
, &inp
->inp_sp
);
320 if (INP_CHECK_SOCKAF(so
, AF_INET6
)) {
321 if (ip6_auto_flowlabel
)
322 inp
->inp_flags
|= IN6P_AUTOFLOWLABEL
;
323 inp
->inp_af
= AF_INET6
;
326 inp
->inp_af
= AF_INET
;
335 * Unlink a pcb with the intention of moving it to another cpu with a
336 * different pcbinfo. While unlinked nothing should attempt to dereference
337 * inp_pcbinfo, NULL it out so we assert if it does.
340 in_pcbunlink_flags(struct inpcb
*inp
, struct inpcbinfo
*pcbinfo
, int flags
)
342 KASSERT(inp
->inp_pcbinfo
== pcbinfo
, ("pcbinfo mismatch"));
343 KASSERT((inp
->inp_flags
& (flags
| INP_CONNECTED
)) == 0,
347 inp
->inp_pcbinfo
= NULL
;
351 in_pcbunlink(struct inpcb
*inp
, struct inpcbinfo
*pcbinfo
)
353 in_pcbunlink_flags(inp
, pcbinfo
, INP_WILDCARD
);
357 * Relink a pcb into a new pcbinfo.
360 in_pcblink_flags(struct inpcb
*inp
, struct inpcbinfo
*pcbinfo
, int flags
)
362 KASSERT(inp
->inp_pcbinfo
== NULL
, ("has pcbinfo"));
363 KASSERT((inp
->inp_flags
& (flags
| INP_CONNECTED
)) == 0,
366 inp
->inp_pcbinfo
= pcbinfo
;
371 in_pcblink(struct inpcb
*inp
, struct inpcbinfo
*pcbinfo
)
373 return in_pcblink_flags(inp
, pcbinfo
, INP_WILDCARD
);
377 in_pcbporthash_update(struct inpcbportinfo
*portinfo
,
378 struct inpcb
*inp
, u_short lport
, struct ucred
*cred
, int wild
)
380 struct inpcbporthead
*porthash
;
383 * This has to be atomic. If the porthash is shared across multiple
384 * protocol threads, e.g. tcp and udp, then the token must be held.
386 porthash
= in_pcbporthash_head(portinfo
, lport
);
387 GET_PORTHASH_TOKEN(porthash
);
389 if (in_pcblookup_local(porthash
, inp
->inp_laddr
, lport
,
390 wild
, cred
) != NULL
) {
391 REL_PORTHASH_TOKEN(porthash
);
394 inp
->inp_lport
= lport
;
395 in_pcbinsporthash(porthash
, inp
);
397 REL_PORTHASH_TOKEN(porthash
);
402 in_pcbsetlport(struct inpcb
*inp
, int wild
, struct ucred
*cred
)
404 struct inpcbinfo
*pcbinfo
= inp
->inp_pcbinfo
;
405 struct inpcbportinfo
*portinfo
;
406 u_short first
, last
, lport
, step
, first0
, last0
;
408 int portinfo_first
, portinfo_idx
;
411 inp
->inp_flags
|= INP_ANONPORT
;
413 step
= pcbinfo
->portinfo_cnt
;
414 portinfo_first
= mycpuid
% pcbinfo
->portinfo_cnt
;
415 portinfo_idx
= portinfo_first
;
417 if (inp
->inp_flags
& INP_HIGHPORT
) {
418 first0
= ipport_hifirstauto
; /* sysctl */
419 last0
= ipport_hilastauto
;
420 } else if (inp
->inp_flags
& INP_LOWPORT
) {
423 priv_check_cred(cred
, PRIV_NETINET_RESERVEDPORT
, 0))) {
424 inp
->inp_laddr
.s_addr
= INADDR_ANY
;
427 first0
= ipport_lowfirstauto
; /* 1023 */
428 last0
= ipport_lowlastauto
; /* 600 */
430 first0
= ipport_firstauto
; /* sysctl */
431 last0
= ipport_lastauto
;
433 if (first0
> last0
) {
438 KKASSERT(last0
>= first0
);
442 portinfo
= &pcbinfo
->portinfo
[portinfo_idx
];
447 * Simple check to ensure all ports are not used up causing
450 in_pcbportrange(&last
, &first
, portinfo
->offset
, step
);
451 lport
= last
- first
;
452 count
= lport
/ step
;
454 lport
= rounddown(cut
% lport
, step
) + first
;
455 KKASSERT(lport
% step
== portinfo
->offset
);
458 if (count
-- < 0) { /* completely used? */
459 error
= EADDRNOTAVAIL
;
463 if (__predict_false(lport
< first
|| lport
> last
)) {
465 KKASSERT(lport
% step
== portinfo
->offset
);
468 if (in_pcbporthash_update(portinfo
, inp
, htons(lport
),
475 KKASSERT(lport
% step
== portinfo
->offset
);
479 /* Try next portinfo */
481 portinfo_idx
%= pcbinfo
->portinfo_cnt
;
482 if (portinfo_idx
!= portinfo_first
)
484 inp
->inp_laddr
.s_addr
= INADDR_ANY
;
490 in_pcbbind(struct inpcb
*inp
, struct sockaddr
*nam
, struct thread
*td
)
492 struct socket
*so
= inp
->inp_socket
;
493 struct sockaddr_in jsin
;
494 struct ucred
*cred
= NULL
;
497 if (TAILQ_EMPTY(&in_ifaddrheads
[mycpuid
])) /* XXX broken! */
498 return (EADDRNOTAVAIL
);
499 if (inp
->inp_lport
!= 0 || inp
->inp_laddr
.s_addr
!= INADDR_ANY
)
500 return (EINVAL
); /* already bound */
502 if (!(so
->so_options
& (SO_REUSEADDR
|SO_REUSEPORT
)))
503 wild
= 1; /* neither SO_REUSEADDR nor SO_REUSEPORT is set */
505 cred
= td
->td_proc
->p_ucred
;
508 struct sockaddr_in
*sin
= (struct sockaddr_in
*)nam
;
509 struct inpcbinfo
*pcbinfo
;
510 struct inpcbportinfo
*portinfo
;
511 struct inpcbporthead
*porthash
;
513 u_short lport
, lport_ho
;
514 int reuseport
= (so
->so_options
& SO_REUSEPORT
);
517 if (nam
->sa_len
!= sizeof *sin
)
521 * We should check the family, but old programs
522 * incorrectly fail to initialize it.
524 if (sin
->sin_family
!= AF_INET
)
525 return (EAFNOSUPPORT
);
527 if (!prison_replace_wildcards(td
, nam
))
530 lport
= sin
->sin_port
;
531 if (IN_MULTICAST(ntohl(sin
->sin_addr
.s_addr
))) {
533 * Treat SO_REUSEADDR as SO_REUSEPORT for multicast;
534 * allow complete duplication of binding if
535 * SO_REUSEPORT is set, or if SO_REUSEADDR is set
536 * and a multicast address is bound on both
537 * new and duplicated sockets.
539 if (so
->so_options
& SO_REUSEADDR
)
540 reuseport
= SO_REUSEADDR
| SO_REUSEPORT
;
541 } else if (sin
->sin_addr
.s_addr
!= INADDR_ANY
) {
542 sin
->sin_port
= 0; /* yech... */
543 bzero(&sin
->sin_zero
, sizeof sin
->sin_zero
);
544 if (ifa_ifwithaddr((struct sockaddr
*)sin
) == NULL
)
545 return (EADDRNOTAVAIL
);
548 inp
->inp_laddr
= sin
->sin_addr
;
550 jsin
.sin_family
= AF_INET
;
551 jsin
.sin_addr
.s_addr
= inp
->inp_laddr
.s_addr
;
552 if (!prison_replace_wildcards(td
, (struct sockaddr
*)&jsin
)) {
553 inp
->inp_laddr
.s_addr
= INADDR_ANY
;
556 inp
->inp_laddr
.s_addr
= jsin
.sin_addr
.s_addr
;
559 /* Auto-select local port */
560 return in_pcbsetlport(inp
, wild
, cred
);
562 lport_ho
= ntohs(lport
);
565 if (lport_ho
< IPPORT_RESERVED
&& cred
&&
567 priv_check_cred(cred
, PRIV_NETINET_RESERVEDPORT
, 0))) {
568 inp
->inp_laddr
.s_addr
= INADDR_ANY
;
573 * Locate the proper portinfo based on lport
575 pcbinfo
= inp
->inp_pcbinfo
;
577 &pcbinfo
->portinfo
[lport_ho
% pcbinfo
->portinfo_cnt
];
578 KKASSERT((lport_ho
% pcbinfo
->portinfo_cnt
) ==
582 * This has to be atomic. If the porthash is shared across
583 * multiple protocol threads, e.g. tcp and udp then the token
586 porthash
= in_pcbporthash_head(portinfo
, lport
);
587 GET_PORTHASH_TOKEN(porthash
);
589 if (so
->so_cred
->cr_uid
!= 0 &&
590 !IN_MULTICAST(ntohl(sin
->sin_addr
.s_addr
))) {
591 t
= in_pcblookup_local(porthash
, sin
->sin_addr
, lport
,
592 INPLOOKUP_WILDCARD
, cred
);
594 (so
->so_cred
->cr_uid
!=
595 t
->inp_socket
->so_cred
->cr_uid
)) {
596 inp
->inp_laddr
.s_addr
= INADDR_ANY
;
601 if (cred
&& !prison_replace_wildcards(td
, nam
)) {
602 inp
->inp_laddr
.s_addr
= INADDR_ANY
;
603 error
= EADDRNOTAVAIL
;
606 t
= in_pcblookup_local(porthash
, sin
->sin_addr
, lport
,
608 if (t
&& !(reuseport
& t
->inp_socket
->so_options
)) {
609 inp
->inp_laddr
.s_addr
= INADDR_ANY
;
613 inp
->inp_lport
= lport
;
614 in_pcbinsporthash(porthash
, inp
);
617 REL_PORTHASH_TOKEN(porthash
);
620 jsin
.sin_family
= AF_INET
;
621 jsin
.sin_addr
.s_addr
= inp
->inp_laddr
.s_addr
;
622 if (!prison_replace_wildcards(td
, (struct sockaddr
*)&jsin
)) {
623 inp
->inp_laddr
.s_addr
= INADDR_ANY
;
626 inp
->inp_laddr
.s_addr
= jsin
.sin_addr
.s_addr
;
628 return in_pcbsetlport(inp
, wild
, cred
);
632 static struct inpcb
*
633 in_pcblookup_localremote(struct inpcbporthead
*porthash
, struct in_addr laddr
,
634 u_short lport
, struct in_addr faddr
, u_short fport
, struct ucred
*cred
)
637 struct inpcbport
*phd
;
638 struct inpcb
*match
= NULL
;
641 * If the porthashbase is shared across several cpus, it must
644 ASSERT_PORTHASH_TOKEN_HELD(porthash
);
647 * Best fit PCB lookup.
649 * First see if this local port is in use by looking on the
652 LIST_FOREACH(phd
, porthash
, phd_hash
) {
653 if (phd
->phd_port
== lport
)
657 LIST_FOREACH(inp
, &phd
->phd_pcblist
, inp_portlist
) {
659 if (!INP_ISIPV4(inp
))
662 if (inp
->inp_laddr
.s_addr
!= INADDR_ANY
&&
663 inp
->inp_laddr
.s_addr
!= laddr
.s_addr
)
666 if (inp
->inp_faddr
.s_addr
!= INADDR_ANY
&&
667 inp
->inp_faddr
.s_addr
!= faddr
.s_addr
)
670 if (inp
->inp_fport
!= 0 && inp
->inp_fport
!= fport
)
675 inp
->inp_socket
->so_cred
->cr_prison
) {
685 in_pcbporthash_update4(struct inpcbportinfo
*portinfo
,
686 struct inpcb
*inp
, u_short lport
, const struct sockaddr_in
*sin
,
689 struct inpcbporthead
*porthash
;
692 * This has to be atomic. If the porthash is shared across multiple
693 * protocol threads, e.g. tcp and udp, then the token must be held.
695 porthash
= in_pcbporthash_head(portinfo
, lport
);
696 GET_PORTHASH_TOKEN(porthash
);
698 if (in_pcblookup_localremote(porthash
, inp
->inp_laddr
,
699 lport
, sin
->sin_addr
, sin
->sin_port
, cred
) != NULL
) {
700 REL_PORTHASH_TOKEN(porthash
);
703 inp
->inp_lport
= lport
;
704 in_pcbinsporthash(porthash
, inp
);
706 REL_PORTHASH_TOKEN(porthash
);
711 in_pcbbind_remote(struct inpcb
*inp
, const struct sockaddr
*remote
,
714 struct proc
*p
= td
->td_proc
;
715 const struct sockaddr_in
*sin
= (const struct sockaddr_in
*)remote
;
716 struct sockaddr_in jsin
;
717 struct inpcbinfo
*pcbinfo
= inp
->inp_pcbinfo
;
718 struct inpcbportinfo
*portinfo
;
719 struct ucred
*cred
= NULL
;
720 u_short first
, last
, lport
, step
, first0
, last0
;
721 int count
, error
, selfconn
;
722 int portinfo_first
, portinfo_idx
;
723 int hash_cpu
, hash_count
, hash_count0
;
724 uint32_t hash_base
= 0, hash
, cut
;
726 if (TAILQ_EMPTY(&in_ifaddrheads
[mycpuid
])) /* XXX broken! */
727 return (EADDRNOTAVAIL
);
729 KKASSERT(inp
->inp_laddr
.s_addr
!= INADDR_ANY
);
730 if (inp
->inp_lport
!= 0)
731 return (EINVAL
); /* already bound */
736 jsin
.sin_family
= AF_INET
;
737 jsin
.sin_addr
.s_addr
= inp
->inp_laddr
.s_addr
;
738 if (!prison_replace_wildcards(td
, (struct sockaddr
*)&jsin
)) {
739 inp
->inp_laddr
.s_addr
= INADDR_ANY
;
742 inp
->inp_laddr
.s_addr
= jsin
.sin_addr
.s_addr
;
744 hash_count0
= ip_porthash_trycount
;
745 if (hash_count0
> 0) {
746 hash_base
= toeplitz_piecemeal_addr(sin
->sin_addr
.s_addr
) ^
747 toeplitz_piecemeal_addr(inp
->inp_laddr
.s_addr
) ^
748 toeplitz_piecemeal_port(sin
->sin_port
);
753 inp
->inp_flags
|= INP_ANONPORT
;
755 step
= pcbinfo
->portinfo_cnt
;
756 portinfo_first
= mycpuid
% pcbinfo
->portinfo_cnt
;
757 portinfo_idx
= portinfo_first
;
759 if (inp
->inp_flags
& INP_HIGHPORT
) {
760 first0
= ipport_hifirstauto
; /* sysctl */
761 last0
= ipport_hilastauto
;
762 } else if (inp
->inp_flags
& INP_LOWPORT
) {
765 priv_check_cred(cred
, PRIV_NETINET_RESERVEDPORT
, 0))) {
766 inp
->inp_laddr
.s_addr
= INADDR_ANY
;
769 first0
= ipport_lowfirstauto
; /* 1023 */
770 last0
= ipport_lowlastauto
; /* 600 */
772 first0
= ipport_firstauto
; /* sysctl */
773 last0
= ipport_lastauto
;
775 if (first0
> last0
) {
780 KKASSERT(last0
>= first0
);
784 hash_cpu
= portinfo_idx
% netisr_ncpus
;
785 portinfo
= &pcbinfo
->portinfo
[portinfo_idx
];
790 /* This could happen on loopback interface */
791 #define IS_SELFCONNECT(inp, lport, sin) \
792 (__predict_false((sin)->sin_port == (lport) && \
793 (sin)->sin_addr.s_addr == (inp)->inp_laddr.s_addr))
796 * Simple check to ensure all ports are not used up causing
799 hash_count
= hash_count0
;
801 in_pcbportrange(&last
, &first
, portinfo
->offset
, step
);
802 lport
= last
- first
;
803 count
= (lport
/ step
) + hash_count
;
805 lport
= rounddown(cut
% lport
, step
) + first
;
806 KKASSERT(lport
% step
== portinfo
->offset
);
811 if (count
-- < 0) { /* completely used? */
812 error
= EADDRNOTAVAIL
;
816 if (__predict_false(lport
< first
|| lport
> last
)) {
818 KKASSERT(lport
% step
== portinfo
->offset
);
821 lport_no
= htons(lport
);
822 if (IS_SELFCONNECT(inp
, lport_no
, sin
)) {
824 ++count
; /* don't count this try */
833 toeplitz_piecemeal_port(lport_no
);
834 if (netisr_hashcpu(hash
) != hash_cpu
&&
839 if (in_pcbporthash_update4(portinfo
,
840 inp
, lport_no
, sin
, cred
)) {
846 KKASSERT(lport
% step
== portinfo
->offset
);
849 #undef IS_SELFCONNECT
852 /* Try next portinfo */
854 portinfo_idx
%= pcbinfo
->portinfo_cnt
;
855 if (portinfo_idx
!= portinfo_first
)
857 inp
->inp_laddr
.s_addr
= INADDR_ANY
;
863 * Transform old in_pcbconnect() into an inner subroutine for new
864 * in_pcbconnect(): Do some validity-checking on the remote
865 * address (in mbuf 'nam') and then determine local host address
866 * (i.e., which interface) to use to access that remote host.
868 * This preserves definition of in_pcbconnect(), while supporting a
869 * slightly different version for T/TCP. (This is more than
870 * a bit of a kludge, but cleaning up the internal interfaces would
871 * have forced minor changes in every protocol).
874 in_pcbladdr_find(struct inpcb
*inp
, struct sockaddr
*nam
,
875 struct sockaddr_in
**plocal_sin
, struct thread
*td
, int find
)
877 struct in_ifaddr
*ia
;
878 struct ucred
*cred
= NULL
;
879 struct sockaddr_in
*sin
= (struct sockaddr_in
*)nam
;
880 struct sockaddr
*jsin
;
881 int jailed
= 0, alloc_route
= 0;
883 if (nam
->sa_len
!= sizeof *sin
)
885 if (sin
->sin_family
!= AF_INET
)
886 return (EAFNOSUPPORT
);
887 if (sin
->sin_port
== 0)
888 return (EADDRNOTAVAIL
);
889 if (td
&& td
->td_proc
&& td
->td_proc
->p_ucred
)
890 cred
= td
->td_proc
->p_ucred
;
891 if (cred
&& cred
->cr_prison
)
893 if (!TAILQ_EMPTY(&in_ifaddrheads
[mycpuid
])) {
894 ia
= TAILQ_FIRST(&in_ifaddrheads
[mycpuid
])->ia
;
896 * If the destination address is INADDR_ANY,
897 * use the primary local address.
898 * If the supplied address is INADDR_BROADCAST,
899 * and the primary interface supports broadcast,
900 * choose the broadcast address for that interface.
902 if (sin
->sin_addr
.s_addr
== INADDR_ANY
)
903 sin
->sin_addr
= IA_SIN(ia
)->sin_addr
;
904 else if (sin
->sin_addr
.s_addr
== (u_long
)INADDR_BROADCAST
&&
905 (ia
->ia_ifp
->if_flags
& IFF_BROADCAST
))
906 sin
->sin_addr
= satosin(&ia
->ia_broadaddr
)->sin_addr
;
913 * If route is known or can be allocated now,
914 * our src addr is taken from the i/f, else punt.
915 * Note that we should check the address family of the cached
916 * destination, in case of sharing the cache with IPv6.
918 ro
= &inp
->inp_route
;
920 (!(ro
->ro_rt
->rt_flags
& RTF_UP
) ||
921 ro
->ro_dst
.sa_family
!= AF_INET
||
922 satosin(&ro
->ro_dst
)->sin_addr
.s_addr
!=
923 sin
->sin_addr
.s_addr
||
924 inp
->inp_socket
->so_options
& SO_DONTROUTE
)) {
928 if (!(inp
->inp_socket
->so_options
& SO_DONTROUTE
) && /*XXX*/
929 (ro
->ro_rt
== NULL
||
930 ro
->ro_rt
->rt_ifp
== NULL
)) {
931 /* No route yet, so try to acquire one */
932 bzero(&ro
->ro_dst
, sizeof(struct sockaddr_in
));
933 ro
->ro_dst
.sa_family
= AF_INET
;
934 ro
->ro_dst
.sa_len
= sizeof(struct sockaddr_in
);
935 ((struct sockaddr_in
*) &ro
->ro_dst
)->sin_addr
=
941 * If we found a route, use the address
942 * corresponding to the outgoing interface
943 * unless it is the loopback (in case a route
944 * to our address on another net goes to loopback).
947 !(ro
->ro_rt
->rt_ifp
->if_flags
& IFF_LOOPBACK
)) {
949 if (jailed_ip(cred
->cr_prison
,
950 ro
->ro_rt
->rt_ifa
->ifa_addr
)) {
951 ia
= ifatoia(ro
->ro_rt
->rt_ifa
);
954 ia
= ifatoia(ro
->ro_rt
->rt_ifa
);
958 u_short fport
= sin
->sin_port
;
961 ia
= ifatoia(ifa_ifwithdstaddr(sintosa(sin
)));
962 if (ia
&& jailed
&& !jailed_ip(cred
->cr_prison
,
963 sintosa(&ia
->ia_addr
)))
966 ia
= ifatoia(ifa_ifwithnet(sintosa(sin
)));
967 if (ia
&& jailed
&& !jailed_ip(cred
->cr_prison
,
968 sintosa(&ia
->ia_addr
)))
970 sin
->sin_port
= fport
;
972 !TAILQ_EMPTY(&in_ifaddrheads
[mycpuid
]))
973 ia
= TAILQ_FIRST(&in_ifaddrheads
[mycpuid
])->ia
;
974 if (ia
&& jailed
&& !jailed_ip(cred
->cr_prison
,
975 sintosa(&ia
->ia_addr
)))
978 if (!jailed
&& ia
== NULL
)
982 * If the destination address is multicast and an outgoing
983 * interface has been set as a multicast option, use the
984 * address of that interface as our source address.
986 if (!jailed
&& IN_MULTICAST(ntohl(sin
->sin_addr
.s_addr
)) &&
987 inp
->inp_moptions
!= NULL
) {
988 struct ip_moptions
*imo
;
991 imo
= inp
->inp_moptions
;
992 if ((ifp
= imo
->imo_multicast_ifp
) != NULL
) {
993 struct in_ifaddr_container
*iac
;
997 &in_ifaddrheads
[mycpuid
], ia_link
) {
998 if (iac
->ia
->ia_ifp
== ifp
) {
1008 * Don't do pcblookup call here; return interface in plocal_sin
1009 * and exit to caller, that will do the lookup.
1011 if (ia
== NULL
&& jailed
) {
1012 if ((jsin
= prison_get_nonlocal(
1013 cred
->cr_prison
, AF_INET
, NULL
)) != NULL
||
1014 (jsin
= prison_get_local(
1015 cred
->cr_prison
, AF_INET
, NULL
)) != NULL
) {
1016 *plocal_sin
= satosin(jsin
);
1018 /* IPv6 only Jail */
1022 *plocal_sin
= &ia
->ia_addr
;
1028 in_pcbresetroute(inp
);
1029 return (EADDRNOTAVAIL
);
1033 in_pcbladdr(struct inpcb
*inp
, struct sockaddr
*nam
,
1034 struct sockaddr_in
**plocal_sin
, struct thread
*td
)
1036 return in_pcbladdr_find(inp
, nam
, plocal_sin
, td
,
1037 (inp
->inp_laddr
.s_addr
== INADDR_ANY
));
1042 * Connect from a socket to a specified address.
1043 * Both address and port must be specified in argument sin.
1044 * If don't have a local address for this socket yet,
1048 in_pcbconnect(struct inpcb
*inp
, struct sockaddr
*nam
, struct thread
*td
)
1050 struct sockaddr_in
*if_sin
;
1051 struct sockaddr_in
*sin
= (struct sockaddr_in
*)nam
;
1054 if_sin
= NULL
; /* avoid gcc warnings */
1056 /* Call inner routine to assign local interface address. */
1057 if ((error
= in_pcbladdr(inp
, nam
, &if_sin
, td
)) != 0)
1060 if (in_pcblookup_hash(inp
->inp_pcbinfo
, sin
->sin_addr
, sin
->sin_port
,
1061 inp
->inp_laddr
.s_addr
?
1062 inp
->inp_laddr
: if_sin
->sin_addr
,
1063 inp
->inp_lport
, FALSE
, NULL
) != NULL
) {
1064 return (EADDRINUSE
);
1066 if (inp
->inp_laddr
.s_addr
== INADDR_ANY
) {
1067 if (inp
->inp_lport
== 0) {
1068 error
= in_pcbbind(inp
, NULL
, td
);
1072 inp
->inp_laddr
= if_sin
->sin_addr
;
1074 inp
->inp_faddr
= sin
->sin_addr
;
1075 inp
->inp_fport
= sin
->sin_port
;
1076 in_pcbinsconnhash(inp
);
1081 in_pcbdisconnect(struct inpcb
*inp
)
1084 in_pcbremconnhash(inp
);
1085 inp
->inp_faddr
.s_addr
= INADDR_ANY
;
1090 in_pcbdetach(struct inpcb
*inp
)
1092 struct socket
*so
= inp
->inp_socket
;
1093 struct inpcbinfo
*ipi
= inp
->inp_pcbinfo
;
1096 ipsec4_delete_pcbpolicy(inp
);
1098 inp
->inp_gencnt
= ++ipi
->ipi_gencnt
;
1099 KKASSERT((so
->so_state
& SS_ASSERTINPROG
) == 0);
1100 in_pcbremlists(inp
);
1102 sofree(so
); /* remove pcb ref */
1103 if (inp
->inp_options
)
1104 m_free(inp
->inp_options
);
1105 if (inp
->inp_route
.ro_rt
)
1106 rtfree(inp
->inp_route
.ro_rt
);
1107 ip_freemoptions(inp
->inp_moptions
);
1112 * The socket may have an invalid PCB, i.e. NULL. For example, a TCP
1113 * socket received RST.
1116 in_setsockaddr(struct socket
*so
, struct sockaddr
**nam
)
1119 struct sockaddr_in
*sin
;
1121 KASSERT(curthread
->td_type
== TD_TYPE_NETISR
, ("not in netisr"));
1124 return (ECONNRESET
);
1126 sin
= kmalloc(sizeof *sin
, M_SONAME
, M_WAITOK
| M_ZERO
);
1127 sin
->sin_family
= AF_INET
;
1128 sin
->sin_len
= sizeof *sin
;
1129 sin
->sin_port
= inp
->inp_lport
;
1130 sin
->sin_addr
= inp
->inp_laddr
;
1132 *nam
= (struct sockaddr
*)sin
;
1137 in_setsockaddr_dispatch(netmsg_t msg
)
1141 error
= in_setsockaddr(msg
->base
.nm_so
, msg
->peeraddr
.nm_nam
);
1142 lwkt_replymsg(&msg
->lmsg
, error
);
1146 * The socket may have an invalid PCB, i.e. NULL. For example, a TCP
1147 * socket received RST.
1150 in_setpeeraddr(struct socket
*so
, struct sockaddr
**nam
)
1153 struct sockaddr_in
*sin
;
1155 KASSERT(curthread
->td_type
== TD_TYPE_NETISR
, ("not in netisr"));
1158 return (ECONNRESET
);
1160 sin
= kmalloc(sizeof *sin
, M_SONAME
, M_WAITOK
| M_ZERO
);
1161 sin
->sin_family
= AF_INET
;
1162 sin
->sin_len
= sizeof *sin
;
1163 sin
->sin_port
= inp
->inp_fport
;
1164 sin
->sin_addr
= inp
->inp_faddr
;
1166 *nam
= (struct sockaddr
*)sin
;
1171 in_setpeeraddr_dispatch(netmsg_t msg
)
1175 error
= in_setpeeraddr(msg
->base
.nm_so
, msg
->peeraddr
.nm_nam
);
1176 lwkt_replymsg(&msg
->lmsg
, error
);
1180 in_pcbnotifyall(struct inpcbinfo
*pcbinfo
, struct in_addr faddr
, int err
,
1181 inp_notify_t notify
)
1183 struct inpcb
*inp
, *marker
;
1185 KASSERT(&curthread
->td_msgport
== netisr_cpuport(pcbinfo
->cpu
),
1186 ("not in the correct netisr"));
1187 marker
= in_pcbmarker();
1191 * - If INP_PLACEMARKER is set we must ignore the rest of the
1192 * structure and skip it.
1193 * - It is safe to nuke inpcbs here, since we are in their own
1196 GET_PCBINFO_TOKEN(pcbinfo
);
1198 LIST_INSERT_HEAD(&pcbinfo
->pcblisthead
, marker
, inp_list
);
1199 while ((inp
= LIST_NEXT(marker
, inp_list
)) != NULL
) {
1200 LIST_REMOVE(marker
, inp_list
);
1201 LIST_INSERT_AFTER(inp
, marker
, inp_list
);
1203 if (inp
->inp_flags
& INP_PLACEMARKER
)
1206 if (!INP_ISIPV4(inp
))
1209 if (inp
->inp_faddr
.s_addr
!= faddr
.s_addr
||
1210 inp
->inp_socket
== NULL
)
1212 (*notify
)(inp
, err
); /* can remove inp from list! */
1214 LIST_REMOVE(marker
, inp_list
);
1216 REL_PCBINFO_TOKEN(pcbinfo
);
1220 in_pcbpurgeif0(struct inpcbinfo
*pcbinfo
, struct ifnet
*ifp
)
1222 struct inpcb
*inp
, *marker
;
1225 * We only need to make sure that we are in netisr0, where all
1226 * multicast operation happen. We could check inpcbinfo which
1227 * does not belong to netisr0 by holding the inpcbinfo's token.
1228 * In this case, the pcbinfo must be able to be shared, i.e.
1229 * pcbinfo->infotoken is not NULL.
1232 KASSERT(pcbinfo
->cpu
== 0 || pcbinfo
->infotoken
!= NULL
,
1233 ("pcbinfo could not be shared"));
1236 * Get a marker for the current netisr (netisr0).
1238 * It is possible that the multicast address deletion blocks,
1239 * which could cause temporary token releasing. So we use
1240 * inpcb marker here to get a coherent view of the inpcb list.
1242 * While, on the other hand, moptions are only added and deleted
1243 * in netisr0, so we would not see staled moption or miss moption
1244 * even if the token was released due to the blocking multicast
1247 marker
= in_pcbmarker();
1249 GET_PCBINFO_TOKEN(pcbinfo
);
1251 LIST_INSERT_HEAD(&pcbinfo
->pcblisthead
, marker
, inp_list
);
1252 while ((inp
= LIST_NEXT(marker
, inp_list
)) != NULL
) {
1253 struct ip_moptions
*imo
;
1255 LIST_REMOVE(marker
, inp_list
);
1256 LIST_INSERT_AFTER(inp
, marker
, inp_list
);
1258 if (inp
->inp_flags
& INP_PLACEMARKER
)
1260 imo
= inp
->inp_moptions
;
1261 if (INP_ISIPV4(inp
) && imo
!= NULL
) {
1265 * Unselect the outgoing interface if it is being
1268 if (imo
->imo_multicast_ifp
== ifp
)
1269 imo
->imo_multicast_ifp
= NULL
;
1272 * Drop multicast group membership if we joined
1273 * through the interface being detached.
1275 for (i
= 0, gap
= 0; i
< imo
->imo_num_memberships
;
1277 if (imo
->imo_membership
[i
]->inm_ifp
== ifp
) {
1280 * This could block and the pcbinfo
1281 * token could be passively released.
1283 in_delmulti(imo
->imo_membership
[i
]);
1285 } else if (gap
!= 0)
1286 imo
->imo_membership
[i
- gap
] =
1287 imo
->imo_membership
[i
];
1289 imo
->imo_num_memberships
-= gap
;
1292 LIST_REMOVE(marker
, inp_list
);
1294 REL_PCBINFO_TOKEN(pcbinfo
);
1298 * Check for alternatives when higher level complains
1299 * about service problems. For now, invalidate cached
1300 * routing information. If the route was created dynamically
1301 * (by a redirect), time to try a default gateway again.
1304 in_losing(struct inpcb
*inp
)
1307 struct rt_addrinfo rtinfo
;
1309 if ((rt
= inp
->inp_route
.ro_rt
)) {
1310 bzero(&rtinfo
, sizeof(struct rt_addrinfo
));
1311 rtinfo
.rti_info
[RTAX_DST
] = rt_key(rt
);
1312 rtinfo
.rti_info
[RTAX_GATEWAY
] = rt
->rt_gateway
;
1313 rtinfo
.rti_info
[RTAX_NETMASK
] = rt_mask(rt
);
1314 rtinfo
.rti_flags
= rt
->rt_flags
;
1315 rt_missmsg(RTM_LOSING
, &rtinfo
, rt
->rt_flags
, 0);
1316 if (rt
->rt_flags
& RTF_DYNAMIC
) {
1317 rtrequest(RTM_DELETE
, rt_key(rt
), rt
->rt_gateway
,
1318 rt_mask(rt
), rt
->rt_flags
, NULL
);
1320 inp
->inp_route
.ro_rt
= NULL
;
1323 * A new route can be allocated
1324 * the next time output is attempted.
1330 * After a routing change, flush old routing
1331 * and allocate a (hopefully) better one.
1334 in_rtchange(struct inpcb
*inp
, int err
)
1336 if (inp
->inp_route
.ro_rt
) {
1337 rtfree(inp
->inp_route
.ro_rt
);
1338 inp
->inp_route
.ro_rt
= NULL
;
1340 * A new route can be allocated the next time
1341 * output is attempted.
1347 * Lookup a PCB based on the local address and port.
1349 static struct inpcb
*
1350 in_pcblookup_local(struct inpcbporthead
*porthash
, struct in_addr laddr
,
1351 u_int lport_arg
, int wild_okay
, struct ucred
*cred
)
1354 int matchwild
= 3, wildcard
;
1355 u_short lport
= lport_arg
;
1356 struct inpcbport
*phd
;
1357 struct inpcb
*match
= NULL
;
1360 * If the porthashbase is shared across several cpus, it must
1363 ASSERT_PORTHASH_TOKEN_HELD(porthash
);
1366 * Best fit PCB lookup.
1368 * First see if this local port is in use by looking on the
1371 LIST_FOREACH(phd
, porthash
, phd_hash
) {
1372 if (phd
->phd_port
== lport
)
1377 * Port is in use by one or more PCBs. Look for best
1380 LIST_FOREACH(inp
, &phd
->phd_pcblist
, inp_portlist
) {
1383 if (!INP_ISIPV4(inp
))
1386 if (inp
->inp_faddr
.s_addr
!= INADDR_ANY
)
1388 if (inp
->inp_laddr
.s_addr
!= INADDR_ANY
) {
1389 if (laddr
.s_addr
== INADDR_ANY
)
1391 else if (inp
->inp_laddr
.s_addr
!= laddr
.s_addr
)
1394 if (laddr
.s_addr
!= INADDR_ANY
)
1397 if (wildcard
&& !wild_okay
)
1399 if (wildcard
< matchwild
&&
1402 inp
->inp_socket
->so_cred
->cr_prison
)) {
1404 matchwild
= wildcard
;
1405 if (matchwild
== 0) {
1415 in_pcblocalgroup_last(const struct inpcbinfo
*pcbinfo
,
1416 const struct inpcb
*inp
)
1418 const struct inp_localgrphead
*hdr
;
1419 const struct inp_localgroup
*grp
;
1422 if (pcbinfo
->localgrphashbase
== NULL
)
1425 GET_PCBINFO_TOKEN(pcbinfo
);
1427 hdr
= &pcbinfo
->localgrphashbase
[
1428 INP_PCBLOCALGRPHASH(inp
->inp_lport
, pcbinfo
->localgrphashmask
)];
1430 LIST_FOREACH(grp
, hdr
, il_list
) {
1431 if (grp
->il_af
== inp
->inp_af
&&
1432 grp
->il_lport
== inp
->inp_lport
&&
1433 memcmp(&grp
->il_dependladdr
,
1434 &inp
->inp_inc
.inc_ie
.ie_dependladdr
,
1435 sizeof(grp
->il_dependladdr
)) == 0) {
1439 if (grp
== NULL
|| grp
->il_inpcnt
== 1) {
1440 REL_PCBINFO_TOKEN(pcbinfo
);
1444 KASSERT(grp
->il_inpcnt
>= 2,
1445 ("invalid localgroup inp count %d", grp
->il_inpcnt
));
1446 for (i
= 0; i
< grp
->il_inpcnt
; ++i
) {
1447 if (grp
->il_inp
[i
] == inp
) {
1448 int last
= grp
->il_inpcnt
- 1;
1451 last
= grp
->il_inpcnt
- 2;
1452 REL_PCBINFO_TOKEN(pcbinfo
);
1453 return grp
->il_inp
[last
];
1456 REL_PCBINFO_TOKEN(pcbinfo
);
1460 static struct inpcb
*
1461 inp_localgroup_lookup(const struct inpcbinfo
*pcbinfo
,
1462 struct in_addr laddr
, uint16_t lport
, uint32_t pkt_hash
)
1464 struct inpcb
*local_wild
= NULL
;
1465 const struct inp_localgrphead
*hdr
;
1466 const struct inp_localgroup
*grp
;
1468 ASSERT_PCBINFO_TOKEN_HELD(pcbinfo
);
1470 hdr
= &pcbinfo
->localgrphashbase
[
1471 INP_PCBLOCALGRPHASH(lport
, pcbinfo
->localgrphashmask
)];
1474 * Order of socket selection:
1478 * NOTE: Local group does not contain jailed sockets
1480 LIST_FOREACH(grp
, hdr
, il_list
) {
1482 if (grp
->il_af
!= AF_INET
)
1485 if (grp
->il_lport
== lport
) {
1489 * Modulo-N is used here, which greatly reduces
1490 * completion queue token contention, thus more
1491 * cpu time is saved.
1493 idx
= netisr_hashlsb(pkt_hash
) % grp
->il_inpcnt
;
1494 if (grp
->il_laddr
.s_addr
== laddr
.s_addr
)
1495 return grp
->il_inp
[idx
];
1496 else if (grp
->il_laddr
.s_addr
== INADDR_ANY
)
1497 local_wild
= grp
->il_inp
[idx
];
1500 if (local_wild
!= NULL
)
1506 * Lookup PCB in hash list.
1509 in_pcblookup_pkthash(struct inpcbinfo
*pcbinfo
, struct in_addr faddr
,
1510 u_int fport_arg
, struct in_addr laddr
, u_int lport_arg
,
1511 boolean_t wildcard
, struct ifnet
*ifp
, const struct mbuf
*m
)
1513 struct inpcbhead
*head
;
1514 struct inpcb
*inp
, *jinp
=NULL
;
1515 u_short fport
= fport_arg
, lport
= lport_arg
;
1518 * First look for an exact match.
1520 head
= &pcbinfo
->hashbase
[INP_PCBCONNHASH(faddr
.s_addr
, fport
,
1521 laddr
.s_addr
, lport
, pcbinfo
->hashmask
)];
1522 LIST_FOREACH(inp
, head
, inp_hash
) {
1524 if (!INP_ISIPV4(inp
))
1527 if (in_hosteq(inp
->inp_faddr
, faddr
) &&
1528 in_hosteq(inp
->inp_laddr
, laddr
) &&
1529 inp
->inp_fport
== fport
&& inp
->inp_lport
== lport
) {
1531 if (inp
->inp_socket
== NULL
||
1532 inp
->inp_socket
->so_cred
->cr_prison
== NULL
) {
1544 struct inpcb
*local_wild
= NULL
;
1545 struct inpcb
*jinp_wild
= NULL
;
1546 struct inpcontainer
*ic
;
1547 struct inpcontainerhead
*chead
;
1548 struct sockaddr_in jsin
;
1551 GET_PCBINFO_TOKEN(pcbinfo
);
1554 * Check local group first
1556 if (pcbinfo
->localgrphashbase
!= NULL
&&
1557 m
!= NULL
&& (m
->m_flags
& M_HASH
) &&
1558 !(ifp
&& ifp
->if_type
== IFT_FAITH
)) {
1559 inp
= inp_localgroup_lookup(pcbinfo
,
1560 laddr
, lport
, m
->m_pkthdr
.hash
);
1562 REL_PCBINFO_TOKEN(pcbinfo
);
1568 * Order of socket selection:
1569 * 1. non-jailed, non-wild.
1570 * 2. non-jailed, wild.
1571 * 3. jailed, non-wild.
1574 jsin
.sin_family
= AF_INET
;
1575 chead
= &pcbinfo
->wildcardhashbase
[
1576 INP_PCBWILDCARDHASH(lport
, pcbinfo
->wildcardhashmask
)];
1577 LIST_FOREACH(ic
, chead
, ic_list
) {
1579 if (inp
->inp_flags
& INP_PLACEMARKER
)
1582 jsin
.sin_addr
.s_addr
= laddr
.s_addr
;
1584 if (!INP_ISIPV4(inp
))
1587 if (inp
->inp_socket
!= NULL
)
1588 cred
= inp
->inp_socket
->so_cred
;
1591 if (cred
!= NULL
&& jailed(cred
)) {
1595 if (!jailed_ip(cred
->cr_prison
,
1596 (struct sockaddr
*)&jsin
))
1599 if (inp
->inp_lport
== lport
) {
1600 if (ifp
&& ifp
->if_type
== IFT_FAITH
&&
1601 !(inp
->inp_flags
& INP_FAITH
))
1603 if (inp
->inp_laddr
.s_addr
== laddr
.s_addr
) {
1604 if (cred
!= NULL
&& jailed(cred
)) {
1607 REL_PCBINFO_TOKEN(pcbinfo
);
1611 if (inp
->inp_laddr
.s_addr
== INADDR_ANY
) {
1612 if (cred
!= NULL
&& jailed(cred
))
1620 REL_PCBINFO_TOKEN(pcbinfo
);
1622 if (local_wild
!= NULL
)
1623 return (local_wild
);
1636 in_pcblookup_hash(struct inpcbinfo
*pcbinfo
, struct in_addr faddr
,
1637 u_int fport_arg
, struct in_addr laddr
, u_int lport_arg
,
1638 boolean_t wildcard
, struct ifnet
*ifp
)
1640 return in_pcblookup_pkthash(pcbinfo
, faddr
, fport_arg
,
1641 laddr
, lport_arg
, wildcard
, ifp
, NULL
);
1645 * Insert PCB into connection hash table.
1648 in_pcbinsconnhash(struct inpcb
*inp
)
1650 struct inpcbinfo
*pcbinfo
= inp
->inp_pcbinfo
;
1651 struct inpcbhead
*bucket
;
1652 u_int32_t hashkey_faddr
, hashkey_laddr
;
1655 if (INP_ISIPV6(inp
)) {
1656 hashkey_faddr
= inp
->in6p_faddr
.s6_addr32
[3] /* XXX JH */;
1657 hashkey_laddr
= inp
->in6p_laddr
.s6_addr32
[3] /* XXX JH */;
1660 hashkey_faddr
= inp
->inp_faddr
.s_addr
;
1661 hashkey_laddr
= inp
->inp_laddr
.s_addr
;
1666 KASSERT(&curthread
->td_msgport
== netisr_cpuport(pcbinfo
->cpu
),
1667 ("not in the correct netisr"));
1668 ASSERT_INP_NOTINHASH(inp
);
1669 inp
->inp_flags
|= INP_CONNECTED
;
1672 * Insert into the connection hash table.
1674 bucket
= &pcbinfo
->hashbase
[INP_PCBCONNHASH(hashkey_faddr
,
1675 inp
->inp_fport
, hashkey_laddr
, inp
->inp_lport
, pcbinfo
->hashmask
)];
1676 LIST_INSERT_HEAD(bucket
, inp
, inp_hash
);
1680 * Remove PCB from connection hash table.
1683 in_pcbremconnhash(struct inpcb
*inp
)
1685 struct inpcbinfo
*pcbinfo __debugvar
= inp
->inp_pcbinfo
;
1687 KASSERT(&curthread
->td_msgport
== netisr_cpuport(pcbinfo
->cpu
),
1688 ("not in the correct netisr"));
1689 KASSERT(inp
->inp_flags
& INP_CONNECTED
, ("inp not connected"));
1691 LIST_REMOVE(inp
, inp_hash
);
1692 inp
->inp_flags
&= ~INP_CONNECTED
;
1696 * Insert PCB into port hash table.
1699 in_pcbinsporthash(struct inpcbporthead
*pcbporthash
, struct inpcb
*inp
)
1701 struct inpcbinfo
*pcbinfo
= inp
->inp_pcbinfo
;
1702 struct inpcbport
*phd
;
1705 * If the porthashbase is shared across several cpus, it must
1708 ASSERT_PORTHASH_TOKEN_HELD(pcbporthash
);
1711 * Insert into the port hash table.
1714 /* Go through port list and look for a head for this lport. */
1715 LIST_FOREACH(phd
, pcbporthash
, phd_hash
) {
1716 if (phd
->phd_port
== inp
->inp_lport
)
1720 /* If none exists, use saved one and tack it on. */
1722 KKASSERT(pcbinfo
->portsave
!= NULL
);
1723 phd
= pcbinfo
->portsave
;
1724 pcbinfo
->portsave
= NULL
;
1725 phd
->phd_port
= inp
->inp_lport
;
1726 LIST_INIT(&phd
->phd_pcblist
);
1727 LIST_INSERT_HEAD(pcbporthash
, phd
, phd_hash
);
1730 inp
->inp_porthash
= pcbporthash
;
1732 LIST_INSERT_HEAD(&phd
->phd_pcblist
, inp
, inp_portlist
);
1735 * Malloc one inpcbport for later use. It is safe to use
1736 * "wait" malloc here (port token would be released, if
1737 * malloc ever blocked), since all changes to the porthash
1740 if (pcbinfo
->portsave
== NULL
) {
1741 pcbinfo
->portsave
= kmalloc(sizeof(*pcbinfo
->portsave
),
1742 M_PCB
, M_INTWAIT
| M_ZERO
);
1747 in_pcbinsporthash_lport(struct inpcb
*inp
)
1749 struct inpcbinfo
*pcbinfo
= inp
->inp_pcbinfo
;
1750 struct inpcbportinfo
*portinfo
;
1751 struct inpcbporthead
*porthash
;
1754 /* Locate the proper portinfo based on lport */
1755 lport_ho
= ntohs(inp
->inp_lport
);
1756 portinfo
= &pcbinfo
->portinfo
[lport_ho
% pcbinfo
->portinfo_cnt
];
1757 KKASSERT((lport_ho
% pcbinfo
->portinfo_cnt
) == portinfo
->offset
);
1759 porthash
= in_pcbporthash_head(portinfo
, inp
->inp_lport
);
1760 GET_PORTHASH_TOKEN(porthash
);
1761 in_pcbinsporthash(porthash
, inp
);
1762 REL_PORTHASH_TOKEN(porthash
);
1766 in_pcbremporthash(struct inpcb
*inp
)
1768 struct inpcbporthead
*porthash
;
1769 struct inpcbport
*phd
;
1771 if (inp
->inp_phd
== NULL
)
1773 KASSERT(inp
->inp_lport
!= 0, ("inpcb has no lport"));
1775 porthash
= inp
->inp_porthash
;
1776 KASSERT(porthash
!= NULL
, ("no porthash"));
1778 GET_PORTHASH_TOKEN(porthash
);
1781 LIST_REMOVE(inp
, inp_portlist
);
1782 if (LIST_FIRST(&phd
->phd_pcblist
) == NULL
) {
1783 LIST_REMOVE(phd
, phd_hash
);
1787 REL_PORTHASH_TOKEN(porthash
);
1789 inp
->inp_phd
= NULL
;
1790 /* NOTE: Don't whack inp_lport, which may be used later */
1793 static struct inp_localgroup
*
1794 inp_localgroup_alloc(u_char af
, uint16_t port
,
1795 const union in_dependaddr
*addr
, int size
)
1797 struct inp_localgroup
*grp
;
1799 grp
= kmalloc(__offsetof(struct inp_localgroup
, il_inp
[size
]),
1800 M_TEMP
, M_INTWAIT
| M_ZERO
);
1802 grp
->il_lport
= port
;
1803 grp
->il_dependladdr
= *addr
;
1804 grp
->il_inpsiz
= size
;
1810 inp_localgroup_free(struct inp_localgroup
*grp
)
1816 inp_localgroup_destroy(struct inp_localgroup
*grp
)
1818 LIST_REMOVE(grp
, il_list
);
1819 inp_localgroup_free(grp
);
1823 inp_localgroup_copy(struct inp_localgroup
*grp
,
1824 const struct inp_localgroup
*old_grp
)
1828 KASSERT(old_grp
->il_inpcnt
< grp
->il_inpsiz
,
1829 ("invalid new local group size %d and old local group count %d",
1830 grp
->il_inpsiz
, old_grp
->il_inpcnt
));
1831 for (i
= 0; i
< old_grp
->il_inpcnt
; ++i
)
1832 grp
->il_inp
[i
] = old_grp
->il_inp
[i
];
1833 grp
->il_inpcnt
= old_grp
->il_inpcnt
;
1837 in_pcbinslocalgrphash_oncpu(struct inpcb
*inp
, struct inpcbinfo
*pcbinfo
)
1839 struct inp_localgrphead
*hdr
;
1840 struct inp_localgroup
*grp
, *grp_alloc
= NULL
;
1844 ASSERT_PCBINFO_TOKEN_HELD(pcbinfo
);
1846 if (pcbinfo
->localgrphashbase
== NULL
)
1850 * XXX don't allow jailed socket to join local group
1852 if (inp
->inp_socket
!= NULL
)
1853 cred
= inp
->inp_socket
->so_cred
;
1856 if (cred
!= NULL
&& jailed(cred
))
1859 hdr
= &pcbinfo
->localgrphashbase
[
1860 INP_PCBLOCALGRPHASH(inp
->inp_lport
, pcbinfo
->localgrphashmask
)];
1863 LIST_FOREACH(grp
, hdr
, il_list
) {
1864 if (grp
->il_af
== inp
->inp_af
&&
1865 grp
->il_lport
== inp
->inp_lport
&&
1866 memcmp(&grp
->il_dependladdr
,
1867 &inp
->inp_inc
.inc_ie
.ie_dependladdr
,
1868 sizeof(grp
->il_dependladdr
)) == 0) {
1874 * Create a new local group
1876 if (grp_alloc
== NULL
) {
1877 grp_alloc
= inp_localgroup_alloc(inp
->inp_af
,
1878 inp
->inp_lport
, &inp
->inp_inc
.inc_ie
.ie_dependladdr
,
1879 INP_LOCALGROUP_SIZMIN
);
1881 * Local group allocation could block and the
1882 * local group w/ the same property might have
1883 * been added by others when we were blocked;
1888 /* Local group has been allocated; link it */
1891 LIST_INSERT_HEAD(hdr
, grp
, il_list
);
1893 } else if (grp
->il_inpcnt
== grp
->il_inpsiz
) {
1894 if (grp
->il_inpsiz
>= INP_LOCALGROUP_SIZMAX
) {
1895 static int limit_logged
= 0;
1897 if (!limit_logged
) {
1899 kprintf("local group port %d, "
1900 "limit reached\n", ntohs(grp
->il_lport
));
1902 if (grp_alloc
!= NULL
) {
1904 * This would happen if the local group
1905 * w/ the same property was expanded when
1906 * our local group allocation blocked.
1908 inp_localgroup_free(grp_alloc
);
1914 * Expand this local group
1916 if (grp_alloc
== NULL
||
1917 grp
->il_inpcnt
>= grp_alloc
->il_inpsiz
) {
1918 if (grp_alloc
!= NULL
)
1919 inp_localgroup_free(grp_alloc
);
1920 grp_alloc
= inp_localgroup_alloc(grp
->il_af
,
1921 grp
->il_lport
, &grp
->il_dependladdr
,
1922 grp
->il_inpsiz
* 2);
1924 * Local group allocation could block and the
1925 * local group w/ the same property might have
1926 * been expanded by others when we were blocked;
1933 * Save the old local group, link the new one, and then
1934 * destroy the old local group
1936 inp_localgroup_copy(grp_alloc
, grp
);
1937 LIST_INSERT_HEAD(hdr
, grp_alloc
, il_list
);
1938 inp_localgroup_destroy(grp
);
1944 * Found the local group
1946 if (grp_alloc
!= NULL
) {
1948 * This would happen if the local group w/ the
1949 * same property was added or expanded when our
1950 * local group allocation blocked.
1952 inp_localgroup_free(grp_alloc
);
1957 KASSERT(grp
->il_inpcnt
< grp
->il_inpsiz
,
1958 ("invalid local group size %d and count %d",
1959 grp
->il_inpsiz
, grp
->il_inpcnt
));
1962 * Keep the local group sorted by the inpcb local group index
1963 * in ascending order.
1965 * This eases the multi-process userland application which uses
1966 * SO_REUSEPORT sockets and binds process to the owner cpu of
1967 * the SO_REUSEPORT socket:
1968 * If we didn't sort the local group by the inpcb local group
1969 * index and one of the process owning an inpcb in this local
1970 * group restarted, e.g. crashed and restarted by watchdog,
1971 * other processes owning a inpcb in this local group would have
1972 * to detect that event, refetch its socket's owner cpu, and
1975 idx
= grp
->il_inpcnt
;
1976 for (i
= 0; i
< idx
; ++i
) {
1977 struct inpcb
*oinp
= grp
->il_inp
[i
];
1979 if (oinp
->inp_lgrpindex
> i
) {
1980 if (inp
->inp_lgrpindex
< 0) {
1981 inp
->inp_lgrpindex
= i
;
1982 } else if (inp
->inp_lgrpindex
!= i
) {
1984 kprintf("inp %p: grpidx %d, "
1985 "assigned to %d, cpu%d\n",
1986 inp
, inp
->inp_lgrpindex
, i
,
1990 grp
->il_inp
[i
] = inp
;
1992 /* Pull down inpcbs */
1993 for (; i
< grp
->il_inpcnt
; ++i
) {
1994 struct inpcb
*oinp1
= grp
->il_inp
[i
+ 1];
1996 grp
->il_inp
[i
+ 1] = oinp
;
2004 if (inp
->inp_lgrpindex
< 0) {
2005 inp
->inp_lgrpindex
= idx
;
2006 } else if (inp
->inp_lgrpindex
!= idx
) {
2008 kprintf("inp %p: grpidx %d, assigned to %d, cpu%d\n",
2009 inp
, inp
->inp_lgrpindex
, idx
, mycpuid
);
2012 grp
->il_inp
[idx
] = inp
;
2017 in_pcbinswildcardhash_oncpu(struct inpcb
*inp
, struct inpcbinfo
*pcbinfo
)
2019 struct inpcontainer
*ic
;
2020 struct inpcontainerhead
*bucket
;
2022 GET_PCBINFO_TOKEN(pcbinfo
);
2024 in_pcbinslocalgrphash_oncpu(inp
, pcbinfo
);
2026 bucket
= &pcbinfo
->wildcardhashbase
[
2027 INP_PCBWILDCARDHASH(inp
->inp_lport
, pcbinfo
->wildcardhashmask
)];
2029 ic
= kmalloc(sizeof(struct inpcontainer
), M_TEMP
, M_INTWAIT
);
2031 LIST_INSERT_HEAD(bucket
, ic
, ic_list
);
2033 REL_PCBINFO_TOKEN(pcbinfo
);
2037 * Insert PCB into wildcard hash table.
2040 in_pcbinswildcardhash(struct inpcb
*inp
)
2042 struct inpcbinfo
*pcbinfo
= inp
->inp_pcbinfo
;
2044 KASSERT(&curthread
->td_msgport
== netisr_cpuport(pcbinfo
->cpu
),
2045 ("not in correct netisr"));
2046 ASSERT_INP_NOTINHASH(inp
);
2047 inp
->inp_flags
|= INP_WILDCARD
;
2049 in_pcbinswildcardhash_oncpu(inp
, pcbinfo
);
2053 in_pcbremlocalgrphash_oncpu(struct inpcb
*inp
, struct inpcbinfo
*pcbinfo
)
2055 struct inp_localgrphead
*hdr
;
2056 struct inp_localgroup
*grp
;
2058 ASSERT_PCBINFO_TOKEN_HELD(pcbinfo
);
2060 if (pcbinfo
->localgrphashbase
== NULL
)
2063 hdr
= &pcbinfo
->localgrphashbase
[
2064 INP_PCBLOCALGRPHASH(inp
->inp_lport
, pcbinfo
->localgrphashmask
)];
2066 LIST_FOREACH(grp
, hdr
, il_list
) {
2069 for (i
= 0; i
< grp
->il_inpcnt
; ++i
) {
2070 if (grp
->il_inp
[i
] != inp
)
2073 if (grp
->il_inpcnt
== 1) {
2074 /* Destroy this local group */
2075 inp_localgroup_destroy(grp
);
2077 /* Pull up inpcbs */
2078 for (; i
+ 1 < grp
->il_inpcnt
; ++i
)
2079 grp
->il_inp
[i
] = grp
->il_inp
[i
+ 1];
2088 in_pcbremwildcardhash_oncpu(struct inpcb
*inp
, struct inpcbinfo
*pcbinfo
)
2090 struct inpcontainer
*ic
;
2091 struct inpcontainerhead
*head
;
2093 GET_PCBINFO_TOKEN(pcbinfo
);
2095 in_pcbremlocalgrphash_oncpu(inp
, pcbinfo
);
2098 head
= &pcbinfo
->wildcardhashbase
[
2099 INP_PCBWILDCARDHASH(inp
->inp_lport
, pcbinfo
->wildcardhashmask
)];
2101 LIST_FOREACH(ic
, head
, ic_list
) {
2102 if (ic
->ic_inp
== inp
)
2105 REL_PCBINFO_TOKEN(pcbinfo
);
2106 return; /* not found! */
2109 LIST_REMOVE(ic
, ic_list
); /* remove container from bucket chain */
2110 REL_PCBINFO_TOKEN(pcbinfo
);
2111 kfree(ic
, M_TEMP
); /* deallocate container */
2115 * Remove PCB from wildcard hash table.
2118 in_pcbremwildcardhash(struct inpcb
*inp
)
2120 struct inpcbinfo
*pcbinfo
= inp
->inp_pcbinfo
;
2122 KASSERT(&curthread
->td_msgport
== netisr_cpuport(pcbinfo
->cpu
),
2123 ("not in correct netisr"));
2124 KASSERT(inp
->inp_flags
& INP_WILDCARD
, ("inp not wildcard"));
2126 in_pcbremwildcardhash_oncpu(inp
, pcbinfo
);
2127 inp
->inp_lgrpindex
= -1;
2128 inp
->inp_flags
&= ~INP_WILDCARD
;
2132 * Remove PCB from various lists.
2135 in_pcbremlists(struct inpcb
*inp
)
2137 in_pcbremporthash(inp
);
2138 if (inp
->inp_flags
& INP_WILDCARD
) {
2139 in_pcbremwildcardhash(inp
);
2140 } else if (inp
->inp_flags
& INP_CONNECTED
) {
2141 in_pcbremconnhash(inp
);
2144 if (inp
->inp_flags
& INP_ONLIST
)
2149 prison_xinpcb(struct thread
*td
, struct inpcb
*inp
)
2153 if (td
->td_proc
== NULL
)
2155 cr
= td
->td_proc
->p_ucred
;
2156 if (cr
->cr_prison
== NULL
)
2158 if (inp
->inp_socket
&& inp
->inp_socket
->so_cred
&&
2159 inp
->inp_socket
->so_cred
->cr_prison
&&
2160 cr
->cr_prison
== inp
->inp_socket
->so_cred
->cr_prison
)
2166 in_pcblist_range(SYSCTL_HANDLER_ARGS
)
2168 struct inpcbinfo
*pcbinfo_arr
= arg1
;
2169 int pcbinfo_arrlen
= arg2
;
2170 struct inpcb
*marker
;
2174 KASSERT(pcbinfo_arrlen
<= netisr_ncpus
&& pcbinfo_arrlen
>= 1,
2175 ("invalid pcbinfo count %d", pcbinfo_arrlen
));
2178 * The process of preparing the TCB list is too time-consuming and
2179 * resource-intensive to repeat twice on every request.
2182 if (req
->oldptr
== NULL
) {
2183 for (cpu
= 0; cpu
< pcbinfo_arrlen
; ++cpu
)
2184 n
+= pcbinfo_arr
[cpu
].ipi_count
;
2185 req
->oldidx
= (n
+ n
/8 + 10) * sizeof(struct xinpcb
);
2189 if (req
->newptr
!= NULL
)
2192 marker
= kmalloc(sizeof(struct inpcb
), M_TEMP
, M_WAITOK
|M_ZERO
);
2193 marker
->inp_flags
|= INP_PLACEMARKER
;
2196 * OK, now we're committed to doing something. Re-fetch ipi_count
2197 * after obtaining the generation count.
2201 for (cpu
= 0; cpu
< pcbinfo_arrlen
&& error
== 0; ++cpu
) {
2202 struct inpcbinfo
*pcbinfo
= &pcbinfo_arr
[cpu
];
2207 lwkt_migratecpu(cpu
);
2209 GET_PCBINFO_TOKEN(pcbinfo
);
2211 n
= pcbinfo
->ipi_count
;
2213 LIST_INSERT_HEAD(&pcbinfo
->pcblisthead
, marker
, inp_list
);
2215 while ((inp
= LIST_NEXT(marker
, inp_list
)) != NULL
&& i
< n
) {
2216 LIST_REMOVE(marker
, inp_list
);
2217 LIST_INSERT_AFTER(inp
, marker
, inp_list
);
2219 if (inp
->inp_flags
& INP_PLACEMARKER
)
2221 if (prison_xinpcb(req
->td
, inp
))
2224 bzero(&xi
, sizeof xi
);
2225 xi
.xi_len
= sizeof xi
;
2226 bcopy(inp
, &xi
.xi_inp
, sizeof *inp
);
2227 if (inp
->inp_socket
)
2228 sotoxsocket(inp
->inp_socket
, &xi
.xi_socket
);
2229 if ((error
= SYSCTL_OUT(req
, &xi
, sizeof xi
)) != 0)
2233 LIST_REMOVE(marker
, inp_list
);
2235 REL_PCBINFO_TOKEN(pcbinfo
);
2237 if (error
== 0 && i
< n
) {
2238 bzero(&xi
, sizeof xi
);
2239 xi
.xi_len
= sizeof xi
;
2241 error
= SYSCTL_OUT(req
, &xi
, sizeof xi
);
2249 lwkt_migratecpu(origcpu
);
2250 kfree(marker
, M_TEMP
);
2255 in_pcblist_ncpus(SYSCTL_HANDLER_ARGS
)
2258 return (in_pcblist_range(oidp
, arg1
, netisr_ncpus
, req
));
2262 in_savefaddr(struct socket
*so
, const struct sockaddr
*faddr
)
2264 struct sockaddr_in
*sin
;
2266 KASSERT(faddr
->sa_family
== AF_INET
,
2267 ("not AF_INET faddr %d", faddr
->sa_family
));
2269 sin
= kmalloc(sizeof(*sin
), M_SONAME
, M_WAITOK
| M_ZERO
);
2270 sin
->sin_family
= AF_INET
;
2271 sin
->sin_len
= sizeof(*sin
);
2272 sin
->sin_port
= ((const struct sockaddr_in
*)faddr
)->sin_port
;
2273 sin
->sin_addr
= ((const struct sockaddr_in
*)faddr
)->sin_addr
;
2275 so
->so_faddr
= (struct sockaddr
*)sin
;
2279 in_pcbportinfo_init(struct inpcbportinfo
*portinfo
, int hashsize
,
2282 memset(portinfo
, 0, sizeof(*portinfo
));
2284 portinfo
->offset
= offset
;
2285 portinfo
->porthashbase
= phashinit(hashsize
, M_PCB
,
2286 &portinfo
->porthashcnt
);
2290 in_pcbportrange(u_short
*hi0
, u_short
*lo0
, u_short ofs
, u_short step
)
2300 hi
= rounddown(hi
, step
);
2305 lo
= roundup(lo
, step
);
2315 in_pcbglobalinit(void)
2319 in_pcbmarkers
= kmalloc(netisr_ncpus
* sizeof(struct inpcb
), M_PCB
,
2321 in_pcbcontainer_markers
=
2322 kmalloc(netisr_ncpus
* sizeof(struct inpcontainer
), M_PCB
,
2325 for (cpu
= 0; cpu
< netisr_ncpus
; ++cpu
) {
2326 struct inpcontainer
*ic
= &in_pcbcontainer_markers
[cpu
];
2327 struct inpcb
*marker
= &in_pcbmarkers
[cpu
];
2329 marker
->inp_flags
|= INP_PLACEMARKER
;
2330 ic
->ic_inp
= marker
;
2338 ASSERT_NETISR_NCPUS(mycpuid
);
2339 return &in_pcbmarkers
[mycpuid
];
2342 struct inpcontainer
*
2343 in_pcbcontainer_marker(void)
2346 ASSERT_NETISR_NCPUS(mycpuid
);
2347 return &in_pcbcontainer_markers
[mycpuid
];
2351 in_pcbresetroute(struct inpcb
*inp
)
2353 struct route
*ro
= &inp
->inp_route
;
2355 if (ro
->ro_rt
!= NULL
)
2357 bzero(ro
, sizeof(*ro
));