2 * Copyright (c) 2004 Jeffrey M. Hsu. All rights reserved.
3 * Copyright (c) 2004 The DragonFly Project. All rights reserved.
5 * This code is derived from software contributed to The DragonFly Project
8 * Redistribution and use in source and binary forms, with or without
9 * modification, are permitted provided that the following conditions
11 * 1. Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * 2. Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in the
15 * documentation and/or other materials provided with the distribution.
16 * 3. Neither the name of The DragonFly Project nor the names of its
17 * contributors may be used to endorse or promote products derived
18 * from this software without specific, prior written permission.
20 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
21 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
22 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
23 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
24 * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
25 * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
26 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
27 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
28 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
29 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
30 * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
35 * Copyright (c) 1982, 1986, 1991, 1993, 1995
36 * The Regents of the University of California. All rights reserved.
38 * Redistribution and use in source and binary forms, with or without
39 * modification, are permitted provided that the following conditions
41 * 1. Redistributions of source code must retain the above copyright
42 * notice, this list of conditions and the following disclaimer.
43 * 2. Redistributions in binary form must reproduce the above copyright
44 * notice, this list of conditions and the following disclaimer in the
45 * documentation and/or other materials provided with the distribution.
46 * 3. Neither the name of the University nor the names of its contributors
47 * may be used to endorse or promote products derived from this software
48 * without specific prior written permission.
50 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
51 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
52 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
53 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
54 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
55 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
56 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
57 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
58 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
59 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
62 * @(#)in_pcb.c 8.4 (Berkeley) 5/24/95
63 * $FreeBSD: src/sys/netinet/in_pcb.c,v 1.59.2.27 2004/01/02 04:06:42 ambrisko Exp $
66 #include "opt_inet6.h"
68 #include <sys/param.h>
69 #include <sys/systm.h>
70 #include <sys/malloc.h>
72 #include <sys/domain.h>
73 #include <sys/protosw.h>
74 #include <sys/socket.h>
75 #include <sys/socketvar.h>
79 #include <sys/kernel.h>
80 #include <sys/sysctl.h>
82 #include <sys/socketvar2.h>
83 #include <sys/msgport2.h>
85 #include <machine/limits.h>
88 #include <net/if_types.h>
89 #include <net/route.h>
90 #include <net/netisr2.h>
91 #include <net/toeplitz2.h>
93 #include <netinet/in.h>
94 #include <netinet/in_pcb.h>
95 #include <netinet/in_var.h>
96 #include <netinet/ip_var.h>
98 #include <netinet/ip6.h>
99 #include <netinet6/ip6_var.h>
102 #define INP_LOCALGROUP_SIZMIN 8
103 #define INP_LOCALGROUP_SIZMAX 256
105 static struct inpcb
*in_pcblookup_local(struct inpcbporthead
*porthash
,
106 struct in_addr laddr
, u_int lport_arg
, int wild_okay
,
109 struct in_addr zeroin_addr
;
112 * These configure the range of local port addresses assigned to
113 * "unspecified" outgoing connections/packets/whatever.
115 int ipport_lowfirstauto
= IPPORT_RESERVED
- 1; /* 1023 */
116 int ipport_lowlastauto
= IPPORT_RESERVEDSTART
; /* 600 */
118 int ipport_firstauto
= IPPORT_RESERVED
; /* 1024 */
119 int ipport_lastauto
= IPPORT_USERRESERVED
; /* 5000 */
121 int ipport_hifirstauto
= IPPORT_HIFIRSTAUTO
; /* 49152 */
122 int ipport_hilastauto
= IPPORT_HILASTAUTO
; /* 65535 */
124 #define RANGECHK(var, min, max) \
125 if ((var) < (min)) { (var) = (min); } \
126 else if ((var) > (max)) { (var) = (max); }
128 int udpencap_enable
= 1; /* enabled by default */
129 int udpencap_port
= 4500; /* triggers decapsulation */
132 * Per-netisr inpcb markers.
133 * NOTE: they should only be used in netisrs.
135 static struct inpcb
*in_pcbmarkers
;
136 static struct inpcontainer
*in_pcbcontainer_markers
;
139 sysctl_net_ipport_check(SYSCTL_HANDLER_ARGS
)
143 error
= sysctl_handle_int(oidp
, oidp
->oid_arg1
, oidp
->oid_arg2
, req
);
145 RANGECHK(ipport_lowfirstauto
, 1, IPPORT_RESERVED
- 1);
146 RANGECHK(ipport_lowlastauto
, 1, IPPORT_RESERVED
- 1);
148 RANGECHK(ipport_firstauto
, IPPORT_RESERVED
, USHRT_MAX
);
149 RANGECHK(ipport_lastauto
, IPPORT_RESERVED
, USHRT_MAX
);
151 RANGECHK(ipport_hifirstauto
, IPPORT_RESERVED
, USHRT_MAX
);
152 RANGECHK(ipport_hilastauto
, IPPORT_RESERVED
, USHRT_MAX
);
159 SYSCTL_NODE(_net_inet_ip
, IPPROTO_IP
, portrange
, CTLFLAG_RW
, 0, "IP Ports");
161 SYSCTL_PROC(_net_inet_ip_portrange
, OID_AUTO
, lowfirst
, CTLTYPE_INT
|CTLFLAG_RW
,
162 &ipport_lowfirstauto
, 0, &sysctl_net_ipport_check
, "I", "");
163 SYSCTL_PROC(_net_inet_ip_portrange
, OID_AUTO
, lowlast
, CTLTYPE_INT
|CTLFLAG_RW
,
164 &ipport_lowlastauto
, 0, &sysctl_net_ipport_check
, "I", "");
165 SYSCTL_PROC(_net_inet_ip_portrange
, OID_AUTO
, first
, CTLTYPE_INT
|CTLFLAG_RW
,
166 &ipport_firstauto
, 0, &sysctl_net_ipport_check
, "I", "");
167 SYSCTL_PROC(_net_inet_ip_portrange
, OID_AUTO
, last
, CTLTYPE_INT
|CTLFLAG_RW
,
168 &ipport_lastauto
, 0, &sysctl_net_ipport_check
, "I", "");
169 SYSCTL_PROC(_net_inet_ip_portrange
, OID_AUTO
, hifirst
, CTLTYPE_INT
|CTLFLAG_RW
,
170 &ipport_hifirstauto
, 0, &sysctl_net_ipport_check
, "I", "");
171 SYSCTL_PROC(_net_inet_ip_portrange
, OID_AUTO
, hilast
, CTLTYPE_INT
|CTLFLAG_RW
,
172 &ipport_hilastauto
, 0, &sysctl_net_ipport_check
, "I", "");
174 /* Initialized by ip_init() */
175 int ip_porthash_trycount
;
176 SYSCTL_INT(_net_inet_ip
, OID_AUTO
, porthash_trycount
, CTLFLAG_RW
,
177 &ip_porthash_trycount
, 0,
178 "Number of tries to find local port matching hash of 4-tuple");
181 * in_pcb.c: manage the Protocol Control Blocks.
183 * NOTE: It is assumed that most of these functions will be called from
184 * a critical section. XXX - There are, unfortunately, a few exceptions
185 * to this rule that should be fixed.
187 * NOTE: The caller should initialize the cpu field to the cpu running the
188 * protocol stack associated with this inpcbinfo.
192 in_pcbinfo_init(struct inpcbinfo
*pcbinfo
, int cpu
, boolean_t shared
)
194 KASSERT(cpu
>= 0 && cpu
< netisr_ncpus
, ("invalid cpu%d", cpu
));
197 LIST_INIT(&pcbinfo
->pcblisthead
);
198 pcbinfo
->portsave
= kmalloc(sizeof(*pcbinfo
->portsave
), M_PCB
,
202 pcbinfo
->infotoken
= kmalloc(sizeof(struct lwkt_token
),
204 lwkt_token_init(pcbinfo
->infotoken
, "infotoken");
206 pcbinfo
->infotoken
= NULL
;
211 in_pcbportinfo_set(struct inpcbinfo
*pcbinfo
, struct inpcbportinfo
*portinfo
,
215 KASSERT(portinfo_cnt
> 0, ("invalid portinfo_cnt %d", portinfo_cnt
));
216 pcbinfo
->portinfo
= portinfo
;
217 pcbinfo
->portinfo_cnt
= portinfo_cnt
;
220 struct baddynamicports baddynamicports
;
223 * Check if the specified port is invalid for dynamic allocation.
226 in_baddynamic(u_int16_t port
, u_int16_t proto
)
230 return (DP_ISSET(baddynamicports
.tcp
, port
));
232 return (DP_ISSET(baddynamicports
.udp
, port
));
239 in_pcbonlist(struct inpcb
*inp
)
241 struct inpcbinfo
*pcbinfo
= inp
->inp_pcbinfo
;
243 KASSERT(&curthread
->td_msgport
== netisr_cpuport(pcbinfo
->cpu
),
244 ("not in the correct netisr"));
245 KASSERT((inp
->inp_flags
& INP_ONLIST
) == 0, ("already on pcblist"));
246 inp
->inp_flags
|= INP_ONLIST
;
248 GET_PCBINFO_TOKEN(pcbinfo
);
249 LIST_INSERT_HEAD(&pcbinfo
->pcblisthead
, inp
, inp_list
);
250 pcbinfo
->ipi_count
++;
251 REL_PCBINFO_TOKEN(pcbinfo
);
255 in_pcbofflist(struct inpcb
*inp
)
257 struct inpcbinfo
*pcbinfo
= inp
->inp_pcbinfo
;
259 KASSERT(&curthread
->td_msgport
== netisr_cpuport(pcbinfo
->cpu
),
260 ("not in the correct netisr"));
261 KASSERT(inp
->inp_flags
& INP_ONLIST
, ("not on pcblist"));
262 inp
->inp_flags
&= ~INP_ONLIST
;
264 GET_PCBINFO_TOKEN(pcbinfo
);
265 LIST_REMOVE(inp
, inp_list
);
266 KASSERT(pcbinfo
->ipi_count
> 0,
267 ("invalid inpcb count %d", pcbinfo
->ipi_count
));
268 pcbinfo
->ipi_count
--;
269 REL_PCBINFO_TOKEN(pcbinfo
);
273 * Allocate a PCB and associate it with the socket.
276 in_pcballoc(struct socket
*so
, struct inpcbinfo
*pcbinfo
)
280 inp
= kmalloc(pcbinfo
->ipi_size
, M_PCB
, M_WAITOK
|M_ZERO
|M_NULLOK
);
283 inp
->inp_lgrpindex
= -1;
284 inp
->inp_gencnt
= ++pcbinfo
->ipi_gencnt
;
285 inp
->inp_pcbinfo
= pcbinfo
;
286 inp
->inp_socket
= so
;
288 if (INP_CHECK_SOCKAF(so
, AF_INET6
)) {
289 if (ip6_auto_flowlabel
)
290 inp
->inp_flags
|= IN6P_AUTOFLOWLABEL
;
291 inp
->inp_af
= AF_INET6
;
294 inp
->inp_af
= AF_INET
;
303 * Unlink a pcb with the intention of moving it to another cpu with a
304 * different pcbinfo. While unlinked nothing should attempt to dereference
305 * inp_pcbinfo, NULL it out so we assert if it does.
308 in_pcbunlink_flags(struct inpcb
*inp
, struct inpcbinfo
*pcbinfo
, int flags
)
310 KASSERT(inp
->inp_pcbinfo
== pcbinfo
, ("pcbinfo mismatch"));
311 KASSERT((inp
->inp_flags
& (flags
| INP_CONNECTED
)) == 0,
315 inp
->inp_pcbinfo
= NULL
;
319 in_pcbunlink(struct inpcb
*inp
, struct inpcbinfo
*pcbinfo
)
321 in_pcbunlink_flags(inp
, pcbinfo
, INP_WILDCARD
);
325 * Relink a pcb into a new pcbinfo.
328 in_pcblink_flags(struct inpcb
*inp
, struct inpcbinfo
*pcbinfo
, int flags
)
330 KASSERT(inp
->inp_pcbinfo
== NULL
, ("has pcbinfo"));
331 KASSERT((inp
->inp_flags
& (flags
| INP_CONNECTED
)) == 0,
334 inp
->inp_pcbinfo
= pcbinfo
;
339 in_pcblink(struct inpcb
*inp
, struct inpcbinfo
*pcbinfo
)
341 return in_pcblink_flags(inp
, pcbinfo
, INP_WILDCARD
);
345 in_pcbporthash_update(struct inpcbportinfo
*portinfo
,
346 struct inpcb
*inp
, u_short lport
, struct ucred
*cred
, int wild
)
348 struct inpcbporthead
*porthash
;
351 * This has to be atomic. If the porthash is shared across multiple
352 * protocol threads, e.g. tcp and udp, then the token must be held.
354 porthash
= in_pcbporthash_head(portinfo
, lport
);
355 GET_PORTHASH_TOKEN(porthash
);
357 if (in_pcblookup_local(porthash
, inp
->inp_laddr
, lport
, wild
, cred
)) {
358 REL_PORTHASH_TOKEN(porthash
);
361 inp
->inp_lport
= lport
;
362 in_pcbinsporthash(porthash
, inp
);
364 REL_PORTHASH_TOKEN(porthash
);
369 in_pcbsetlport(struct inpcb
*inp
, int wild
, struct ucred
*cred
)
371 struct inpcbinfo
*pcbinfo
= inp
->inp_pcbinfo
;
372 struct inpcbportinfo
*portinfo
;
373 u_short first
, last
, lport
, step
, first0
, last0
;
375 int portinfo_first
, portinfo_idx
;
378 inp
->inp_flags
|= INP_ANONPORT
;
380 step
= pcbinfo
->portinfo_cnt
;
381 portinfo_first
= mycpuid
% pcbinfo
->portinfo_cnt
;
382 portinfo_idx
= portinfo_first
;
384 if (inp
->inp_flags
& INP_HIGHPORT
) {
385 first0
= ipport_hifirstauto
; /* sysctl */
386 last0
= ipport_hilastauto
;
387 } else if (inp
->inp_flags
& INP_LOWPORT
) {
389 (error
= caps_priv_check(cred
, SYSCAP_NONET_RESPORT
)))
391 inp
->inp_laddr
.s_addr
= INADDR_ANY
;
394 first0
= ipport_lowfirstauto
; /* 1023 */
395 last0
= ipport_lowlastauto
; /* 600 */
397 first0
= ipport_firstauto
; /* sysctl */
398 last0
= ipport_lastauto
;
400 if (first0
> last0
) {
405 KKASSERT(last0
>= first0
);
409 portinfo
= &pcbinfo
->portinfo
[portinfo_idx
];
414 * Simple check to ensure all ports are not used up causing
417 in_pcbportrange(&last
, &first
, portinfo
->offset
, step
);
418 lport
= last
- first
;
419 count
= lport
/ step
;
421 lport
= rounddown(cut
% lport
, step
) + first
;
422 KKASSERT(lport
% step
== portinfo
->offset
);
425 if (count
-- < 0) { /* completely used? */
426 error
= EADDRNOTAVAIL
;
430 if (__predict_false(lport
< first
|| lport
> last
)) {
432 KKASSERT(lport
% step
== portinfo
->offset
);
435 if (in_pcbporthash_update(portinfo
, inp
, htons(lport
),
442 KKASSERT(lport
% step
== portinfo
->offset
);
446 /* Try next portinfo */
448 portinfo_idx
%= pcbinfo
->portinfo_cnt
;
449 if (portinfo_idx
!= portinfo_first
)
451 inp
->inp_laddr
.s_addr
= INADDR_ANY
;
456 static __inline
struct inpcbporthead
*
457 OBTAIN_LPORTHASH_TOKEN(struct inpcbinfo
*pcbinfo
, u_short lport
)
459 struct inpcbportinfo
*portinfo
;
460 struct inpcbporthead
*porthash
;
461 u_short lport_ho
= ntohs(lport
);
464 * Locate the proper portinfo based on lport
466 portinfo
= &pcbinfo
->portinfo
[lport_ho
% pcbinfo
->portinfo_cnt
];
467 KKASSERT((lport_ho
% pcbinfo
->portinfo_cnt
) == portinfo
->offset
);
469 porthash
= in_pcbporthash_head(portinfo
, lport
);
470 GET_PORTHASH_TOKEN(porthash
);
476 in_pcbbind_laddr(struct sockaddr_in
*sin
, struct in_addr
*laddr
,
479 struct sockaddr_in jsin
;
481 if (!prison_replace_wildcards(td
, (struct sockaddr
*)sin
))
484 if (!IN_MULTICAST(ntohl(sin
->sin_addr
.s_addr
)) &&
485 sin
->sin_addr
.s_addr
!= INADDR_ANY
) {
486 sin
->sin_port
= 0; /* yech... */
487 bzero(&sin
->sin_zero
, sizeof sin
->sin_zero
);
488 if (ifa_ifwithaddr((struct sockaddr
*)sin
) == NULL
)
489 return (EADDRNOTAVAIL
);
492 *laddr
= sin
->sin_addr
;
494 jsin
.sin_family
= AF_INET
;
495 jsin
.sin_addr
.s_addr
= laddr
->s_addr
;
496 if (!prison_replace_wildcards(td
, (struct sockaddr
*)&jsin
)) {
497 laddr
->s_addr
= INADDR_ANY
;
500 laddr
->s_addr
= jsin
.sin_addr
.s_addr
;
506 in_pcbbind_laddrport_check(const struct socket
*so
, struct sockaddr_in
*sin
,
507 struct inpcbporthead
*porthash
, int wild
, struct ucred
*cred
,
510 int reuseport
= (so
->so_options
& SO_REUSEPORT
);
513 ASSERT_PORTHASH_TOKEN_HELD(porthash
);
515 if (IN_MULTICAST(ntohl(sin
->sin_addr
.s_addr
))) {
517 * Treat SO_REUSEADDR as SO_REUSEPORT for multicast;
518 * allow complete duplication of binding if
519 * SO_REUSEPORT is set, or if SO_REUSEADDR is set
520 * and a multicast address is bound on both
521 * new and duplicated sockets.
523 if (so
->so_options
& SO_REUSEADDR
)
524 reuseport
= SO_REUSEADDR
| SO_REUSEPORT
;
527 if (so
->so_cred
->cr_uid
!= 0 &&
528 !IN_MULTICAST(ntohl(sin
->sin_addr
.s_addr
))) {
529 t
= in_pcblookup_local(porthash
, sin
->sin_addr
, sin
->sin_port
,
530 INPLOOKUP_WILDCARD
, cred
);
531 if (t
&& t
->inp_socket
!= so
&&
532 (so
->so_cred
->cr_uid
!= t
->inp_socket
->so_cred
->cr_uid
))
535 if (cred
&& !prison_replace_wildcards(td
, (struct sockaddr
*)sin
))
536 return (EADDRNOTAVAIL
);
539 * When binding to a local port if the best match is against
540 * an accepted socket we generally want to allow the binding.
541 * This means that there is no longer any specific socket
542 * bound or bound for listening.
544 t
= in_pcblookup_local(porthash
, sin
->sin_addr
, sin
->sin_port
,
546 if (t
&& t
->inp_socket
!= so
&&
547 (reuseport
& t
->inp_socket
->so_options
) == 0 &&
548 (t
->inp_socket
->so_state
& SS_ACCEPTMECH
) == 0)
555 in_pcbsrcaddr_check(const struct inpcb
*inp
, struct sockaddr_in
*sin
,
556 struct in_addr
*laddr
, struct thread
*td
)
558 const struct socket
*so
= inp
->inp_socket
;
559 struct inpcbporthead
*porthash
;
560 struct ucred
*cred
= NULL
;
564 /* inp must be bound beforehand. */
565 KKASSERT(inp
->inp_lport
!= 0);
566 KKASSERT(sin
->sin_len
== sizeof(*sin
));
568 if (!(so
->so_options
& (SO_REUSEADDR
|SO_REUSEPORT
)))
569 wild
= 1; /* neither SO_REUSEADDR nor SO_REUSEPORT is set */
571 cred
= td
->td_proc
->p_ucred
;
573 /* Always use inp_lport */
574 sin
->sin_port
= inp
->inp_lport
;
576 error
= in_pcbbind_laddr(sin
, laddr
, td
);
580 if (IN_MULTICAST(ntohl(laddr
->s_addr
))) {
581 /* Unlike bind, multicast src address is not allowed. */
585 if (inp
->inp_laddr
.s_addr
== laddr
->s_addr
) {
587 * src address is same as what we bound to.
589 * inp_laddr == INADDR_ANY && srcaddr == INADDR_ANY
590 * is allowed, which does not really matter.
593 } else if (inp
->inp_laddr
.s_addr
!= INADDR_ANY
&&
594 !IN_MULTICAST(ntohl(inp
->inp_laddr
.s_addr
))) {
595 /* Already bound to a specific address */
600 * This has to be atomic. If the porthash is shared across
601 * multiple protocol threads, e.g. tcp and udp then the token
604 porthash
= OBTAIN_LPORTHASH_TOKEN(inp
->inp_pcbinfo
, inp
->inp_lport
);
607 * Restore the sin_port whacked by in_pcbbind_ladddr();
608 * sin->sin_port is checked by in_pcbbind_laddrport_check().
610 sin
->sin_port
= inp
->inp_lport
;
612 error
= in_pcbbind_laddrport_check(so
, sin
, porthash
, wild
, cred
, td
);
614 laddr
->s_addr
= INADDR_ANY
;
616 REL_PORTHASH_TOKEN(porthash
);
621 in_pcbbind(struct inpcb
*inp
, struct sockaddr
*nam
, struct thread
*td
)
623 const struct socket
*so
= inp
->inp_socket
;
624 struct ucred
*cred
= NULL
;
627 if (inp
->inp_lport
!= 0 || inp
->inp_laddr
.s_addr
!= INADDR_ANY
)
628 return (EINVAL
); /* already bound */
630 if (!(so
->so_options
& (SO_REUSEADDR
|SO_REUSEPORT
)))
631 wild
= 1; /* neither SO_REUSEADDR nor SO_REUSEPORT is set */
633 cred
= td
->td_proc
->p_ucred
;
636 struct sockaddr_in
*sin
= (struct sockaddr_in
*)nam
;
637 struct inpcbporthead
*porthash
;
641 if (nam
->sa_len
!= sizeof *sin
)
645 * We should check the family, but old programs
646 * incorrectly fail to initialize it.
648 if (sin
->sin_family
!= AF_INET
)
649 return (EAFNOSUPPORT
);
653 * Save sin_port for later use, since it will
654 * be whacked by in_pcbbind_laddr().
656 lport
= sin
->sin_port
;
658 error
= in_pcbbind_laddr(sin
, &inp
->inp_laddr
, td
);
663 /* Auto-select local port */
664 return in_pcbsetlport(inp
, wild
, cred
);
668 if (ntohs(lport
) < IPPORT_RESERVED
&& cred
&&
669 (error
= caps_priv_check(cred
, SYSCAP_NONET_RESPORT
)))
671 inp
->inp_laddr
.s_addr
= INADDR_ANY
;
676 * This has to be atomic. If the porthash is shared across
677 * multiple protocol threads, e.g. tcp and udp then the token
680 porthash
= OBTAIN_LPORTHASH_TOKEN(inp
->inp_pcbinfo
, lport
);
683 * Restore the sin_port whacked by in_pcbbind_ladddr();
684 * sin->sin_port is checked by in_pcbbind_laddrport_check().
686 sin
->sin_port
= lport
;
688 error
= in_pcbbind_laddrport_check(so
, sin
, porthash
,
691 inp
->inp_laddr
.s_addr
= INADDR_ANY
;
695 inp
->inp_lport
= lport
;
696 in_pcbinsporthash(porthash
, inp
);
699 REL_PORTHASH_TOKEN(porthash
);
702 struct sockaddr_in jsin
;
704 jsin
.sin_family
= AF_INET
;
705 jsin
.sin_addr
.s_addr
= inp
->inp_laddr
.s_addr
;
706 if (!prison_replace_wildcards(td
, (struct sockaddr
*)&jsin
)) {
707 inp
->inp_laddr
.s_addr
= INADDR_ANY
;
710 inp
->inp_laddr
.s_addr
= jsin
.sin_addr
.s_addr
;
712 return in_pcbsetlport(inp
, wild
, cred
);
717 * Lookup a PCB based on the local and remote address and port.
719 * This function is only used when scanning for a free port.
721 static struct inpcb
*
722 in_pcblookup_localremote(struct inpcbporthead
*porthash
, struct in_addr laddr
,
723 u_short lport
, struct in_addr faddr
, u_short fport
,
727 struct inpcbport
*phd
;
728 struct inpcb
*match
= NULL
;
729 struct prison
*pscan
;
733 * If the porthashbase is shared across several cpus, it must
736 ASSERT_PORTHASH_TOKEN_HELD(porthash
);
739 * Best fit PCB lookup.
741 * First see if this local port is in use by looking on the
744 LIST_FOREACH(phd
, porthash
, phd_hash
) {
745 if (phd
->phd_port
== lport
)
749 pr
= cred
? cred
->cr_prison
: NULL
;
751 LIST_FOREACH(inp
, &phd
->phd_pcblist
, inp_portlist
) {
753 if (!INP_ISIPV4(inp
))
756 if (inp
->inp_laddr
.s_addr
== INADDR_ANY
) {
757 if (inp
->inp_socket
&& inp
->inp_socket
->so_cred
)
758 pscan
= inp
->inp_socket
->so_cred
->cr_prison
;
764 if (inp
->inp_laddr
.s_addr
!= laddr
.s_addr
)
768 if (inp
->inp_faddr
.s_addr
!= INADDR_ANY
&&
769 inp
->inp_faddr
.s_addr
!= faddr
.s_addr
)
772 if (inp
->inp_fport
!= 0 && inp
->inp_fport
!= fport
)
783 in_pcbporthash_update4(struct inpcbportinfo
*portinfo
, struct inpcb
*inp
,
784 u_short lport
, const struct sockaddr_in
*sin
,
787 struct inpcbporthead
*porthash
;
790 * This has to be atomic. If the porthash is shared across multiple
791 * protocol threads, e.g. tcp and udp, then the token must be held.
793 porthash
= in_pcbporthash_head(portinfo
, lport
);
794 GET_PORTHASH_TOKEN(porthash
);
796 if (in_pcblookup_localremote(porthash
, inp
->inp_laddr
, lport
,
797 sin
->sin_addr
, sin
->sin_port
, cred
)) {
798 REL_PORTHASH_TOKEN(porthash
);
801 inp
->inp_lport
= lport
;
802 in_pcbinsporthash(porthash
, inp
);
804 REL_PORTHASH_TOKEN(porthash
);
809 in_pcbbind_remote(struct inpcb
*inp
, const struct sockaddr
*remote
,
812 struct proc
*p
= td
->td_proc
;
813 const struct sockaddr_in
*sin
= (const struct sockaddr_in
*)remote
;
814 struct sockaddr_in jsin
;
815 struct inpcbinfo
*pcbinfo
= inp
->inp_pcbinfo
;
816 struct ucred
*cred
= NULL
;
817 u_short first
, last
, lport
;
818 int count
, hash_count
;
819 int error
, selfconn
= 0;
821 uint32_t hash_base
= 0, hash
;
823 ASSERT_NETISR_NCPUS(cpuid
);
825 if (TAILQ_EMPTY(&in_ifaddrheads
[cpuid
])) /* XXX broken! */
826 return (EADDRNOTAVAIL
);
828 KKASSERT(inp
->inp_laddr
.s_addr
!= INADDR_ANY
);
829 if (inp
->inp_lport
!= 0)
830 return (EINVAL
); /* already bound */
835 jsin
.sin_family
= AF_INET
;
836 jsin
.sin_addr
.s_addr
= inp
->inp_laddr
.s_addr
;
837 if (!prison_replace_wildcards(td
, (struct sockaddr
*)&jsin
)) {
838 inp
->inp_laddr
.s_addr
= INADDR_ANY
;
841 inp
->inp_laddr
.s_addr
= jsin
.sin_addr
.s_addr
;
843 hash_count
= ip_porthash_trycount
;
844 if (hash_count
> 0) {
845 hash_base
= toeplitz_piecemeal_addr(sin
->sin_addr
.s_addr
) ^
846 toeplitz_piecemeal_addr(inp
->inp_laddr
.s_addr
) ^
847 toeplitz_piecemeal_port(sin
->sin_port
);
852 inp
->inp_flags
|= INP_ANONPORT
;
854 if (inp
->inp_flags
& INP_HIGHPORT
) {
855 first
= ipport_hifirstauto
; /* sysctl */
856 last
= ipport_hilastauto
;
857 } else if (inp
->inp_flags
& INP_LOWPORT
) {
859 (error
= caps_priv_check(cred
, SYSCAP_NONET_RESPORT
)))
861 inp
->inp_laddr
.s_addr
= INADDR_ANY
;
864 first
= ipport_lowfirstauto
; /* 1023 */
865 last
= ipport_lowlastauto
; /* 600 */
867 first
= ipport_firstauto
; /* sysctl */
868 last
= ipport_lastauto
;
875 KKASSERT(last
>= first
);
877 count
= last
- first
;
878 lport
= (karc4random() % count
) + first
;
882 * Simple check to ensure all ports are not used up causing
888 if (count
-- < 0) { /* completely used? */
889 error
= EADDRNOTAVAIL
;
893 if (__predict_false(lport
< first
|| lport
> last
))
895 lport_no
= htons(lport
);
897 /* This could happen on loopback interface */
898 if (__predict_false(sin
->sin_port
== lport_no
&&
899 sin
->sin_addr
.s_addr
== inp
->inp_laddr
.s_addr
)) {
901 ++count
; /* don't count this try */
910 toeplitz_piecemeal_port(lport_no
);
911 if (netisr_hashcpu(hash
) != cpuid
&& hash_count
)
915 if (in_pcbporthash_update4(
916 &pcbinfo
->portinfo
[lport
% pcbinfo
->portinfo_cnt
],
917 inp
, lport_no
, sin
, cred
)) {
926 inp
->inp_laddr
.s_addr
= INADDR_ANY
;
931 * Figure out the local interface address to pair against the requested
932 * target address, as well as validate the target address.
935 in_pcbladdr_find(struct inpcb
*inp
, struct sockaddr
*nam
,
936 struct sockaddr_in
**plocal_sin
, struct thread
*td
, int find
)
938 struct in_ifaddr_container
*iac
;
939 struct in_ifaddr
*ia
;
940 struct ucred
*cred
= NULL
;
941 struct sockaddr_in
*sin
= (struct sockaddr_in
*)nam
;
942 struct sockaddr
*jsin
;
947 if (nam
->sa_len
!= sizeof *sin
)
949 if (sin
->sin_family
!= AF_INET
)
950 return (EAFNOSUPPORT
);
951 if (sin
->sin_port
== 0)
952 return (EADDRNOTAVAIL
);
958 if (td
&& td
->td_proc
&& td
->td_proc
->p_ucred
)
959 cred
= td
->td_proc
->p_ucred
;
961 pr
= cred
->cr_prison
;
964 * If the destination address is INADDR_ANY then use the primary
967 * If the supplied address is INADDR_BROADCAST, and the primary
968 * interface supports broadcast, choose the broadcast address for
971 * If jailed, locate an interface address acceptable to the jail.
973 if (sin
->sin_addr
.s_addr
== INADDR_ANY
) {
974 TAILQ_FOREACH(iac
, &in_ifaddrheads
[mycpuid
], ia_link
) {
977 jailed_ip(pr
, sintosa(&ia
->ia_addr
))) {
978 sin
->sin_addr
= IA_SIN(ia
)->sin_addr
;
982 } else if (sin
->sin_addr
.s_addr
== (u_long
)INADDR_BROADCAST
) {
983 TAILQ_FOREACH(iac
, &in_ifaddrheads
[mycpuid
], ia_link
) {
986 jailed_ip(pr
, sintosa(&ia
->ia_addr
))) &&
987 (iac
->ia
->ia_ifp
->if_flags
& IFF_BROADCAST
)) {
989 satosin(&ia
->ia_broadaddr
)->sin_addr
;
996 * If asked to do a search, use the cached route or do a route table
997 * lookup to try to find an acceptable local interface IP.
1005 * If we have a cached route, check to see if it is acceptable.
1008 ro
= &inp
->inp_route
;
1010 (!(ro
->ro_rt
->rt_flags
& RTF_UP
) ||
1011 ro
->ro_dst
.sa_family
!= AF_INET
||
1012 satosin(&ro
->ro_dst
)->sin_addr
.s_addr
!=
1013 sin
->sin_addr
.s_addr
||
1014 inp
->inp_socket
->so_options
& SO_DONTROUTE
)) {
1020 * If we do not have a route, construct one and do a lookup,
1021 * unless we are forbidden to do so.
1023 * Note that we should check the address family of the cached
1024 * destination, in case of sharing the cache with IPv6.
1026 if (!(inp
->inp_socket
->so_options
& SO_DONTROUTE
) && /*XXX*/
1027 (ro
->ro_rt
== NULL
|| ro
->ro_rt
->rt_ifp
== NULL
)) {
1028 bzero(&ro
->ro_dst
, sizeof(struct sockaddr_in
));
1029 ro
->ro_dst
.sa_family
= AF_INET
;
1030 ro
->ro_dst
.sa_len
= sizeof(struct sockaddr_in
);
1031 ((struct sockaddr_in
*)&ro
->ro_dst
)->sin_addr
= sin
->sin_addr
;
1037 * If we found a route, use the address corresponding to the
1038 * outgoing interface.
1040 * If jailed, try to find a compatible address on the outgoing
1044 ia
= ifatoia(ro
->ro_rt
->rt_ifa
);
1047 if (jailed_ip(pr
, sintosa(&ia
->ia_addr
)))
1049 TAILQ_FOREACH(iac
, &in_ifaddrheads
[mycpuid
], ia_link
) {
1050 if (iac
->ia
->ia_ifp
!= ia
->ia_ifp
)
1053 if (jailed_ip(pr
, sintosa(&ia
->ia_addr
)))
1061 * If the route didn't work or there was no route,
1062 * fall-back to the first address in in_ifaddrheads[].
1064 * If jailed and this address is not available for
1065 * the jail, leave ia set to NULL.
1068 u_short fport
= sin
->sin_port
;
1071 ia
= ifatoia(ifa_ifwithdstaddr(sintosa(sin
)));
1072 if (ia
&& pr
&& !jailed_ip(pr
, sintosa(&ia
->ia_addr
)))
1076 ia
= ifatoia(ifa_ifwithnet(sintosa(sin
)));
1077 if (ia
&& pr
&& !jailed_ip(pr
, sintosa(&ia
->ia_addr
)))
1080 sin
->sin_port
= fport
;
1081 if (ia
== NULL
&& !TAILQ_EMPTY(&in_ifaddrheads
[mycpuid
]))
1082 ia
= TAILQ_FIRST(&in_ifaddrheads
[mycpuid
])->ia
;
1084 if (ia
&& pr
&& !jailed_ip(pr
, sintosa(&ia
->ia_addr
)))
1087 if (pr
== NULL
&& ia
== NULL
)
1092 * If the destination address is multicast and an outgoing
1093 * interface has been set as a multicast option, use the
1094 * address of that interface as our source address.
1096 if (pr
== NULL
&& IN_MULTICAST(ntohl(sin
->sin_addr
.s_addr
)) &&
1097 inp
->inp_moptions
!= NULL
) {
1098 struct ip_moptions
*imo
;
1101 imo
= inp
->inp_moptions
;
1102 if ((ifp
= imo
->imo_multicast_ifp
) != NULL
) {
1103 struct in_ifaddr_container
*iac
;
1106 TAILQ_FOREACH(iac
, &in_ifaddrheads
[mycpuid
], ia_link
) {
1107 if (iac
->ia
->ia_ifp
== ifp
) {
1118 * If we still don't have a local address, and are jailed,
1119 * use the jail's first non-localhost IP. If there isn't
1120 * one, use the jail's first localhost IP.
1122 * Don't do pcblookup call here; return interface in plocal_sin
1123 * and exit to caller, that will do the lookup.
1125 if (ia
== NULL
&& pr
) {
1126 jsin
= prison_get_nonlocal(cred
->cr_prison
, AF_INET
, NULL
);
1128 jsin
= prison_get_local(cred
->cr_prison
, AF_INET
, NULL
);
1130 *plocal_sin
= satosin(jsin
);
1134 *plocal_sin
= &ia
->ia_addr
;
1141 in_pcbresetroute(inp
);
1142 return (EADDRNOTAVAIL
);
1146 in_pcbladdr(struct inpcb
*inp
, struct sockaddr
*nam
,
1147 struct sockaddr_in
**plocal_sin
, struct thread
*td
)
1149 return in_pcbladdr_find(inp
, nam
, plocal_sin
, td
,
1150 (inp
->inp_laddr
.s_addr
== INADDR_ANY
));
1155 * Connect from a socket to a specified address.
1156 * Both address and port must be specified in argument sin.
1157 * If don't have a local address for this socket yet,
1161 in_pcbconnect(struct inpcb
*inp
, struct sockaddr
*nam
, struct thread
*td
)
1163 struct sockaddr_in
*if_sin
;
1164 struct sockaddr_in
*sin
= (struct sockaddr_in
*)nam
;
1167 if_sin
= NULL
; /* avoid gcc warnings */
1169 /* Call inner routine to assign local interface address. */
1170 if ((error
= in_pcbladdr(inp
, nam
, &if_sin
, td
)) != 0)
1173 if (in_pcblookup_hash(inp
->inp_pcbinfo
, sin
->sin_addr
, sin
->sin_port
,
1174 inp
->inp_laddr
.s_addr
?
1175 inp
->inp_laddr
: if_sin
->sin_addr
,
1176 inp
->inp_lport
, FALSE
, NULL
) != NULL
) {
1177 return (EADDRINUSE
);
1179 if (inp
->inp_laddr
.s_addr
== INADDR_ANY
) {
1180 if (inp
->inp_lport
== 0) {
1181 error
= in_pcbbind(inp
, NULL
, td
);
1185 inp
->inp_laddr
= if_sin
->sin_addr
;
1187 inp
->inp_faddr
= sin
->sin_addr
;
1188 inp
->inp_fport
= sin
->sin_port
;
1189 in_pcbinsconnhash(inp
);
1194 in_pcbdisconnect(struct inpcb
*inp
)
1197 in_pcbremconnhash(inp
);
1198 inp
->inp_faddr
.s_addr
= INADDR_ANY
;
1203 in_pcbdetach(struct inpcb
*inp
)
1205 struct socket
*so
= inp
->inp_socket
;
1206 struct inpcbinfo
*ipi
= inp
->inp_pcbinfo
;
1208 inp
->inp_gencnt
= ++ipi
->ipi_gencnt
;
1209 KKASSERT((so
->so_state
& SS_ASSERTINPROG
) == 0);
1210 in_pcbremlists(inp
);
1212 sofree(so
); /* remove pcb ref */
1213 if (inp
->inp_options
)
1214 m_free(inp
->inp_options
);
1215 if (inp
->inp_route
.ro_rt
)
1216 rtfree(inp
->inp_route
.ro_rt
);
1217 ip_freemoptions(inp
->inp_moptions
);
1222 * The socket may have an invalid PCB, i.e. NULL. For example, a TCP
1223 * socket received RST.
1226 in_setsockaddr(struct socket
*so
, struct sockaddr
**nam
)
1229 struct sockaddr_in
*sin
;
1231 KASSERT(curthread
->td_type
== TD_TYPE_NETISR
, ("not in netisr"));
1234 return (ECONNRESET
);
1236 sin
= kmalloc(sizeof *sin
, M_SONAME
, M_WAITOK
| M_ZERO
);
1237 sin
->sin_family
= AF_INET
;
1238 sin
->sin_len
= sizeof *sin
;
1239 sin
->sin_port
= inp
->inp_lport
;
1240 sin
->sin_addr
= inp
->inp_laddr
;
1242 *nam
= (struct sockaddr
*)sin
;
1247 in_setsockaddr_dispatch(netmsg_t msg
)
1251 error
= in_setsockaddr(msg
->base
.nm_so
, msg
->peeraddr
.nm_nam
);
1252 lwkt_replymsg(&msg
->lmsg
, error
);
1256 * The socket may have an invalid PCB, i.e. NULL. For example, a TCP
1257 * socket received RST.
1260 in_setpeeraddr(struct socket
*so
, struct sockaddr
**nam
)
1263 struct sockaddr_in
*sin
;
1265 KASSERT(curthread
->td_type
== TD_TYPE_NETISR
, ("not in netisr"));
1268 return (ECONNRESET
);
1270 sin
= kmalloc(sizeof *sin
, M_SONAME
, M_WAITOK
| M_ZERO
);
1271 sin
->sin_family
= AF_INET
;
1272 sin
->sin_len
= sizeof *sin
;
1273 sin
->sin_port
= inp
->inp_fport
;
1274 sin
->sin_addr
= inp
->inp_faddr
;
1276 *nam
= (struct sockaddr
*)sin
;
1281 in_setpeeraddr_dispatch(netmsg_t msg
)
1285 error
= in_setpeeraddr(msg
->base
.nm_so
, msg
->peeraddr
.nm_nam
);
1286 lwkt_replymsg(&msg
->lmsg
, error
);
1290 in_pcbnotifyall(struct inpcbinfo
*pcbinfo
, struct in_addr faddr
, int err
,
1291 inp_notify_t notify
)
1293 struct inpcb
*inp
, *marker
;
1295 KASSERT(&curthread
->td_msgport
== netisr_cpuport(pcbinfo
->cpu
),
1296 ("not in the correct netisr"));
1297 marker
= in_pcbmarker();
1301 * - If INP_PLACEMARKER is set we must ignore the rest of the
1302 * structure and skip it.
1303 * - It is safe to nuke inpcbs here, since we are in their own
1306 GET_PCBINFO_TOKEN(pcbinfo
);
1308 LIST_INSERT_HEAD(&pcbinfo
->pcblisthead
, marker
, inp_list
);
1309 while ((inp
= LIST_NEXT(marker
, inp_list
)) != NULL
) {
1310 LIST_REMOVE(marker
, inp_list
);
1311 LIST_INSERT_AFTER(inp
, marker
, inp_list
);
1313 if (inp
->inp_flags
& INP_PLACEMARKER
)
1316 if (!INP_ISIPV4(inp
))
1319 if (inp
->inp_faddr
.s_addr
!= faddr
.s_addr
||
1320 inp
->inp_socket
== NULL
)
1322 (*notify
)(inp
, err
); /* can remove inp from list! */
1324 LIST_REMOVE(marker
, inp_list
);
1326 REL_PCBINFO_TOKEN(pcbinfo
);
1330 in_pcbpurgeif0(struct inpcbinfo
*pcbinfo
, struct ifnet
*ifp
)
1332 struct inpcb
*inp
, *marker
;
1335 * We only need to make sure that we are in netisr0, where all
1336 * multicast operation happen. We could check inpcbinfo which
1337 * does not belong to netisr0 by holding the inpcbinfo's token.
1338 * In this case, the pcbinfo must be able to be shared, i.e.
1339 * pcbinfo->infotoken is not NULL.
1342 KASSERT(pcbinfo
->cpu
== 0 || pcbinfo
->infotoken
!= NULL
,
1343 ("pcbinfo could not be shared"));
1346 * Get a marker for the current netisr (netisr0).
1348 * It is possible that the multicast address deletion blocks,
1349 * which could cause temporary token releasing. So we use
1350 * inpcb marker here to get a coherent view of the inpcb list.
1352 * While, on the other hand, moptions are only added and deleted
1353 * in netisr0, so we would not see staled moption or miss moption
1354 * even if the token was released due to the blocking multicast
1357 marker
= in_pcbmarker();
1359 GET_PCBINFO_TOKEN(pcbinfo
);
1361 LIST_INSERT_HEAD(&pcbinfo
->pcblisthead
, marker
, inp_list
);
1362 while ((inp
= LIST_NEXT(marker
, inp_list
)) != NULL
) {
1363 struct ip_moptions
*imo
;
1365 LIST_REMOVE(marker
, inp_list
);
1366 LIST_INSERT_AFTER(inp
, marker
, inp_list
);
1368 if (inp
->inp_flags
& INP_PLACEMARKER
)
1370 imo
= inp
->inp_moptions
;
1371 if (INP_ISIPV4(inp
) && imo
!= NULL
) {
1375 * Unselect the outgoing interface if it is being
1378 if (imo
->imo_multicast_ifp
== ifp
)
1379 imo
->imo_multicast_ifp
= NULL
;
1382 * Drop multicast group membership if we joined
1383 * through the interface being detached.
1385 for (i
= 0, gap
= 0; i
< imo
->imo_num_memberships
;
1387 if (imo
->imo_membership
[i
]->inm_ifp
== ifp
) {
1390 * This could block and the pcbinfo
1391 * token could be passively released.
1393 in_delmulti(imo
->imo_membership
[i
]);
1395 } else if (gap
!= 0)
1396 imo
->imo_membership
[i
- gap
] =
1397 imo
->imo_membership
[i
];
1399 imo
->imo_num_memberships
-= gap
;
1402 LIST_REMOVE(marker
, inp_list
);
1404 REL_PCBINFO_TOKEN(pcbinfo
);
1408 * Check for alternatives when higher level complains
1409 * about service problems. For now, invalidate cached
1410 * routing information. If the route was created dynamically
1411 * (by a redirect), time to try a default gateway again.
1414 in_losing(struct inpcb
*inp
)
1417 struct rt_addrinfo rtinfo
;
1419 if ((rt
= inp
->inp_route
.ro_rt
)) {
1420 bzero(&rtinfo
, sizeof(struct rt_addrinfo
));
1421 rtinfo
.rti_info
[RTAX_DST
] = rt_key(rt
);
1422 rtinfo
.rti_info
[RTAX_GATEWAY
] = rt
->rt_gateway
;
1423 rtinfo
.rti_info
[RTAX_NETMASK
] = rt_mask(rt
);
1424 rtinfo
.rti_flags
= rt
->rt_flags
;
1425 rt_missmsg(RTM_LOSING
, &rtinfo
, rt
->rt_flags
, 0);
1426 if (rt
->rt_flags
& RTF_DYNAMIC
) {
1427 rtrequest(RTM_DELETE
, rt_key(rt
), rt
->rt_gateway
,
1428 rt_mask(rt
), rt
->rt_flags
, NULL
);
1430 inp
->inp_route
.ro_rt
= NULL
;
1433 * A new route can be allocated
1434 * the next time output is attempted.
1440 * After a routing change, flush old routing
1441 * and allocate a (hopefully) better one.
1444 in_rtchange(struct inpcb
*inp
, int err
)
1446 if (inp
->inp_route
.ro_rt
) {
1447 rtfree(inp
->inp_route
.ro_rt
);
1448 inp
->inp_route
.ro_rt
= NULL
;
1450 * A new route can be allocated the next time
1451 * output is attempted.
1457 * Lookup a PCB based on the local address and port.
1459 * This function is only used when scanning for a free port.
1461 static struct inpcb
*
1462 in_pcblookup_local(struct inpcbporthead
*porthash
, struct in_addr laddr
,
1463 u_int lport_arg
, int wild_okay
, struct ucred
*cred
)
1465 struct prison
*pscan
;
1468 int matchwild
= 3, wildcard
;
1469 u_short lport
= lport_arg
;
1470 struct inpcbport
*phd
;
1471 struct inpcb
*match
= NULL
;
1474 * If the porthashbase is shared across several cpus, it must
1477 ASSERT_PORTHASH_TOKEN_HELD(porthash
);
1480 * Best fit PCB lookup.
1482 * First see if this local port is in use by looking on the
1485 LIST_FOREACH(phd
, porthash
, phd_hash
) {
1486 if (phd
->phd_port
== lport
)
1490 pr
= cred
? cred
->cr_prison
: NULL
;
1493 * Port is in use by one or more PCBs. Look for best
1496 * If in a prison we may wish to allow the jail to override
1497 * a wildcard listen on the host. Since the jail forces its
1498 * own wildcard listens to a specific set of jail IPs, this
1499 * override allows most services on the host to remain as
1500 * they were and still be 'jail friendly'.
1502 LIST_FOREACH(inp
, &phd
->phd_pcblist
, inp_portlist
) {
1505 if (!INP_ISIPV4(inp
))
1508 if (inp
->inp_faddr
.s_addr
!= INADDR_ANY
)
1512 * Prison are independent of each other in terms
1513 * of allowing bindings. This can result in multiple
1514 * overloaded bindings which in_pcblookup_pkthash()
1515 * will have to sort out.
1517 * Allow wildcarded entries to co-exist with specific
1518 * entries. Specific entries override wildcarded
1521 if (inp
->inp_socket
&& inp
->inp_socket
->so_cred
)
1522 pscan
= inp
->inp_socket
->so_cred
->cr_prison
;
1527 if (inp
->inp_laddr
.s_addr
== INADDR_ANY
) {
1528 if (laddr
.s_addr
!= INADDR_ANY
)
1531 if (laddr
.s_addr
== INADDR_ANY
)
1533 else if (inp
->inp_laddr
.s_addr
!= laddr
.s_addr
)
1536 if (wildcard
&& !wild_okay
)
1538 if (wildcard
< matchwild
) {
1540 matchwild
= wildcard
;
1550 in_pcblocalgroup_last(const struct inpcbinfo
*pcbinfo
,
1551 const struct inpcb
*inp
)
1553 const struct inp_localgrphead
*hdr
;
1554 const struct inp_localgroup
*grp
;
1557 if (pcbinfo
->localgrphashbase
== NULL
)
1560 GET_PCBINFO_TOKEN(pcbinfo
);
1562 hdr
= &pcbinfo
->localgrphashbase
[
1563 INP_PCBLOCALGRPHASH(inp
->inp_lport
, pcbinfo
->localgrphashmask
)];
1565 LIST_FOREACH(grp
, hdr
, il_list
) {
1566 if (grp
->il_af
== inp
->inp_af
&&
1567 grp
->il_lport
== inp
->inp_lport
&&
1568 memcmp(&grp
->il_dependladdr
,
1569 &inp
->inp_inc
.inc_ie
.ie_dependladdr
,
1570 sizeof(grp
->il_dependladdr
)) == 0) {
1574 if (grp
== NULL
|| grp
->il_inpcnt
== 1) {
1575 REL_PCBINFO_TOKEN(pcbinfo
);
1579 KASSERT(grp
->il_inpcnt
>= 2,
1580 ("invalid localgroup inp count %d", grp
->il_inpcnt
));
1581 for (i
= 0; i
< grp
->il_inpcnt
; ++i
) {
1582 if (grp
->il_inp
[i
] == inp
) {
1583 int last
= grp
->il_inpcnt
- 1;
1586 last
= grp
->il_inpcnt
- 2;
1587 REL_PCBINFO_TOKEN(pcbinfo
);
1588 return grp
->il_inp
[last
];
1591 REL_PCBINFO_TOKEN(pcbinfo
);
1595 static struct inpcb
*
1596 inp_localgroup_lookup(const struct inpcbinfo
*pcbinfo
,
1597 struct in_addr laddr
, uint16_t lport
, uint32_t pkt_hash
)
1599 struct inpcb
*local_wild
;
1601 struct inpcb
*jinp_wild
;
1603 const struct inp_localgrphead
*hdr
;
1604 const struct inp_localgroup
*grp
;
1605 struct sockaddr_in jsin
;
1609 int net_listen_ov_local
;
1610 int net_listen_ov_wild
;
1612 ASSERT_PCBINFO_TOKEN_HELD(pcbinfo
);
1614 hdr
= &pcbinfo
->localgrphashbase
[
1615 INP_PCBLOCALGRPHASH(lport
, pcbinfo
->localgrphashmask
)];
1618 * Order of socket selection:
1622 * NOTE: Local group does not contain jailed sockets
1624 jsin
.sin_family
= AF_INET
;
1625 jsin
.sin_addr
.s_addr
= laddr
.s_addr
;
1630 net_listen_ov_local
= 0;
1631 net_listen_ov_wild
= 0;
1633 LIST_FOREACH(grp
, hdr
, il_list
) {
1635 if (grp
->il_af
!= AF_INET
)
1638 if (grp
->il_lport
!= lport
)
1644 idx
= netisr_hashlsb(pkt_hash
) % grp
->il_inpcnt
;
1645 inp
= grp
->il_inp
[idx
];
1648 * Modulo-N is used here, which greatly reduces
1649 * completion queue token contention, thus more
1650 * cpu time is saved.
1652 if (grp
->il_jailed
) {
1653 if (inp
->inp_socket
== NULL
)
1655 cred
= inp
->inp_socket
->so_cred
;
1658 pr
= cred
->cr_prison
;
1661 if (!jailed_ip(pr
, (struct sockaddr
*)&jsin
))
1663 if (grp
->il_laddr
.s_addr
== laddr
.s_addr
) {
1665 if (PRISON_CAP_ISSET(pr
->pr_caps
, PRISON_CAP_NET_LISTEN_OVERRIDE
))
1666 net_listen_ov_local
= 1;
1668 } else if (grp
->il_laddr
.s_addr
== INADDR_ANY
&&
1669 jinp_wild
== NULL
) {
1671 if (PRISON_CAP_ISSET(pr
->pr_caps
, PRISON_CAP_NET_LISTEN_OVERRIDE
))
1672 net_listen_ov_wild
= 1;
1675 if (grp
->il_laddr
.s_addr
== laddr
.s_addr
) {
1677 } else if (grp
->il_laddr
.s_addr
== INADDR_ANY
) {
1683 if (net_listen_ov_local
)
1685 if (net_listen_ov_wild
)
1688 return (local_wild
);
1695 * Lookup PCB in hash list.
1697 * This is used to match incoming packets to a pcb
1700 in_pcblookup_pkthash(struct inpcbinfo
*pcbinfo
, struct in_addr faddr
,
1701 u_int fport_arg
, struct in_addr laddr
, u_int lport_arg
,
1702 boolean_t wildcard
, struct ifnet
*ifp
, const struct mbuf
*m
)
1704 struct inpcbhead
*head
;
1705 struct inpcb
*inp
, *jinp
=NULL
;
1706 u_short fport
= fport_arg
, lport
= lport_arg
;
1709 * First look for an exact match.
1711 head
= &pcbinfo
->hashbase
[INP_PCBCONNHASH(faddr
.s_addr
, fport
,
1712 laddr
.s_addr
, lport
,
1713 pcbinfo
->hashmask
)];
1714 LIST_FOREACH(inp
, head
, inp_hash
) {
1716 if (!INP_ISIPV4(inp
))
1719 if (in_hosteq(inp
->inp_faddr
, faddr
) &&
1720 in_hosteq(inp
->inp_laddr
, laddr
) &&
1721 inp
->inp_fport
== fport
&& inp
->inp_lport
== lport
) {
1723 * Found specific address, host overrides jailed
1726 if (inp
->inp_socket
== NULL
||
1727 inp
->inp_socket
->so_cred
->cr_prison
== NULL
) {
1738 * We generally get here for connections to wildcarded listeners.
1739 * Any wildcarded listeners in jails must be restricted to the
1743 struct inpcb
*local_wild
= NULL
;
1744 struct inpcb
*jinp_wild
= NULL
;
1745 struct inpcontainer
*ic
;
1746 struct inpcontainerhead
*chead
;
1747 struct sockaddr_in jsin
;
1750 int net_listen_ov_local
= 0;
1751 int net_listen_ov_wild
= 0;
1753 GET_PCBINFO_TOKEN(pcbinfo
);
1756 * Check local group first. When present, the localgroup
1757 * hash utilizes the same non-jailed-vs/jailed priortization
1758 * that the normal wildcardhash does.
1760 if (pcbinfo
->localgrphashbase
!= NULL
&&
1761 m
!= NULL
&& (m
->m_flags
& M_HASH
)) {
1762 inp
= inp_localgroup_lookup(pcbinfo
, laddr
, lport
,
1765 REL_PCBINFO_TOKEN(pcbinfo
);
1771 * Order of socket selection:
1773 * 1. non-jailed, non-wild.
1774 * 2. non-jailed, wild. (allow_listen_override on)
1775 * 3. jailed, non-wild.
1777 * 5. non-jailed, wild. (allow_listen_override off)
1779 * NOTE: jailed wildcards are still restricted to the jail
1782 * NOTE: (1) and (3) already handled above.
1784 jsin
.sin_family
= AF_INET
;
1785 chead
= &pcbinfo
->wildcardhashbase
[
1786 INP_PCBWILDCARDHASH(lport
, pcbinfo
->wildcardhashmask
)];
1788 LIST_FOREACH(ic
, chead
, ic_list
) {
1790 if (inp
->inp_flags
& INP_PLACEMARKER
)
1797 if (!INP_ISIPV4(inp
))
1800 if (inp
->inp_lport
!= lport
)
1804 * Calculate prison, setup jsin for jailed_ip()
1807 jsin
.sin_addr
.s_addr
= laddr
.s_addr
;
1810 if (inp
->inp_socket
) {
1811 cred
= inp
->inp_socket
->so_cred
;
1813 pr
= cred
->cr_prison
;
1817 * Assign jinp, jinp_wild, and local_wild as
1818 * appropriate, track whether the jail supports
1822 if (!jailed_ip(pr
, (struct sockaddr
*)&jsin
))
1824 if (inp
->inp_laddr
.s_addr
== laddr
.s_addr
&&
1827 if (PRISON_CAP_ISSET(pr
->pr_caps
, PRISON_CAP_NET_LISTEN_OVERRIDE
))
1828 net_listen_ov_local
= 1;
1830 if (inp
->inp_laddr
.s_addr
== INADDR_ANY
&&
1831 jinp_wild
== NULL
) {
1833 if (PRISON_CAP_ISSET(pr
->pr_caps
, PRISON_CAP_NET_LISTEN_OVERRIDE
))
1834 net_listen_ov_wild
= 1;
1837 if (inp
->inp_laddr
.s_addr
== laddr
.s_addr
) {
1838 REL_PCBINFO_TOKEN(pcbinfo
);
1841 if (inp
->inp_laddr
.s_addr
== INADDR_ANY
)
1846 REL_PCBINFO_TOKEN(pcbinfo
);
1848 if (net_listen_ov_local
)
1850 if (net_listen_ov_wild
)
1853 return (local_wild
);
1866 in_pcblookup_hash(struct inpcbinfo
*pcbinfo
, struct in_addr faddr
,
1867 u_int fport_arg
, struct in_addr laddr
, u_int lport_arg
,
1868 boolean_t wildcard
, struct ifnet
*ifp
)
1870 return in_pcblookup_pkthash(pcbinfo
, faddr
, fport_arg
,
1871 laddr
, lport_arg
, wildcard
, ifp
, NULL
);
1875 * Insert PCB into connection hash table.
1878 in_pcbinsconnhash(struct inpcb
*inp
)
1880 struct inpcbinfo
*pcbinfo
= inp
->inp_pcbinfo
;
1881 struct inpcbhead
*bucket
;
1882 u_int32_t hashkey_faddr
, hashkey_laddr
;
1885 if (INP_ISIPV6(inp
)) {
1886 hashkey_faddr
= inp
->in6p_faddr
.s6_addr32
[3] /* XXX JH */;
1887 hashkey_laddr
= inp
->in6p_laddr
.s6_addr32
[3] /* XXX JH */;
1890 hashkey_faddr
= inp
->inp_faddr
.s_addr
;
1891 hashkey_laddr
= inp
->inp_laddr
.s_addr
;
1896 KASSERT(&curthread
->td_msgport
== netisr_cpuport(pcbinfo
->cpu
),
1897 ("not in the correct netisr"));
1898 ASSERT_INP_NOTINHASH(inp
);
1899 inp
->inp_flags
|= INP_CONNECTED
;
1902 * Insert into the connection hash table.
1904 bucket
= &pcbinfo
->hashbase
[INP_PCBCONNHASH(hashkey_faddr
,
1905 inp
->inp_fport
, hashkey_laddr
, inp
->inp_lport
, pcbinfo
->hashmask
)];
1906 LIST_INSERT_HEAD(bucket
, inp
, inp_hash
);
1910 * Remove PCB from connection hash table.
1913 in_pcbremconnhash(struct inpcb
*inp
)
1915 struct inpcbinfo
*pcbinfo __debugvar
= inp
->inp_pcbinfo
;
1917 KASSERT(&curthread
->td_msgport
== netisr_cpuport(pcbinfo
->cpu
),
1918 ("not in the correct netisr"));
1919 KASSERT(inp
->inp_flags
& INP_CONNECTED
, ("inp not connected"));
1921 LIST_REMOVE(inp
, inp_hash
);
1922 inp
->inp_flags
&= ~INP_CONNECTED
;
1926 * Insert PCB into port hash table.
1929 in_pcbinsporthash(struct inpcbporthead
*pcbporthash
, struct inpcb
*inp
)
1931 struct inpcbinfo
*pcbinfo
= inp
->inp_pcbinfo
;
1932 struct inpcbport
*phd
;
1935 * If the porthashbase is shared across several cpus, it must
1938 ASSERT_PORTHASH_TOKEN_HELD(pcbporthash
);
1941 * Insert into the port hash table.
1944 /* Go through port list and look for a head for this lport. */
1945 LIST_FOREACH(phd
, pcbporthash
, phd_hash
) {
1946 if (phd
->phd_port
== inp
->inp_lport
)
1950 /* If none exists, use saved one and tack it on. */
1952 KKASSERT(pcbinfo
->portsave
!= NULL
);
1953 phd
= pcbinfo
->portsave
;
1954 pcbinfo
->portsave
= NULL
;
1955 phd
->phd_port
= inp
->inp_lport
;
1956 LIST_INIT(&phd
->phd_pcblist
);
1957 LIST_INSERT_HEAD(pcbporthash
, phd
, phd_hash
);
1960 inp
->inp_porthash
= pcbporthash
;
1962 LIST_INSERT_HEAD(&phd
->phd_pcblist
, inp
, inp_portlist
);
1965 * Malloc one inpcbport for later use. It is safe to use
1966 * "wait" malloc here (port token would be released, if
1967 * malloc ever blocked), since all changes to the porthash
1970 if (pcbinfo
->portsave
== NULL
) {
1971 pcbinfo
->portsave
= kmalloc(sizeof(*pcbinfo
->portsave
),
1972 M_PCB
, M_INTWAIT
| M_ZERO
);
1977 in_pcbinsporthash_lport(struct inpcb
*inp
)
1979 struct inpcbporthead
*porthash
;
1981 porthash
= OBTAIN_LPORTHASH_TOKEN(inp
->inp_pcbinfo
, inp
->inp_lport
);
1982 in_pcbinsporthash(porthash
, inp
);
1983 REL_PORTHASH_TOKEN(porthash
);
1987 in_pcbremporthash(struct inpcb
*inp
)
1989 struct inpcbporthead
*porthash
;
1990 struct inpcbport
*phd
;
1992 if (inp
->inp_phd
== NULL
)
1994 KASSERT(inp
->inp_lport
!= 0, ("inpcb has no lport"));
1996 porthash
= inp
->inp_porthash
;
1997 KASSERT(porthash
!= NULL
, ("no porthash"));
1999 GET_PORTHASH_TOKEN(porthash
);
2002 LIST_REMOVE(inp
, inp_portlist
);
2003 if (LIST_FIRST(&phd
->phd_pcblist
) == NULL
) {
2004 LIST_REMOVE(phd
, phd_hash
);
2008 REL_PORTHASH_TOKEN(porthash
);
2010 inp
->inp_phd
= NULL
;
2011 /* NOTE: Don't whack inp_lport, which may be used later */
2014 static struct inp_localgroup
*
2015 inp_localgroup_alloc(u_char af
, uint16_t port
,
2016 const union in_dependaddr
*addr
, int size
)
2018 struct inp_localgroup
*grp
;
2020 grp
= kmalloc(__offsetof(struct inp_localgroup
, il_inp
[size
]),
2021 M_TEMP
, M_INTWAIT
| M_ZERO
);
2023 grp
->il_lport
= port
;
2024 grp
->il_dependladdr
= *addr
;
2025 grp
->il_inpsiz
= size
;
2031 inp_localgroup_free(struct inp_localgroup
*grp
)
2037 inp_localgroup_destroy(struct inp_localgroup
*grp
)
2039 LIST_REMOVE(grp
, il_list
);
2040 inp_localgroup_free(grp
);
2044 inp_localgroup_copy(struct inp_localgroup
*grp
,
2045 const struct inp_localgroup
*old_grp
)
2049 KASSERT(old_grp
->il_inpcnt
< grp
->il_inpsiz
,
2050 ("invalid new local group size %d and old local group count %d",
2051 grp
->il_inpsiz
, old_grp
->il_inpcnt
));
2052 for (i
= 0; i
< old_grp
->il_inpcnt
; ++i
)
2053 grp
->il_inp
[i
] = old_grp
->il_inp
[i
];
2054 grp
->il_inpcnt
= old_grp
->il_inpcnt
;
2058 in_pcbinslocalgrphash_oncpu(struct inpcb
*inp
, struct inpcbinfo
*pcbinfo
)
2060 struct inp_localgrphead
*hdr
;
2061 struct inp_localgroup
*grp
, *grp_alloc
= NULL
;
2065 ASSERT_PCBINFO_TOKEN_HELD(pcbinfo
);
2067 if (pcbinfo
->localgrphashbase
== NULL
)
2071 * Further separate groups by whether the inp is jailed or not.
2072 * This allows the inp_localgroup_lookup() code to manage port
2073 * overloading between jails and non-jails.
2075 * XXX all jails are collected into one group, which works fine
2076 * as we expect the jails to be listening on different addresses.
2077 * If this changes in the future we may have to break the groups
2078 * up by prison pointer as well.
2080 if (inp
->inp_socket
&& inp
->inp_socket
->so_cred
)
2081 isjailed
= jailed(inp
->inp_socket
->so_cred
);
2085 hdr
= &pcbinfo
->localgrphashbase
[
2086 INP_PCBLOCALGRPHASH(inp
->inp_lport
, pcbinfo
->localgrphashmask
)];
2089 LIST_FOREACH(grp
, hdr
, il_list
) {
2090 if (grp
->il_af
== inp
->inp_af
&&
2091 grp
->il_lport
== inp
->inp_lport
&&
2092 grp
->il_jailed
== isjailed
&&
2093 memcmp(&grp
->il_dependladdr
,
2094 &inp
->inp_inc
.inc_ie
.ie_dependladdr
,
2095 sizeof(grp
->il_dependladdr
)) == 0) {
2101 * Create a new local group
2103 if (grp_alloc
== NULL
) {
2104 grp_alloc
= inp_localgroup_alloc(inp
->inp_af
,
2105 inp
->inp_lport
, &inp
->inp_inc
.inc_ie
.ie_dependladdr
,
2106 INP_LOCALGROUP_SIZMIN
);
2108 * Local group allocation could block and the
2109 * local group w/ the same property might have
2110 * been added by others when we were blocked;
2115 /* Local group has been allocated; link it */
2117 grp
->il_jailed
= isjailed
;
2119 LIST_INSERT_HEAD(hdr
, grp
, il_list
);
2121 } else if (grp
->il_inpcnt
== grp
->il_inpsiz
) {
2124 * REMOVED - Ensure that all entries are placed in the
2125 * localgroup so jail operations can be
2126 * deterministic on a il_lport basis.
2128 if (grp
->il_inpsiz
>= INP_LOCALGROUP_SIZMAX
) {
2129 static int limit_logged
= 0;
2131 if (!limit_logged
) {
2133 kprintf("local group port %d, "
2134 "limit reached\n", ntohs(grp
->il_lport
));
2136 if (grp_alloc
!= NULL
) {
2138 * This would happen if the local group
2139 * w/ the same property was expanded when
2140 * our local group allocation blocked.
2142 inp_localgroup_free(grp_alloc
);
2149 * Expand this local group
2151 if (grp_alloc
== NULL
||
2152 grp
->il_inpcnt
>= grp_alloc
->il_inpsiz
) {
2153 if (grp_alloc
!= NULL
)
2154 inp_localgroup_free(grp_alloc
);
2155 grp_alloc
= inp_localgroup_alloc(grp
->il_af
,
2156 grp
->il_lport
, &grp
->il_dependladdr
,
2157 grp
->il_inpsiz
* 2);
2159 * Local group allocation could block and the
2160 * local group w/ the same property might have
2161 * been expanded by others when we were blocked;
2168 * Save the old local group, link the new one, and then
2169 * destroy the old local group
2171 inp_localgroup_copy(grp_alloc
, grp
);
2172 LIST_INSERT_HEAD(hdr
, grp_alloc
, il_list
);
2173 inp_localgroup_destroy(grp
);
2176 grp
->il_jailed
= isjailed
;
2180 * Found the local group
2182 if (grp_alloc
!= NULL
) {
2184 * This would happen if the local group w/ the
2185 * same property was added or expanded when our
2186 * local group allocation blocked.
2188 inp_localgroup_free(grp_alloc
);
2193 KASSERT(grp
->il_inpcnt
< grp
->il_inpsiz
,
2194 ("invalid local group size %d and count %d",
2195 grp
->il_inpsiz
, grp
->il_inpcnt
));
2198 * Keep the local group sorted by the inpcb local group index
2199 * in ascending order.
2201 * This eases the multi-process userland application which uses
2202 * SO_REUSEPORT sockets and binds process to the owner cpu of
2203 * the SO_REUSEPORT socket:
2204 * If we didn't sort the local group by the inpcb local group
2205 * index and one of the process owning an inpcb in this local
2206 * group restarted, e.g. crashed and restarted by watchdog,
2207 * other processes owning a inpcb in this local group would have
2208 * to detect that event, refetch its socket's owner cpu, and
2211 idx
= grp
->il_inpcnt
;
2212 for (i
= 0; i
< idx
; ++i
) {
2213 struct inpcb
*oinp
= grp
->il_inp
[i
];
2215 if (oinp
->inp_lgrpindex
> i
) {
2216 if (inp
->inp_lgrpindex
< 0) {
2217 inp
->inp_lgrpindex
= i
;
2218 } else if (inp
->inp_lgrpindex
!= i
) {
2220 kprintf("inp %p: grpidx %d, "
2221 "assigned to %d, cpu%d\n",
2222 inp
, inp
->inp_lgrpindex
, i
,
2226 grp
->il_inp
[i
] = inp
;
2228 /* Pull down inpcbs */
2229 for (; i
< grp
->il_inpcnt
; ++i
) {
2230 struct inpcb
*oinp1
= grp
->il_inp
[i
+ 1];
2232 grp
->il_inp
[i
+ 1] = oinp
;
2240 if (inp
->inp_lgrpindex
< 0) {
2241 inp
->inp_lgrpindex
= idx
;
2242 } else if (inp
->inp_lgrpindex
!= idx
) {
2244 kprintf("inp %p: grpidx %d, assigned to %d, cpu%d\n",
2245 inp
, inp
->inp_lgrpindex
, idx
, mycpuid
);
2248 grp
->il_inp
[idx
] = inp
;
2253 in_pcbinswildcardhash_oncpu(struct inpcb
*inp
, struct inpcbinfo
*pcbinfo
)
2255 struct inpcontainer
*ic
;
2256 struct inpcontainerhead
*bucket
;
2258 GET_PCBINFO_TOKEN(pcbinfo
);
2260 in_pcbinslocalgrphash_oncpu(inp
, pcbinfo
);
2262 bucket
= &pcbinfo
->wildcardhashbase
[
2263 INP_PCBWILDCARDHASH(inp
->inp_lport
, pcbinfo
->wildcardhashmask
)];
2265 ic
= kmalloc(sizeof(struct inpcontainer
), M_TEMP
, M_INTWAIT
);
2267 LIST_INSERT_HEAD(bucket
, ic
, ic_list
);
2269 REL_PCBINFO_TOKEN(pcbinfo
);
2273 * Insert PCB into wildcard hash table.
2276 in_pcbinswildcardhash(struct inpcb
*inp
)
2278 struct inpcbinfo
*pcbinfo
= inp
->inp_pcbinfo
;
2280 KASSERT(&curthread
->td_msgport
== netisr_cpuport(pcbinfo
->cpu
),
2281 ("not in correct netisr"));
2282 ASSERT_INP_NOTINHASH(inp
);
2283 inp
->inp_flags
|= INP_WILDCARD
;
2285 in_pcbinswildcardhash_oncpu(inp
, pcbinfo
);
2289 in_pcbremlocalgrphash_oncpu(struct inpcb
*inp
, struct inpcbinfo
*pcbinfo
)
2291 struct inp_localgrphead
*hdr
;
2292 struct inp_localgroup
*grp
;
2294 ASSERT_PCBINFO_TOKEN_HELD(pcbinfo
);
2296 if (pcbinfo
->localgrphashbase
== NULL
)
2299 hdr
= &pcbinfo
->localgrphashbase
[
2300 INP_PCBLOCALGRPHASH(inp
->inp_lport
, pcbinfo
->localgrphashmask
)];
2302 LIST_FOREACH(grp
, hdr
, il_list
) {
2305 for (i
= 0; i
< grp
->il_inpcnt
; ++i
) {
2306 if (grp
->il_inp
[i
] != inp
)
2309 if (grp
->il_inpcnt
== 1) {
2310 /* Destroy this local group */
2311 inp_localgroup_destroy(grp
);
2313 /* Pull up inpcbs */
2314 for (; i
+ 1 < grp
->il_inpcnt
; ++i
)
2315 grp
->il_inp
[i
] = grp
->il_inp
[i
+ 1];
2324 in_pcbremwildcardhash_oncpu(struct inpcb
*inp
, struct inpcbinfo
*pcbinfo
)
2326 struct inpcontainer
*ic
;
2327 struct inpcontainerhead
*head
;
2329 GET_PCBINFO_TOKEN(pcbinfo
);
2331 in_pcbremlocalgrphash_oncpu(inp
, pcbinfo
);
2334 head
= &pcbinfo
->wildcardhashbase
[
2335 INP_PCBWILDCARDHASH(inp
->inp_lport
, pcbinfo
->wildcardhashmask
)];
2337 LIST_FOREACH(ic
, head
, ic_list
) {
2338 if (ic
->ic_inp
== inp
)
2341 REL_PCBINFO_TOKEN(pcbinfo
);
2342 return; /* not found! */
2345 LIST_REMOVE(ic
, ic_list
); /* remove container from bucket chain */
2346 REL_PCBINFO_TOKEN(pcbinfo
);
2347 kfree(ic
, M_TEMP
); /* deallocate container */
2351 * Remove PCB from wildcard hash table.
2354 in_pcbremwildcardhash(struct inpcb
*inp
)
2356 struct inpcbinfo
*pcbinfo
= inp
->inp_pcbinfo
;
2358 KASSERT(&curthread
->td_msgport
== netisr_cpuport(pcbinfo
->cpu
),
2359 ("not in correct netisr"));
2360 KASSERT(inp
->inp_flags
& INP_WILDCARD
, ("inp not wildcard"));
2362 in_pcbremwildcardhash_oncpu(inp
, pcbinfo
);
2363 inp
->inp_lgrpindex
= -1;
2364 inp
->inp_flags
&= ~INP_WILDCARD
;
2368 * Remove PCB from various lists.
2371 in_pcbremlists(struct inpcb
*inp
)
2373 in_pcbremporthash(inp
);
2374 if (inp
->inp_flags
& INP_WILDCARD
) {
2375 in_pcbremwildcardhash(inp
);
2376 } else if (inp
->inp_flags
& INP_CONNECTED
) {
2377 in_pcbremconnhash(inp
);
2380 if (inp
->inp_flags
& INP_ONLIST
)
2385 prison_xinpcb(struct thread
*td
, struct inpcb
*inp
)
2389 if (td
->td_proc
== NULL
)
2391 cr
= td
->td_proc
->p_ucred
;
2392 if (cr
->cr_prison
== NULL
)
2394 if (inp
->inp_socket
&& inp
->inp_socket
->so_cred
&&
2395 inp
->inp_socket
->so_cred
->cr_prison
&&
2396 cr
->cr_prison
== inp
->inp_socket
->so_cred
->cr_prison
)
2402 in_pcblist_range(SYSCTL_HANDLER_ARGS
)
2404 struct inpcbinfo
*pcbinfo_arr
= arg1
;
2405 int pcbinfo_arrlen
= arg2
;
2406 struct inpcb
*marker
;
2410 KASSERT(pcbinfo_arrlen
<= netisr_ncpus
&& pcbinfo_arrlen
>= 1,
2411 ("invalid pcbinfo count %d", pcbinfo_arrlen
));
2414 * The process of preparing the TCB list is too time-consuming and
2415 * resource-intensive to repeat twice on every request.
2418 if (req
->oldptr
== NULL
) {
2419 for (cpu
= 0; cpu
< pcbinfo_arrlen
; ++cpu
)
2420 n
+= pcbinfo_arr
[cpu
].ipi_count
;
2421 req
->oldidx
= (n
+ n
/8 + 10) * sizeof(struct xinpcb
);
2425 if (req
->newptr
!= NULL
)
2428 marker
= kmalloc(sizeof(struct inpcb
), M_TEMP
, M_WAITOK
|M_ZERO
);
2429 marker
->inp_flags
|= INP_PLACEMARKER
;
2432 * OK, now we're committed to doing something. Re-fetch ipi_count
2433 * after obtaining the generation count.
2437 for (cpu
= 0; cpu
< pcbinfo_arrlen
&& error
== 0; ++cpu
) {
2438 struct inpcbinfo
*pcbinfo
= &pcbinfo_arr
[cpu
];
2443 lwkt_migratecpu(cpu
);
2445 GET_PCBINFO_TOKEN(pcbinfo
);
2447 n
= pcbinfo
->ipi_count
;
2449 LIST_INSERT_HEAD(&pcbinfo
->pcblisthead
, marker
, inp_list
);
2451 while ((inp
= LIST_NEXT(marker
, inp_list
)) != NULL
&& i
< n
) {
2452 LIST_REMOVE(marker
, inp_list
);
2453 LIST_INSERT_AFTER(inp
, marker
, inp_list
);
2455 if (inp
->inp_flags
& INP_PLACEMARKER
)
2457 if (prison_xinpcb(req
->td
, inp
))
2460 bzero(&xi
, sizeof xi
);
2461 xi
.xi_len
= sizeof xi
;
2462 bcopy(inp
, &xi
.xi_inp
, sizeof *inp
);
2463 if (inp
->inp_socket
)
2464 sotoxsocket(inp
->inp_socket
, &xi
.xi_socket
);
2465 if ((error
= SYSCTL_OUT(req
, &xi
, sizeof xi
)) != 0)
2469 LIST_REMOVE(marker
, inp_list
);
2471 REL_PCBINFO_TOKEN(pcbinfo
);
2473 if (error
== 0 && i
< n
) {
2474 bzero(&xi
, sizeof xi
);
2475 xi
.xi_len
= sizeof xi
;
2477 error
= SYSCTL_OUT(req
, &xi
, sizeof xi
);
2485 lwkt_migratecpu(origcpu
);
2486 kfree(marker
, M_TEMP
);
2491 in_pcblist_ncpus(SYSCTL_HANDLER_ARGS
)
2494 return (in_pcblist_range(oidp
, arg1
, netisr_ncpus
, req
));
2498 in_savefaddr(struct socket
*so
, const struct sockaddr
*faddr
)
2500 struct sockaddr_in
*sin
;
2502 KASSERT(faddr
->sa_family
== AF_INET
,
2503 ("not AF_INET faddr %d", faddr
->sa_family
));
2505 sin
= kmalloc(sizeof(*sin
), M_SONAME
, M_WAITOK
| M_ZERO
);
2506 sin
->sin_family
= AF_INET
;
2507 sin
->sin_len
= sizeof(*sin
);
2508 sin
->sin_port
= ((const struct sockaddr_in
*)faddr
)->sin_port
;
2509 sin
->sin_addr
= ((const struct sockaddr_in
*)faddr
)->sin_addr
;
2511 so
->so_faddr
= (struct sockaddr
*)sin
;
2515 in_pcbportinfo_init(struct inpcbportinfo
*portinfo
, int hashsize
,
2518 memset(portinfo
, 0, sizeof(*portinfo
));
2520 portinfo
->offset
= offset
;
2521 portinfo
->porthashbase
= phashinit(hashsize
, M_PCB
,
2522 &portinfo
->porthashcnt
);
2526 in_pcbportrange(u_short
*hi0
, u_short
*lo0
, u_short ofs
, u_short step
)
2536 hi
= rounddown(hi
, step
);
2541 lo
= roundup(lo
, step
);
2551 in_pcbglobalinit(void)
2555 in_pcbmarkers
= kmalloc(netisr_ncpus
* sizeof(struct inpcb
), M_PCB
,
2557 in_pcbcontainer_markers
=
2558 kmalloc(netisr_ncpus
* sizeof(struct inpcontainer
), M_PCB
,
2561 for (cpu
= 0; cpu
< netisr_ncpus
; ++cpu
) {
2562 struct inpcontainer
*ic
= &in_pcbcontainer_markers
[cpu
];
2563 struct inpcb
*marker
= &in_pcbmarkers
[cpu
];
2565 marker
->inp_flags
|= INP_PLACEMARKER
;
2566 ic
->ic_inp
= marker
;
2574 ASSERT_NETISR_NCPUS(mycpuid
);
2575 return &in_pcbmarkers
[mycpuid
];
2578 struct inpcontainer
*
2579 in_pcbcontainer_marker(void)
2582 ASSERT_NETISR_NCPUS(mycpuid
);
2583 return &in_pcbcontainer_markers
[mycpuid
];
2587 in_pcbresetroute(struct inpcb
*inp
)
2589 struct route
*ro
= &inp
->inp_route
;
2591 if (ro
->ro_rt
!= NULL
)
2593 bzero(ro
, sizeof(*ro
));