2 * Copyright (c) 2004 Jeffrey M. Hsu. All rights reserved.
3 * Copyright (c) 2004 The DragonFly Project. All rights reserved.
5 * This code is derived from software contributed to The DragonFly Project
8 * Redistribution and use in source and binary forms, with or without
9 * modification, are permitted provided that the following conditions
11 * 1. Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * 2. Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in the
15 * documentation and/or other materials provided with the distribution.
16 * 3. Neither the name of The DragonFly Project nor the names of its
17 * contributors may be used to endorse or promote products derived
18 * from this software without specific, prior written permission.
20 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
21 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
22 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
23 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
24 * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
25 * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
26 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
27 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
28 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
29 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
30 * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
35 * Copyright (c) 1982, 1986, 1991, 1993, 1995
36 * The Regents of the University of California. All rights reserved.
38 * Redistribution and use in source and binary forms, with or without
39 * modification, are permitted provided that the following conditions
41 * 1. Redistributions of source code must retain the above copyright
42 * notice, this list of conditions and the following disclaimer.
43 * 2. Redistributions in binary form must reproduce the above copyright
44 * notice, this list of conditions and the following disclaimer in the
45 * documentation and/or other materials provided with the distribution.
46 * 3. Neither the name of the University nor the names of its contributors
47 * may be used to endorse or promote products derived from this software
48 * without specific prior written permission.
50 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
51 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
52 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
53 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
54 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
55 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
56 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
57 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
58 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
59 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
62 * @(#)in_pcb.c 8.4 (Berkeley) 5/24/95
63 * $FreeBSD: src/sys/netinet/in_pcb.c,v 1.59.2.27 2004/01/02 04:06:42 ambrisko Exp $
66 #include "opt_ipsec.h"
67 #include "opt_inet6.h"
69 #include <sys/param.h>
70 #include <sys/systm.h>
71 #include <sys/malloc.h>
73 #include <sys/domain.h>
74 #include <sys/protosw.h>
75 #include <sys/socket.h>
76 #include <sys/socketvar.h>
80 #include <sys/kernel.h>
81 #include <sys/sysctl.h>
83 #include <sys/thread2.h>
84 #include <sys/socketvar2.h>
85 #include <sys/msgport2.h>
87 #include <machine/limits.h>
90 #include <net/if_types.h>
91 #include <net/route.h>
92 #include <net/netisr2.h>
93 #include <net/toeplitz2.h>
95 #include <netinet/in.h>
96 #include <netinet/in_pcb.h>
97 #include <netinet/in_var.h>
98 #include <netinet/ip_var.h>
100 #include <netinet/ip6.h>
101 #include <netinet6/ip6_var.h>
105 #include <netinet6/ipsec.h>
106 #include <netproto/key/key.h>
107 #include <netproto/ipsec/esp_var.h>
111 #if defined(IPSEC) || defined(IPSEC_ESP)
112 #error "Bad idea: don't compile with both IPSEC and FAST_IPSEC!"
115 #include <netproto/ipsec/ipsec.h>
116 #include <netproto/ipsec/key.h>
118 #endif /* FAST_IPSEC */
120 #define INP_LOCALGROUP_SIZMIN 8
121 #define INP_LOCALGROUP_SIZMAX 256
123 static struct inpcb
*in_pcblookup_local(struct inpcbporthead
*porthash
,
124 struct in_addr laddr
, u_int lport_arg
, int wild_okay
,
127 struct in_addr zeroin_addr
;
130 * These configure the range of local port addresses assigned to
131 * "unspecified" outgoing connections/packets/whatever.
133 int ipport_lowfirstauto
= IPPORT_RESERVED
- 1; /* 1023 */
134 int ipport_lowlastauto
= IPPORT_RESERVEDSTART
; /* 600 */
136 int ipport_firstauto
= IPPORT_RESERVED
; /* 1024 */
137 int ipport_lastauto
= IPPORT_USERRESERVED
; /* 5000 */
139 int ipport_hifirstauto
= IPPORT_HIFIRSTAUTO
; /* 49152 */
140 int ipport_hilastauto
= IPPORT_HILASTAUTO
; /* 65535 */
142 #define RANGECHK(var, min, max) \
143 if ((var) < (min)) { (var) = (min); } \
144 else if ((var) > (max)) { (var) = (max); }
146 int udpencap_enable
= 1; /* enabled by default */
147 int udpencap_port
= 4500; /* triggers decapsulation */
150 * Per-netisr inpcb markers.
151 * NOTE: they should only be used in netisrs.
153 static struct inpcb
*in_pcbmarkers
;
154 static struct inpcontainer
*in_pcbcontainer_markers
;
157 sysctl_net_ipport_check(SYSCTL_HANDLER_ARGS
)
161 error
= sysctl_handle_int(oidp
, oidp
->oid_arg1
, oidp
->oid_arg2
, req
);
163 RANGECHK(ipport_lowfirstauto
, 1, IPPORT_RESERVED
- 1);
164 RANGECHK(ipport_lowlastauto
, 1, IPPORT_RESERVED
- 1);
166 RANGECHK(ipport_firstauto
, IPPORT_RESERVED
, USHRT_MAX
);
167 RANGECHK(ipport_lastauto
, IPPORT_RESERVED
, USHRT_MAX
);
169 RANGECHK(ipport_hifirstauto
, IPPORT_RESERVED
, USHRT_MAX
);
170 RANGECHK(ipport_hilastauto
, IPPORT_RESERVED
, USHRT_MAX
);
177 SYSCTL_NODE(_net_inet_ip
, IPPROTO_IP
, portrange
, CTLFLAG_RW
, 0, "IP Ports");
179 SYSCTL_PROC(_net_inet_ip_portrange
, OID_AUTO
, lowfirst
, CTLTYPE_INT
|CTLFLAG_RW
,
180 &ipport_lowfirstauto
, 0, &sysctl_net_ipport_check
, "I", "");
181 SYSCTL_PROC(_net_inet_ip_portrange
, OID_AUTO
, lowlast
, CTLTYPE_INT
|CTLFLAG_RW
,
182 &ipport_lowlastauto
, 0, &sysctl_net_ipport_check
, "I", "");
183 SYSCTL_PROC(_net_inet_ip_portrange
, OID_AUTO
, first
, CTLTYPE_INT
|CTLFLAG_RW
,
184 &ipport_firstauto
, 0, &sysctl_net_ipport_check
, "I", "");
185 SYSCTL_PROC(_net_inet_ip_portrange
, OID_AUTO
, last
, CTLTYPE_INT
|CTLFLAG_RW
,
186 &ipport_lastauto
, 0, &sysctl_net_ipport_check
, "I", "");
187 SYSCTL_PROC(_net_inet_ip_portrange
, OID_AUTO
, hifirst
, CTLTYPE_INT
|CTLFLAG_RW
,
188 &ipport_hifirstauto
, 0, &sysctl_net_ipport_check
, "I", "");
189 SYSCTL_PROC(_net_inet_ip_portrange
, OID_AUTO
, hilast
, CTLTYPE_INT
|CTLFLAG_RW
,
190 &ipport_hilastauto
, 0, &sysctl_net_ipport_check
, "I", "");
192 static int ip_porthash_trycount
= 15;
193 SYSCTL_INT(_net_inet_ip
, OID_AUTO
, porthash_trycount
, CTLFLAG_RW
,
194 &ip_porthash_trycount
, 0,
195 "Number of tries to find local port matching hash of 4-tuple");
198 * in_pcb.c: manage the Protocol Control Blocks.
200 * NOTE: It is assumed that most of these functions will be called from
201 * a critical section. XXX - There are, unfortunately, a few exceptions
202 * to this rule that should be fixed.
204 * NOTE: The caller should initialize the cpu field to the cpu running the
205 * protocol stack associated with this inpcbinfo.
209 in_pcbinfo_init(struct inpcbinfo
*pcbinfo
, int cpu
, boolean_t shared
)
211 KASSERT(cpu
>= 0 && cpu
< ncpus
, ("invalid cpu%d", cpu
));
214 LIST_INIT(&pcbinfo
->pcblisthead
);
215 pcbinfo
->portsave
= kmalloc(sizeof(*pcbinfo
->portsave
), M_PCB
,
219 pcbinfo
->infotoken
= kmalloc(sizeof(struct lwkt_token
),
221 lwkt_token_init(pcbinfo
->infotoken
, "infotoken");
223 pcbinfo
->infotoken
= NULL
;
227 struct baddynamicports baddynamicports
;
230 * Check if the specified port is invalid for dynamic allocation.
233 in_baddynamic(u_int16_t port
, u_int16_t proto
)
237 return (DP_ISSET(baddynamicports
.tcp
, port
));
240 /* Cannot preset this as it is a sysctl */
241 if (port
== udpencap_port
)
244 return (DP_ISSET(baddynamicports
.udp
, port
));
251 in_pcbonlist(struct inpcb
*inp
)
253 struct inpcbinfo
*pcbinfo
= inp
->inp_pcbinfo
;
255 KASSERT(&curthread
->td_msgport
== netisr_cpuport(pcbinfo
->cpu
),
256 ("not in the correct netisr"));
257 KASSERT((inp
->inp_flags
& INP_ONLIST
) == 0, ("already on pcblist"));
258 inp
->inp_flags
|= INP_ONLIST
;
260 GET_PCBINFO_TOKEN(pcbinfo
);
261 LIST_INSERT_HEAD(&pcbinfo
->pcblisthead
, inp
, inp_list
);
262 pcbinfo
->ipi_count
++;
263 REL_PCBINFO_TOKEN(pcbinfo
);
267 in_pcbofflist(struct inpcb
*inp
)
269 struct inpcbinfo
*pcbinfo
= inp
->inp_pcbinfo
;
271 KASSERT(&curthread
->td_msgport
== netisr_cpuport(pcbinfo
->cpu
),
272 ("not in the correct netisr"));
273 KASSERT(inp
->inp_flags
& INP_ONLIST
, ("not on pcblist"));
274 inp
->inp_flags
&= ~INP_ONLIST
;
276 GET_PCBINFO_TOKEN(pcbinfo
);
277 LIST_REMOVE(inp
, inp_list
);
278 KASSERT(pcbinfo
->ipi_count
> 0,
279 ("invalid inpcb count %d", pcbinfo
->ipi_count
));
280 pcbinfo
->ipi_count
--;
281 REL_PCBINFO_TOKEN(pcbinfo
);
285 * Allocate a PCB and associate it with the socket.
288 in_pcballoc(struct socket
*so
, struct inpcbinfo
*pcbinfo
)
295 inp
= kmalloc(pcbinfo
->ipi_size
, M_PCB
, M_WAITOK
|M_ZERO
|M_NULLOK
);
298 inp
->inp_lgrpindex
= -1;
299 inp
->inp_gencnt
= ++pcbinfo
->ipi_gencnt
;
300 inp
->inp_pcbinfo
= pcbinfo
;
301 inp
->inp_socket
= so
;
303 error
= ipsec_init_policy(so
, &inp
->inp_sp
);
310 if (INP_CHECK_SOCKAF(so
, AF_INET6
)) {
311 if (ip6_auto_flowlabel
)
312 inp
->inp_flags
|= IN6P_AUTOFLOWLABEL
;
313 inp
->inp_af
= AF_INET6
;
316 inp
->inp_af
= AF_INET
;
325 * Unlink a pcb with the intention of moving it to another cpu with a
326 * different pcbinfo. While unlinked nothing should attempt to dereference
327 * inp_pcbinfo, NULL it out so we assert if it does.
330 in_pcbunlink_flags(struct inpcb
*inp
, struct inpcbinfo
*pcbinfo
, int flags
)
332 KASSERT(inp
->inp_pcbinfo
== pcbinfo
, ("pcbinfo mismatch"));
333 KASSERT((inp
->inp_flags
& (flags
| INP_CONNECTED
)) == 0,
337 inp
->inp_pcbinfo
= NULL
;
341 in_pcbunlink(struct inpcb
*inp
, struct inpcbinfo
*pcbinfo
)
343 in_pcbunlink_flags(inp
, pcbinfo
, INP_WILDCARD
);
347 * Relink a pcb into a new pcbinfo.
350 in_pcblink_flags(struct inpcb
*inp
, struct inpcbinfo
*pcbinfo
, int flags
)
352 KASSERT(inp
->inp_pcbinfo
== NULL
, ("has pcbinfo"));
353 KASSERT((inp
->inp_flags
& (flags
| INP_CONNECTED
)) == 0,
356 inp
->inp_pcbinfo
= pcbinfo
;
361 in_pcblink(struct inpcb
*inp
, struct inpcbinfo
*pcbinfo
)
363 return in_pcblink_flags(inp
, pcbinfo
, INP_WILDCARD
);
367 in_pcbporthash_update(struct inpcbportinfo
*portinfo
,
368 struct inpcb
*inp
, u_short lport
, struct ucred
*cred
, int wild
)
370 struct inpcbporthead
*porthash
;
373 * This has to be atomic. If the porthash is shared across multiple
374 * protocol threads, e.g. tcp and udp, then the token must be held.
376 porthash
= in_pcbporthash_head(portinfo
, lport
);
377 GET_PORTHASH_TOKEN(porthash
);
379 if (in_pcblookup_local(porthash
, inp
->inp_laddr
, lport
,
380 wild
, cred
) != NULL
) {
381 REL_PORTHASH_TOKEN(porthash
);
384 inp
->inp_lport
= lport
;
385 in_pcbinsporthash(porthash
, inp
);
387 REL_PORTHASH_TOKEN(porthash
);
392 in_pcbsetlport(struct inpcb
*inp
, int wild
, struct ucred
*cred
)
394 struct inpcbinfo
*pcbinfo
= inp
->inp_pcbinfo
;
395 struct inpcbportinfo
*portinfo
;
396 u_short first
, last
, lport
, step
;
399 int portinfo_first
, portinfo_idx
;
401 inp
->inp_flags
|= INP_ANONPORT
;
403 step
= pcbinfo
->portinfo_mask
+ 1;
404 portinfo_first
= mycpuid
& pcbinfo
->portinfo_mask
;
405 portinfo_idx
= portinfo_first
;
407 portinfo
= &pcbinfo
->portinfo
[portinfo_idx
];
409 if (inp
->inp_flags
& INP_HIGHPORT
) {
410 first
= ipport_hifirstauto
; /* sysctl */
411 last
= ipport_hilastauto
;
412 lastport
= &portinfo
->lasthi
;
413 } else if (inp
->inp_flags
& INP_LOWPORT
) {
416 priv_check_cred(cred
, PRIV_NETINET_RESERVEDPORT
, 0))) {
417 inp
->inp_laddr
.s_addr
= INADDR_ANY
;
420 first
= ipport_lowfirstauto
; /* 1023 */
421 last
= ipport_lowlastauto
; /* 600 */
422 lastport
= &portinfo
->lastlow
;
424 first
= ipport_firstauto
; /* sysctl */
425 last
= ipport_lastauto
;
426 lastport
= &portinfo
->lastport
;
430 * Simple check to ensure all ports are not used up causing
433 * We split the two cases (up and down) so that the direction
434 * is not being tested on each round of the loop.
440 in_pcbportrange(&first
, &last
, portinfo
->offset
, step
);
441 count
= (first
- last
) / step
;
444 if (count
-- < 0) { /* completely used? */
445 error
= EADDRNOTAVAIL
;
448 lport
= in_pcblastport_down(lastport
,
450 KKASSERT((lport
& pcbinfo
->portinfo_mask
) ==
452 lport
= htons(lport
);
454 if (in_pcbporthash_update(portinfo
, inp
, lport
,
464 in_pcbportrange(&last
, &first
, portinfo
->offset
, step
);
465 count
= (last
- first
) / step
;
468 if (count
-- < 0) { /* completely used? */
469 error
= EADDRNOTAVAIL
;
472 lport
= in_pcblastport_up(lastport
, first
, last
, step
);
473 KKASSERT((lport
& pcbinfo
->portinfo_mask
) ==
475 lport
= htons(lport
);
477 if (in_pcbporthash_update(portinfo
, inp
, lport
,
486 /* Try next portinfo */
488 portinfo_idx
&= pcbinfo
->portinfo_mask
;
489 if (portinfo_idx
!= portinfo_first
)
491 inp
->inp_laddr
.s_addr
= INADDR_ANY
;
497 in_pcbbind(struct inpcb
*inp
, struct sockaddr
*nam
, struct thread
*td
)
499 struct socket
*so
= inp
->inp_socket
;
500 struct sockaddr_in jsin
;
501 struct ucred
*cred
= NULL
;
504 if (TAILQ_EMPTY(&in_ifaddrheads
[mycpuid
])) /* XXX broken! */
505 return (EADDRNOTAVAIL
);
506 if (inp
->inp_lport
!= 0 || inp
->inp_laddr
.s_addr
!= INADDR_ANY
)
507 return (EINVAL
); /* already bound */
509 if (!(so
->so_options
& (SO_REUSEADDR
|SO_REUSEPORT
)))
510 wild
= 1; /* neither SO_REUSEADDR nor SO_REUSEPORT is set */
512 cred
= td
->td_proc
->p_ucred
;
515 struct sockaddr_in
*sin
= (struct sockaddr_in
*)nam
;
516 struct inpcbinfo
*pcbinfo
;
517 struct inpcbportinfo
*portinfo
;
518 struct inpcbporthead
*porthash
;
520 u_short lport
, lport_ho
;
521 int reuseport
= (so
->so_options
& SO_REUSEPORT
);
524 if (nam
->sa_len
!= sizeof *sin
)
528 * We should check the family, but old programs
529 * incorrectly fail to initialize it.
531 if (sin
->sin_family
!= AF_INET
)
532 return (EAFNOSUPPORT
);
534 if (!prison_replace_wildcards(td
, nam
))
537 lport
= sin
->sin_port
;
538 if (IN_MULTICAST(ntohl(sin
->sin_addr
.s_addr
))) {
540 * Treat SO_REUSEADDR as SO_REUSEPORT for multicast;
541 * allow complete duplication of binding if
542 * SO_REUSEPORT is set, or if SO_REUSEADDR is set
543 * and a multicast address is bound on both
544 * new and duplicated sockets.
546 if (so
->so_options
& SO_REUSEADDR
)
547 reuseport
= SO_REUSEADDR
| SO_REUSEPORT
;
548 } else if (sin
->sin_addr
.s_addr
!= INADDR_ANY
) {
549 sin
->sin_port
= 0; /* yech... */
550 bzero(&sin
->sin_zero
, sizeof sin
->sin_zero
);
551 if (ifa_ifwithaddr((struct sockaddr
*)sin
) == NULL
)
552 return (EADDRNOTAVAIL
);
555 inp
->inp_laddr
= sin
->sin_addr
;
557 jsin
.sin_family
= AF_INET
;
558 jsin
.sin_addr
.s_addr
= inp
->inp_laddr
.s_addr
;
559 if (!prison_replace_wildcards(td
, (struct sockaddr
*)&jsin
)) {
560 inp
->inp_laddr
.s_addr
= INADDR_ANY
;
563 inp
->inp_laddr
.s_addr
= jsin
.sin_addr
.s_addr
;
566 /* Auto-select local port */
567 return in_pcbsetlport(inp
, wild
, cred
);
569 lport_ho
= ntohs(lport
);
572 if (lport_ho
< IPPORT_RESERVED
&& cred
&&
574 priv_check_cred(cred
, PRIV_NETINET_RESERVEDPORT
, 0))) {
575 inp
->inp_laddr
.s_addr
= INADDR_ANY
;
580 * Locate the proper portinfo based on lport
582 pcbinfo
= inp
->inp_pcbinfo
;
584 &pcbinfo
->portinfo
[lport_ho
& pcbinfo
->portinfo_mask
];
585 KKASSERT((lport_ho
& pcbinfo
->portinfo_mask
) ==
589 * This has to be atomic. If the porthash is shared across
590 * multiple protocol threads, e.g. tcp and udp then the token
593 porthash
= in_pcbporthash_head(portinfo
, lport
);
594 GET_PORTHASH_TOKEN(porthash
);
596 if (so
->so_cred
->cr_uid
!= 0 &&
597 !IN_MULTICAST(ntohl(sin
->sin_addr
.s_addr
))) {
598 t
= in_pcblookup_local(porthash
, sin
->sin_addr
, lport
,
599 INPLOOKUP_WILDCARD
, cred
);
601 (so
->so_cred
->cr_uid
!=
602 t
->inp_socket
->so_cred
->cr_uid
)) {
603 inp
->inp_laddr
.s_addr
= INADDR_ANY
;
608 if (cred
&& !prison_replace_wildcards(td
, nam
)) {
609 inp
->inp_laddr
.s_addr
= INADDR_ANY
;
610 error
= EADDRNOTAVAIL
;
613 t
= in_pcblookup_local(porthash
, sin
->sin_addr
, lport
,
615 if (t
&& !(reuseport
& t
->inp_socket
->so_options
)) {
616 inp
->inp_laddr
.s_addr
= INADDR_ANY
;
620 inp
->inp_lport
= lport
;
621 in_pcbinsporthash(porthash
, inp
);
624 REL_PORTHASH_TOKEN(porthash
);
627 jsin
.sin_family
= AF_INET
;
628 jsin
.sin_addr
.s_addr
= inp
->inp_laddr
.s_addr
;
629 if (!prison_replace_wildcards(td
, (struct sockaddr
*)&jsin
)) {
630 inp
->inp_laddr
.s_addr
= INADDR_ANY
;
633 inp
->inp_laddr
.s_addr
= jsin
.sin_addr
.s_addr
;
635 return in_pcbsetlport(inp
, wild
, cred
);
639 static struct inpcb
*
640 in_pcblookup_localremote(struct inpcbporthead
*porthash
, struct in_addr laddr
,
641 u_short lport
, struct in_addr faddr
, u_short fport
, struct ucred
*cred
)
644 struct inpcbport
*phd
;
645 struct inpcb
*match
= NULL
;
648 * If the porthashbase is shared across several cpus, it must
651 ASSERT_PORTHASH_TOKEN_HELD(porthash
);
654 * Best fit PCB lookup.
656 * First see if this local port is in use by looking on the
659 LIST_FOREACH(phd
, porthash
, phd_hash
) {
660 if (phd
->phd_port
== lport
)
664 LIST_FOREACH(inp
, &phd
->phd_pcblist
, inp_portlist
) {
666 if (!INP_ISIPV4(inp
))
669 if (inp
->inp_laddr
.s_addr
!= INADDR_ANY
&&
670 inp
->inp_laddr
.s_addr
!= laddr
.s_addr
)
673 if (inp
->inp_faddr
.s_addr
!= INADDR_ANY
&&
674 inp
->inp_faddr
.s_addr
!= faddr
.s_addr
)
677 if (inp
->inp_fport
!= 0 && inp
->inp_fport
!= fport
)
682 inp
->inp_socket
->so_cred
->cr_prison
) {
692 in_pcbporthash_update4(struct inpcbportinfo
*portinfo
,
693 struct inpcb
*inp
, u_short lport
, const struct sockaddr_in
*sin
,
696 struct inpcbporthead
*porthash
;
699 * This has to be atomic. If the porthash is shared across multiple
700 * protocol threads, e.g. tcp and udp, then the token must be held.
702 porthash
= in_pcbporthash_head(portinfo
, lport
);
703 GET_PORTHASH_TOKEN(porthash
);
705 if (in_pcblookup_localremote(porthash
, inp
->inp_laddr
,
706 lport
, sin
->sin_addr
, sin
->sin_port
, cred
) != NULL
) {
707 REL_PORTHASH_TOKEN(porthash
);
710 inp
->inp_lport
= lport
;
711 in_pcbinsporthash(porthash
, inp
);
713 REL_PORTHASH_TOKEN(porthash
);
718 in_pcbbind_remote(struct inpcb
*inp
, const struct sockaddr
*remote
,
721 struct proc
*p
= td
->td_proc
;
723 const struct sockaddr_in
*sin
= (const struct sockaddr_in
*)remote
;
724 struct sockaddr_in jsin
;
725 struct inpcbinfo
*pcbinfo
= inp
->inp_pcbinfo
;
726 struct inpcbportinfo
*portinfo
;
727 struct ucred
*cred
= NULL
;
728 u_short first
, last
, lport
, step
;
729 int count
, error
, selfconn
;
730 int portinfo_first
, portinfo_idx
;
731 int hash_cpu
, hash_count
, hash_count0
;
732 uint32_t hash_base
= 0, hash
;
734 if (TAILQ_EMPTY(&in_ifaddrheads
[mycpuid
])) /* XXX broken! */
735 return (EADDRNOTAVAIL
);
737 KKASSERT(inp
->inp_laddr
.s_addr
!= INADDR_ANY
);
738 if (inp
->inp_lport
!= 0)
739 return (EINVAL
); /* already bound */
744 jsin
.sin_family
= AF_INET
;
745 jsin
.sin_addr
.s_addr
= inp
->inp_laddr
.s_addr
;
746 if (!prison_replace_wildcards(td
, (struct sockaddr
*)&jsin
)) {
747 inp
->inp_laddr
.s_addr
= INADDR_ANY
;
750 inp
->inp_laddr
.s_addr
= jsin
.sin_addr
.s_addr
;
752 hash_count0
= ip_porthash_trycount
;
753 if (hash_count0
> 0) {
754 hash_base
= toeplitz_piecemeal_addr(sin
->sin_addr
.s_addr
) ^
755 toeplitz_piecemeal_addr(inp
->inp_laddr
.s_addr
) ^
756 toeplitz_piecemeal_port(sin
->sin_port
);
761 inp
->inp_flags
|= INP_ANONPORT
;
763 step
= pcbinfo
->portinfo_mask
+ 1;
764 portinfo_first
= mycpuid
& pcbinfo
->portinfo_mask
;
765 portinfo_idx
= portinfo_first
;
767 hash_cpu
= portinfo_idx
& ncpus2_mask
;
768 portinfo
= &pcbinfo
->portinfo
[portinfo_idx
];
771 if (inp
->inp_flags
& INP_HIGHPORT
) {
772 first
= ipport_hifirstauto
; /* sysctl */
773 last
= ipport_hilastauto
;
774 lastport
= &portinfo
->lasthi
;
775 } else if (inp
->inp_flags
& INP_LOWPORT
) {
778 priv_check_cred(cred
, PRIV_NETINET_RESERVEDPORT
, 0))) {
779 inp
->inp_laddr
.s_addr
= INADDR_ANY
;
782 first
= ipport_lowfirstauto
; /* 1023 */
783 last
= ipport_lowlastauto
; /* 600 */
784 lastport
= &portinfo
->lastlow
;
786 first
= ipport_firstauto
; /* sysctl */
787 last
= ipport_lastauto
;
788 lastport
= &portinfo
->lastport
;
791 /* This could happen on loopback interface */
792 #define IS_SELFCONNECT(inp, lport, sin) \
793 (__predict_false((sin)->sin_port == (lport) && \
794 (sin)->sin_addr.s_addr == (inp)->inp_laddr.s_addr))
797 * Simple check to ensure all ports are not used up causing
800 * We split the two cases (up and down) so that the direction
801 * is not being tested on each round of the loop.
803 hash_count
= hash_count0
;
808 in_pcbportrange(&first
, &last
, portinfo
->offset
, step
);
809 count
= ((first
- last
) / step
) + hash_count
;
812 if (count
-- < 0) { /* completely used? */
813 error
= EADDRNOTAVAIL
;
817 lport
= in_pcblastport_down(lastport
, first
, last
,
819 KKASSERT((lport
& pcbinfo
->portinfo_mask
) ==
821 lport
= htons(lport
);
822 if (IS_SELFCONNECT(inp
, lport
, sin
)) {
824 ++count
; /* don't count this try */
833 toeplitz_piecemeal_port(lport
);
834 if ((hash
& ncpus2_mask
) != hash_cpu
&& hash_count
)
838 if (in_pcbporthash_update4(portinfo
,
839 inp
, lport
, sin
, cred
)) {
848 in_pcbportrange(&last
, &first
, portinfo
->offset
, step
);
849 count
= ((last
- first
) / step
) + hash_count
;
852 if (count
-- < 0) { /* completely used? */
853 error
= EADDRNOTAVAIL
;
857 lport
= in_pcblastport_up(lastport
, first
, last
, step
);
858 KKASSERT((lport
& pcbinfo
->portinfo_mask
) ==
860 lport
= htons(lport
);
861 if (IS_SELFCONNECT(inp
, lport
, sin
)) {
863 ++count
; /* don't count this try */
872 toeplitz_piecemeal_port(lport
);
873 if ((hash
& ncpus2_mask
) != hash_cpu
&& hash_count
)
877 if (in_pcbporthash_update4(portinfo
,
878 inp
, lport
, sin
, cred
)) {
885 #undef IS_SELFCONNECT
888 /* Try next portinfo */
890 portinfo_idx
&= pcbinfo
->portinfo_mask
;
891 if (portinfo_idx
!= portinfo_first
)
893 inp
->inp_laddr
.s_addr
= INADDR_ANY
;
899 * Transform old in_pcbconnect() into an inner subroutine for new
900 * in_pcbconnect(): Do some validity-checking on the remote
901 * address (in mbuf 'nam') and then determine local host address
902 * (i.e., which interface) to use to access that remote host.
904 * This preserves definition of in_pcbconnect(), while supporting a
905 * slightly different version for T/TCP. (This is more than
906 * a bit of a kludge, but cleaning up the internal interfaces would
907 * have forced minor changes in every protocol).
910 in_pcbladdr_find(struct inpcb
*inp
, struct sockaddr
*nam
,
911 struct sockaddr_in
**plocal_sin
, struct thread
*td
, int find
)
913 struct in_ifaddr
*ia
;
914 struct ucred
*cred
= NULL
;
915 struct sockaddr_in
*sin
= (struct sockaddr_in
*)nam
;
916 struct sockaddr
*jsin
;
917 int jailed
= 0, alloc_route
= 0;
919 if (nam
->sa_len
!= sizeof *sin
)
921 if (sin
->sin_family
!= AF_INET
)
922 return (EAFNOSUPPORT
);
923 if (sin
->sin_port
== 0)
924 return (EADDRNOTAVAIL
);
925 if (td
&& td
->td_proc
&& td
->td_proc
->p_ucred
)
926 cred
= td
->td_proc
->p_ucred
;
927 if (cred
&& cred
->cr_prison
)
929 if (!TAILQ_EMPTY(&in_ifaddrheads
[mycpuid
])) {
930 ia
= TAILQ_FIRST(&in_ifaddrheads
[mycpuid
])->ia
;
932 * If the destination address is INADDR_ANY,
933 * use the primary local address.
934 * If the supplied address is INADDR_BROADCAST,
935 * and the primary interface supports broadcast,
936 * choose the broadcast address for that interface.
938 if (sin
->sin_addr
.s_addr
== INADDR_ANY
)
939 sin
->sin_addr
= IA_SIN(ia
)->sin_addr
;
940 else if (sin
->sin_addr
.s_addr
== (u_long
)INADDR_BROADCAST
&&
941 (ia
->ia_ifp
->if_flags
& IFF_BROADCAST
))
942 sin
->sin_addr
= satosin(&ia
->ia_broadaddr
)->sin_addr
;
949 * If route is known or can be allocated now,
950 * our src addr is taken from the i/f, else punt.
951 * Note that we should check the address family of the cached
952 * destination, in case of sharing the cache with IPv6.
954 ro
= &inp
->inp_route
;
956 (!(ro
->ro_rt
->rt_flags
& RTF_UP
) ||
957 ro
->ro_dst
.sa_family
!= AF_INET
||
958 satosin(&ro
->ro_dst
)->sin_addr
.s_addr
!=
959 sin
->sin_addr
.s_addr
||
960 inp
->inp_socket
->so_options
& SO_DONTROUTE
)) {
964 if (!(inp
->inp_socket
->so_options
& SO_DONTROUTE
) && /*XXX*/
965 (ro
->ro_rt
== NULL
||
966 ro
->ro_rt
->rt_ifp
== NULL
)) {
967 /* No route yet, so try to acquire one */
968 bzero(&ro
->ro_dst
, sizeof(struct sockaddr_in
));
969 ro
->ro_dst
.sa_family
= AF_INET
;
970 ro
->ro_dst
.sa_len
= sizeof(struct sockaddr_in
);
971 ((struct sockaddr_in
*) &ro
->ro_dst
)->sin_addr
=
977 * If we found a route, use the address
978 * corresponding to the outgoing interface
979 * unless it is the loopback (in case a route
980 * to our address on another net goes to loopback).
983 !(ro
->ro_rt
->rt_ifp
->if_flags
& IFF_LOOPBACK
)) {
985 if (jailed_ip(cred
->cr_prison
,
986 ro
->ro_rt
->rt_ifa
->ifa_addr
)) {
987 ia
= ifatoia(ro
->ro_rt
->rt_ifa
);
990 ia
= ifatoia(ro
->ro_rt
->rt_ifa
);
994 u_short fport
= sin
->sin_port
;
997 ia
= ifatoia(ifa_ifwithdstaddr(sintosa(sin
)));
998 if (ia
&& jailed
&& !jailed_ip(cred
->cr_prison
,
999 sintosa(&ia
->ia_addr
)))
1002 ia
= ifatoia(ifa_ifwithnet(sintosa(sin
)));
1003 if (ia
&& jailed
&& !jailed_ip(cred
->cr_prison
,
1004 sintosa(&ia
->ia_addr
)))
1006 sin
->sin_port
= fport
;
1008 !TAILQ_EMPTY(&in_ifaddrheads
[mycpuid
]))
1009 ia
= TAILQ_FIRST(&in_ifaddrheads
[mycpuid
])->ia
;
1010 if (ia
&& jailed
&& !jailed_ip(cred
->cr_prison
,
1011 sintosa(&ia
->ia_addr
)))
1014 if (!jailed
&& ia
== NULL
)
1018 * If the destination address is multicast and an outgoing
1019 * interface has been set as a multicast option, use the
1020 * address of that interface as our source address.
1022 if (!jailed
&& IN_MULTICAST(ntohl(sin
->sin_addr
.s_addr
)) &&
1023 inp
->inp_moptions
!= NULL
) {
1024 struct ip_moptions
*imo
;
1027 imo
= inp
->inp_moptions
;
1028 if ((ifp
= imo
->imo_multicast_ifp
) != NULL
) {
1029 struct in_ifaddr_container
*iac
;
1033 &in_ifaddrheads
[mycpuid
], ia_link
) {
1034 if (iac
->ia
->ia_ifp
== ifp
) {
1044 * Don't do pcblookup call here; return interface in plocal_sin
1045 * and exit to caller, that will do the lookup.
1047 if (ia
== NULL
&& jailed
) {
1048 if ((jsin
= prison_get_nonlocal(
1049 cred
->cr_prison
, AF_INET
, NULL
)) != NULL
||
1050 (jsin
= prison_get_local(
1051 cred
->cr_prison
, AF_INET
, NULL
)) != NULL
) {
1052 *plocal_sin
= satosin(jsin
);
1054 /* IPv6 only Jail */
1058 *plocal_sin
= &ia
->ia_addr
;
1064 in_pcbresetroute(inp
);
1065 return (EADDRNOTAVAIL
);
1069 in_pcbladdr(struct inpcb
*inp
, struct sockaddr
*nam
,
1070 struct sockaddr_in
**plocal_sin
, struct thread
*td
)
1072 return in_pcbladdr_find(inp
, nam
, plocal_sin
, td
,
1073 (inp
->inp_laddr
.s_addr
== INADDR_ANY
));
1078 * Connect from a socket to a specified address.
1079 * Both address and port must be specified in argument sin.
1080 * If don't have a local address for this socket yet,
1084 in_pcbconnect(struct inpcb
*inp
, struct sockaddr
*nam
, struct thread
*td
)
1086 struct sockaddr_in
*if_sin
;
1087 struct sockaddr_in
*sin
= (struct sockaddr_in
*)nam
;
1090 /* Call inner routine to assign local interface address. */
1091 if ((error
= in_pcbladdr(inp
, nam
, &if_sin
, td
)) != 0)
1094 if (in_pcblookup_hash(inp
->inp_pcbinfo
, sin
->sin_addr
, sin
->sin_port
,
1095 inp
->inp_laddr
.s_addr
?
1096 inp
->inp_laddr
: if_sin
->sin_addr
,
1097 inp
->inp_lport
, FALSE
, NULL
) != NULL
) {
1098 return (EADDRINUSE
);
1100 if (inp
->inp_laddr
.s_addr
== INADDR_ANY
) {
1101 if (inp
->inp_lport
== 0) {
1102 error
= in_pcbbind(inp
, NULL
, td
);
1106 inp
->inp_laddr
= if_sin
->sin_addr
;
1108 inp
->inp_faddr
= sin
->sin_addr
;
1109 inp
->inp_fport
= sin
->sin_port
;
1110 in_pcbinsconnhash(inp
);
1115 in_pcbdisconnect(struct inpcb
*inp
)
1118 in_pcbremconnhash(inp
);
1119 inp
->inp_faddr
.s_addr
= INADDR_ANY
;
1124 in_pcbdetach(struct inpcb
*inp
)
1126 struct socket
*so
= inp
->inp_socket
;
1127 struct inpcbinfo
*ipi
= inp
->inp_pcbinfo
;
1130 ipsec4_delete_pcbpolicy(inp
);
1132 inp
->inp_gencnt
= ++ipi
->ipi_gencnt
;
1133 KKASSERT((so
->so_state
& SS_ASSERTINPROG
) == 0);
1134 in_pcbremlists(inp
);
1136 sofree(so
); /* remove pcb ref */
1137 if (inp
->inp_options
)
1138 m_free(inp
->inp_options
);
1139 if (inp
->inp_route
.ro_rt
)
1140 rtfree(inp
->inp_route
.ro_rt
);
1141 ip_freemoptions(inp
->inp_moptions
);
1146 * The socket may have an invalid PCB, i.e. NULL. For example, a TCP
1147 * socket received RST.
1150 in_setsockaddr(struct socket
*so
, struct sockaddr
**nam
)
1153 struct sockaddr_in
*sin
;
1155 KASSERT(curthread
->td_type
== TD_TYPE_NETISR
, ("not in netisr"));
1158 return (ECONNRESET
);
1160 sin
= kmalloc(sizeof *sin
, M_SONAME
, M_WAITOK
| M_ZERO
);
1161 sin
->sin_family
= AF_INET
;
1162 sin
->sin_len
= sizeof *sin
;
1163 sin
->sin_port
= inp
->inp_lport
;
1164 sin
->sin_addr
= inp
->inp_laddr
;
1166 *nam
= (struct sockaddr
*)sin
;
1171 in_setsockaddr_dispatch(netmsg_t msg
)
1175 error
= in_setsockaddr(msg
->base
.nm_so
, msg
->peeraddr
.nm_nam
);
1176 lwkt_replymsg(&msg
->lmsg
, error
);
1180 * The socket may have an invalid PCB, i.e. NULL. For example, a TCP
1181 * socket received RST.
1184 in_setpeeraddr(struct socket
*so
, struct sockaddr
**nam
)
1187 struct sockaddr_in
*sin
;
1189 KASSERT(curthread
->td_type
== TD_TYPE_NETISR
, ("not in netisr"));
1192 return (ECONNRESET
);
1194 sin
= kmalloc(sizeof *sin
, M_SONAME
, M_WAITOK
| M_ZERO
);
1195 sin
->sin_family
= AF_INET
;
1196 sin
->sin_len
= sizeof *sin
;
1197 sin
->sin_port
= inp
->inp_fport
;
1198 sin
->sin_addr
= inp
->inp_faddr
;
1200 *nam
= (struct sockaddr
*)sin
;
1205 in_setpeeraddr_dispatch(netmsg_t msg
)
1209 error
= in_setpeeraddr(msg
->base
.nm_so
, msg
->peeraddr
.nm_nam
);
1210 lwkt_replymsg(&msg
->lmsg
, error
);
1214 in_pcbnotifyall(struct inpcbinfo
*pcbinfo
, struct in_addr faddr
, int err
,
1215 inp_notify_t notify
)
1217 struct inpcb
*inp
, *marker
;
1219 KASSERT(&curthread
->td_msgport
== netisr_cpuport(pcbinfo
->cpu
),
1220 ("not in the correct netisr"));
1221 marker
= &in_pcbmarkers
[mycpuid
];
1225 * - If INP_PLACEMARKER is set we must ignore the rest of the
1226 * structure and skip it.
1227 * - It is safe to nuke inpcbs here, since we are in their own
1230 GET_PCBINFO_TOKEN(pcbinfo
);
1232 LIST_INSERT_HEAD(&pcbinfo
->pcblisthead
, marker
, inp_list
);
1233 while ((inp
= LIST_NEXT(marker
, inp_list
)) != NULL
) {
1234 LIST_REMOVE(marker
, inp_list
);
1235 LIST_INSERT_AFTER(inp
, marker
, inp_list
);
1237 if (inp
->inp_flags
& INP_PLACEMARKER
)
1240 if (!INP_ISIPV4(inp
))
1243 if (inp
->inp_faddr
.s_addr
!= faddr
.s_addr
||
1244 inp
->inp_socket
== NULL
)
1246 (*notify
)(inp
, err
); /* can remove inp from list! */
1248 LIST_REMOVE(marker
, inp_list
);
1250 REL_PCBINFO_TOKEN(pcbinfo
);
1254 in_pcbpurgeif0(struct inpcbinfo
*pcbinfo
, struct ifnet
*ifp
)
1256 struct inpcb
*inp
, *marker
;
1259 * We only need to make sure that we are in netisr0, where all
1260 * multicast operation happen. We could check inpcbinfo which
1261 * does not belong to netisr0 by holding the inpcbinfo's token.
1262 * In this case, the pcbinfo must be able to be shared, i.e.
1263 * pcbinfo->infotoken is not NULL.
1265 ASSERT_IN_NETISR(0);
1266 KASSERT(pcbinfo
->cpu
== 0 || pcbinfo
->infotoken
!= NULL
,
1267 ("pcbinfo could not be shared"));
1270 * Get a marker for the current netisr (netisr0).
1272 * It is possible that the multicast address deletion blocks,
1273 * which could cause temporary token releasing. So we use
1274 * inpcb marker here to get a coherent view of the inpcb list.
1276 * While, on the other hand, moptions are only added and deleted
1277 * in netisr0, so we would not see staled moption or miss moption
1278 * even if the token was released due to the blocking multicast
1281 marker
= &in_pcbmarkers
[mycpuid
];
1283 GET_PCBINFO_TOKEN(pcbinfo
);
1285 LIST_INSERT_HEAD(&pcbinfo
->pcblisthead
, marker
, inp_list
);
1286 while ((inp
= LIST_NEXT(marker
, inp_list
)) != NULL
) {
1287 struct ip_moptions
*imo
;
1289 LIST_REMOVE(marker
, inp_list
);
1290 LIST_INSERT_AFTER(inp
, marker
, inp_list
);
1292 if (inp
->inp_flags
& INP_PLACEMARKER
)
1294 imo
= inp
->inp_moptions
;
1295 if (INP_ISIPV4(inp
) && imo
!= NULL
) {
1299 * Unselect the outgoing interface if it is being
1302 if (imo
->imo_multicast_ifp
== ifp
)
1303 imo
->imo_multicast_ifp
= NULL
;
1306 * Drop multicast group membership if we joined
1307 * through the interface being detached.
1309 for (i
= 0, gap
= 0; i
< imo
->imo_num_memberships
;
1311 if (imo
->imo_membership
[i
]->inm_ifp
== ifp
) {
1314 * This could block and the pcbinfo
1315 * token could be passively released.
1317 in_delmulti(imo
->imo_membership
[i
]);
1319 } else if (gap
!= 0)
1320 imo
->imo_membership
[i
- gap
] =
1321 imo
->imo_membership
[i
];
1323 imo
->imo_num_memberships
-= gap
;
1326 LIST_REMOVE(marker
, inp_list
);
1328 REL_PCBINFO_TOKEN(pcbinfo
);
1332 * Check for alternatives when higher level complains
1333 * about service problems. For now, invalidate cached
1334 * routing information. If the route was created dynamically
1335 * (by a redirect), time to try a default gateway again.
1338 in_losing(struct inpcb
*inp
)
1341 struct rt_addrinfo rtinfo
;
1343 if ((rt
= inp
->inp_route
.ro_rt
)) {
1344 bzero(&rtinfo
, sizeof(struct rt_addrinfo
));
1345 rtinfo
.rti_info
[RTAX_DST
] = rt_key(rt
);
1346 rtinfo
.rti_info
[RTAX_GATEWAY
] = rt
->rt_gateway
;
1347 rtinfo
.rti_info
[RTAX_NETMASK
] = rt_mask(rt
);
1348 rtinfo
.rti_flags
= rt
->rt_flags
;
1349 rt_missmsg(RTM_LOSING
, &rtinfo
, rt
->rt_flags
, 0);
1350 if (rt
->rt_flags
& RTF_DYNAMIC
) {
1351 rtrequest(RTM_DELETE
, rt_key(rt
), rt
->rt_gateway
,
1352 rt_mask(rt
), rt
->rt_flags
, NULL
);
1354 inp
->inp_route
.ro_rt
= NULL
;
1357 * A new route can be allocated
1358 * the next time output is attempted.
1364 * After a routing change, flush old routing
1365 * and allocate a (hopefully) better one.
1368 in_rtchange(struct inpcb
*inp
, int err
)
1370 if (inp
->inp_route
.ro_rt
) {
1371 rtfree(inp
->inp_route
.ro_rt
);
1372 inp
->inp_route
.ro_rt
= NULL
;
1374 * A new route can be allocated the next time
1375 * output is attempted.
1381 * Lookup a PCB based on the local address and port.
1383 static struct inpcb
*
1384 in_pcblookup_local(struct inpcbporthead
*porthash
, struct in_addr laddr
,
1385 u_int lport_arg
, int wild_okay
, struct ucred
*cred
)
1388 int matchwild
= 3, wildcard
;
1389 u_short lport
= lport_arg
;
1390 struct inpcbport
*phd
;
1391 struct inpcb
*match
= NULL
;
1394 * If the porthashbase is shared across several cpus, it must
1397 ASSERT_PORTHASH_TOKEN_HELD(porthash
);
1400 * Best fit PCB lookup.
1402 * First see if this local port is in use by looking on the
1405 LIST_FOREACH(phd
, porthash
, phd_hash
) {
1406 if (phd
->phd_port
== lport
)
1411 * Port is in use by one or more PCBs. Look for best
1414 LIST_FOREACH(inp
, &phd
->phd_pcblist
, inp_portlist
) {
1417 if (!INP_ISIPV4(inp
))
1420 if (inp
->inp_faddr
.s_addr
!= INADDR_ANY
)
1422 if (inp
->inp_laddr
.s_addr
!= INADDR_ANY
) {
1423 if (laddr
.s_addr
== INADDR_ANY
)
1425 else if (inp
->inp_laddr
.s_addr
!= laddr
.s_addr
)
1428 if (laddr
.s_addr
!= INADDR_ANY
)
1431 if (wildcard
&& !wild_okay
)
1433 if (wildcard
< matchwild
&&
1436 inp
->inp_socket
->so_cred
->cr_prison
)) {
1438 matchwild
= wildcard
;
1439 if (matchwild
== 0) {
1449 in_pcblocalgroup_last(const struct inpcbinfo
*pcbinfo
,
1450 const struct inpcb
*inp
)
1452 const struct inp_localgrphead
*hdr
;
1453 const struct inp_localgroup
*grp
;
1456 if (pcbinfo
->localgrphashbase
== NULL
)
1459 GET_PCBINFO_TOKEN(pcbinfo
);
1461 hdr
= &pcbinfo
->localgrphashbase
[
1462 INP_PCBLOCALGRPHASH(inp
->inp_lport
, pcbinfo
->localgrphashmask
)];
1464 LIST_FOREACH(grp
, hdr
, il_list
) {
1465 if (grp
->il_af
== inp
->inp_af
&&
1466 grp
->il_lport
== inp
->inp_lport
&&
1467 memcmp(&grp
->il_dependladdr
,
1468 &inp
->inp_inc
.inc_ie
.ie_dependladdr
,
1469 sizeof(grp
->il_dependladdr
)) == 0) {
1473 if (grp
== NULL
|| grp
->il_inpcnt
== 1) {
1474 REL_PCBINFO_TOKEN(pcbinfo
);
1478 KASSERT(grp
->il_inpcnt
>= 2,
1479 ("invalid localgroup inp count %d", grp
->il_inpcnt
));
1480 for (i
= 0; i
< grp
->il_inpcnt
; ++i
) {
1481 if (grp
->il_inp
[i
] == inp
) {
1482 int last
= grp
->il_inpcnt
- 1;
1485 last
= grp
->il_inpcnt
- 2;
1486 REL_PCBINFO_TOKEN(pcbinfo
);
1487 return grp
->il_inp
[last
];
1490 REL_PCBINFO_TOKEN(pcbinfo
);
1494 static struct inpcb
*
1495 inp_localgroup_lookup(const struct inpcbinfo
*pcbinfo
,
1496 struct in_addr laddr
, uint16_t lport
, uint32_t pkt_hash
)
1498 struct inpcb
*local_wild
= NULL
;
1499 const struct inp_localgrphead
*hdr
;
1500 const struct inp_localgroup
*grp
;
1502 ASSERT_PCBINFO_TOKEN_HELD(pcbinfo
);
1504 hdr
= &pcbinfo
->localgrphashbase
[
1505 INP_PCBLOCALGRPHASH(lport
, pcbinfo
->localgrphashmask
)];
1508 * Order of socket selection:
1512 * NOTE: Local group does not contain jailed sockets
1514 LIST_FOREACH(grp
, hdr
, il_list
) {
1516 if (grp
->il_af
!= AF_INET
)
1519 if (grp
->il_lport
== lport
) {
1523 * Modulo-N is used here, which greatly reduces
1524 * completion queue token contention, thus more
1525 * cpu time is saved.
1527 idx
= pkt_hash
% grp
->il_inpcnt
;
1528 if (grp
->il_laddr
.s_addr
== laddr
.s_addr
)
1529 return grp
->il_inp
[idx
];
1530 else if (grp
->il_laddr
.s_addr
== INADDR_ANY
)
1531 local_wild
= grp
->il_inp
[idx
];
1534 if (local_wild
!= NULL
)
1540 * Lookup PCB in hash list.
1543 in_pcblookup_pkthash(struct inpcbinfo
*pcbinfo
, struct in_addr faddr
,
1544 u_int fport_arg
, struct in_addr laddr
, u_int lport_arg
,
1545 boolean_t wildcard
, struct ifnet
*ifp
, const struct mbuf
*m
)
1547 struct inpcbhead
*head
;
1548 struct inpcb
*inp
, *jinp
=NULL
;
1549 u_short fport
= fport_arg
, lport
= lport_arg
;
1552 * First look for an exact match.
1554 head
= &pcbinfo
->hashbase
[INP_PCBCONNHASH(faddr
.s_addr
, fport
,
1555 laddr
.s_addr
, lport
, pcbinfo
->hashmask
)];
1556 LIST_FOREACH(inp
, head
, inp_hash
) {
1558 if (!INP_ISIPV4(inp
))
1561 if (in_hosteq(inp
->inp_faddr
, faddr
) &&
1562 in_hosteq(inp
->inp_laddr
, laddr
) &&
1563 inp
->inp_fport
== fport
&& inp
->inp_lport
== lport
) {
1565 if (inp
->inp_socket
== NULL
||
1566 inp
->inp_socket
->so_cred
->cr_prison
== NULL
) {
1578 struct inpcb
*local_wild
= NULL
;
1579 struct inpcb
*jinp_wild
= NULL
;
1580 struct inpcontainer
*ic
;
1581 struct inpcontainerhead
*chead
;
1582 struct sockaddr_in jsin
;
1585 GET_PCBINFO_TOKEN(pcbinfo
);
1588 * Check local group first
1590 if (pcbinfo
->localgrphashbase
!= NULL
&&
1591 m
!= NULL
&& (m
->m_flags
& M_HASH
) &&
1592 !(ifp
&& ifp
->if_type
== IFT_FAITH
)) {
1593 inp
= inp_localgroup_lookup(pcbinfo
,
1594 laddr
, lport
, m
->m_pkthdr
.hash
);
1596 REL_PCBINFO_TOKEN(pcbinfo
);
1602 * Order of socket selection:
1603 * 1. non-jailed, non-wild.
1604 * 2. non-jailed, wild.
1605 * 3. jailed, non-wild.
1608 jsin
.sin_family
= AF_INET
;
1609 chead
= &pcbinfo
->wildcardhashbase
[
1610 INP_PCBWILDCARDHASH(lport
, pcbinfo
->wildcardhashmask
)];
1611 LIST_FOREACH(ic
, chead
, ic_list
) {
1613 if (inp
->inp_flags
& INP_PLACEMARKER
)
1616 jsin
.sin_addr
.s_addr
= laddr
.s_addr
;
1618 if (!INP_ISIPV4(inp
))
1621 if (inp
->inp_socket
!= NULL
)
1622 cred
= inp
->inp_socket
->so_cred
;
1625 if (cred
!= NULL
&& jailed(cred
)) {
1629 if (!jailed_ip(cred
->cr_prison
,
1630 (struct sockaddr
*)&jsin
))
1633 if (inp
->inp_lport
== lport
) {
1634 if (ifp
&& ifp
->if_type
== IFT_FAITH
&&
1635 !(inp
->inp_flags
& INP_FAITH
))
1637 if (inp
->inp_laddr
.s_addr
== laddr
.s_addr
) {
1638 if (cred
!= NULL
&& jailed(cred
)) {
1641 REL_PCBINFO_TOKEN(pcbinfo
);
1645 if (inp
->inp_laddr
.s_addr
== INADDR_ANY
) {
1646 if (cred
!= NULL
&& jailed(cred
))
1654 REL_PCBINFO_TOKEN(pcbinfo
);
1656 if (local_wild
!= NULL
)
1657 return (local_wild
);
1670 in_pcblookup_hash(struct inpcbinfo
*pcbinfo
, struct in_addr faddr
,
1671 u_int fport_arg
, struct in_addr laddr
, u_int lport_arg
,
1672 boolean_t wildcard
, struct ifnet
*ifp
)
1674 return in_pcblookup_pkthash(pcbinfo
, faddr
, fport_arg
,
1675 laddr
, lport_arg
, wildcard
, ifp
, NULL
);
1679 * Insert PCB into connection hash table.
1682 in_pcbinsconnhash(struct inpcb
*inp
)
1684 struct inpcbinfo
*pcbinfo
= inp
->inp_pcbinfo
;
1685 struct inpcbhead
*bucket
;
1686 u_int32_t hashkey_faddr
, hashkey_laddr
;
1689 if (INP_ISIPV6(inp
)) {
1690 hashkey_faddr
= inp
->in6p_faddr
.s6_addr32
[3] /* XXX JH */;
1691 hashkey_laddr
= inp
->in6p_laddr
.s6_addr32
[3] /* XXX JH */;
1694 hashkey_faddr
= inp
->inp_faddr
.s_addr
;
1695 hashkey_laddr
= inp
->inp_laddr
.s_addr
;
1700 KASSERT(&curthread
->td_msgport
== netisr_cpuport(pcbinfo
->cpu
),
1701 ("not in the correct netisr"));
1702 ASSERT_INP_NOTINHASH(inp
);
1703 inp
->inp_flags
|= INP_CONNECTED
;
1706 * Insert into the connection hash table.
1708 bucket
= &pcbinfo
->hashbase
[INP_PCBCONNHASH(hashkey_faddr
,
1709 inp
->inp_fport
, hashkey_laddr
, inp
->inp_lport
, pcbinfo
->hashmask
)];
1710 LIST_INSERT_HEAD(bucket
, inp
, inp_hash
);
1714 * Remove PCB from connection hash table.
1717 in_pcbremconnhash(struct inpcb
*inp
)
1719 struct inpcbinfo
*pcbinfo __debugvar
= inp
->inp_pcbinfo
;
1721 KASSERT(&curthread
->td_msgport
== netisr_cpuport(pcbinfo
->cpu
),
1722 ("not in the correct netisr"));
1723 KASSERT(inp
->inp_flags
& INP_CONNECTED
, ("inp not connected"));
1725 LIST_REMOVE(inp
, inp_hash
);
1726 inp
->inp_flags
&= ~INP_CONNECTED
;
1730 * Insert PCB into port hash table.
1733 in_pcbinsporthash(struct inpcbporthead
*pcbporthash
, struct inpcb
*inp
)
1735 struct inpcbinfo
*pcbinfo
= inp
->inp_pcbinfo
;
1736 struct inpcbport
*phd
;
1739 * If the porthashbase is shared across several cpus, it must
1742 ASSERT_PORTHASH_TOKEN_HELD(pcbporthash
);
1745 * Insert into the port hash table.
1748 /* Go through port list and look for a head for this lport. */
1749 LIST_FOREACH(phd
, pcbporthash
, phd_hash
) {
1750 if (phd
->phd_port
== inp
->inp_lport
)
1754 /* If none exists, use saved one and tack it on. */
1756 KKASSERT(pcbinfo
->portsave
!= NULL
);
1757 phd
= pcbinfo
->portsave
;
1758 pcbinfo
->portsave
= NULL
;
1759 phd
->phd_port
= inp
->inp_lport
;
1760 LIST_INIT(&phd
->phd_pcblist
);
1761 LIST_INSERT_HEAD(pcbporthash
, phd
, phd_hash
);
1764 inp
->inp_porthash
= pcbporthash
;
1766 LIST_INSERT_HEAD(&phd
->phd_pcblist
, inp
, inp_portlist
);
1769 * Malloc one inpcbport for later use. It is safe to use
1770 * "wait" malloc here (port token would be released, if
1771 * malloc ever blocked), since all changes to the porthash
1774 if (pcbinfo
->portsave
== NULL
) {
1775 pcbinfo
->portsave
= kmalloc(sizeof(*pcbinfo
->portsave
),
1776 M_PCB
, M_INTWAIT
| M_ZERO
);
1781 in_pcbinsporthash_lport(struct inpcb
*inp
)
1783 struct inpcbinfo
*pcbinfo
= inp
->inp_pcbinfo
;
1784 struct inpcbportinfo
*portinfo
;
1785 struct inpcbporthead
*porthash
;
1788 /* Locate the proper portinfo based on lport */
1789 lport_ho
= ntohs(inp
->inp_lport
);
1790 portinfo
= &pcbinfo
->portinfo
[lport_ho
& pcbinfo
->portinfo_mask
];
1791 KKASSERT((lport_ho
& pcbinfo
->portinfo_mask
) == portinfo
->offset
);
1793 porthash
= in_pcbporthash_head(portinfo
, inp
->inp_lport
);
1794 GET_PORTHASH_TOKEN(porthash
);
1795 in_pcbinsporthash(porthash
, inp
);
1796 REL_PORTHASH_TOKEN(porthash
);
1800 in_pcbremporthash(struct inpcb
*inp
)
1802 struct inpcbporthead
*porthash
;
1803 struct inpcbport
*phd
;
1805 if (inp
->inp_phd
== NULL
)
1807 KASSERT(inp
->inp_lport
!= 0, ("inpcb has no lport"));
1809 porthash
= inp
->inp_porthash
;
1810 GET_PORTHASH_TOKEN(porthash
);
1813 LIST_REMOVE(inp
, inp_portlist
);
1814 if (LIST_FIRST(&phd
->phd_pcblist
) == NULL
) {
1815 LIST_REMOVE(phd
, phd_hash
);
1819 REL_PORTHASH_TOKEN(porthash
);
1821 inp
->inp_phd
= NULL
;
1822 /* NOTE: Don't whack inp_lport, which may be used later */
1825 static struct inp_localgroup
*
1826 inp_localgroup_alloc(u_char af
, uint16_t port
,
1827 const union in_dependaddr
*addr
, int size
)
1829 struct inp_localgroup
*grp
;
1831 grp
= kmalloc(__offsetof(struct inp_localgroup
, il_inp
[size
]),
1832 M_TEMP
, M_INTWAIT
| M_ZERO
);
1834 grp
->il_lport
= port
;
1835 grp
->il_dependladdr
= *addr
;
1836 grp
->il_inpsiz
= size
;
1842 inp_localgroup_free(struct inp_localgroup
*grp
)
1848 inp_localgroup_destroy(struct inp_localgroup
*grp
)
1850 LIST_REMOVE(grp
, il_list
);
1851 inp_localgroup_free(grp
);
1855 inp_localgroup_copy(struct inp_localgroup
*grp
,
1856 const struct inp_localgroup
*old_grp
)
1860 KASSERT(old_grp
->il_inpcnt
< grp
->il_inpsiz
,
1861 ("invalid new local group size %d and old local group count %d",
1862 grp
->il_inpsiz
, old_grp
->il_inpcnt
));
1863 for (i
= 0; i
< old_grp
->il_inpcnt
; ++i
)
1864 grp
->il_inp
[i
] = old_grp
->il_inp
[i
];
1865 grp
->il_inpcnt
= old_grp
->il_inpcnt
;
1869 in_pcbinslocalgrphash_oncpu(struct inpcb
*inp
, struct inpcbinfo
*pcbinfo
)
1871 struct inp_localgrphead
*hdr
;
1872 struct inp_localgroup
*grp
, *grp_alloc
= NULL
;
1876 ASSERT_PCBINFO_TOKEN_HELD(pcbinfo
);
1878 if (pcbinfo
->localgrphashbase
== NULL
)
1882 * XXX don't allow jailed socket to join local group
1884 if (inp
->inp_socket
!= NULL
)
1885 cred
= inp
->inp_socket
->so_cred
;
1888 if (cred
!= NULL
&& jailed(cred
))
1891 hdr
= &pcbinfo
->localgrphashbase
[
1892 INP_PCBLOCALGRPHASH(inp
->inp_lport
, pcbinfo
->localgrphashmask
)];
1895 LIST_FOREACH(grp
, hdr
, il_list
) {
1896 if (grp
->il_af
== inp
->inp_af
&&
1897 grp
->il_lport
== inp
->inp_lport
&&
1898 memcmp(&grp
->il_dependladdr
,
1899 &inp
->inp_inc
.inc_ie
.ie_dependladdr
,
1900 sizeof(grp
->il_dependladdr
)) == 0) {
1906 * Create a new local group
1908 if (grp_alloc
== NULL
) {
1909 grp_alloc
= inp_localgroup_alloc(inp
->inp_af
,
1910 inp
->inp_lport
, &inp
->inp_inc
.inc_ie
.ie_dependladdr
,
1911 INP_LOCALGROUP_SIZMIN
);
1913 * Local group allocation could block and the
1914 * local group w/ the same property might have
1915 * been added by others when we were blocked;
1920 /* Local group has been allocated; link it */
1923 LIST_INSERT_HEAD(hdr
, grp
, il_list
);
1925 } else if (grp
->il_inpcnt
== grp
->il_inpsiz
) {
1926 if (grp
->il_inpsiz
>= INP_LOCALGROUP_SIZMAX
) {
1927 static int limit_logged
= 0;
1929 if (!limit_logged
) {
1931 kprintf("local group port %d, "
1932 "limit reached\n", ntohs(grp
->il_lport
));
1934 if (grp_alloc
!= NULL
) {
1936 * This would happen if the local group
1937 * w/ the same property was expanded when
1938 * our local group allocation blocked.
1940 inp_localgroup_free(grp_alloc
);
1946 * Expand this local group
1948 if (grp_alloc
== NULL
||
1949 grp
->il_inpcnt
>= grp_alloc
->il_inpsiz
) {
1950 if (grp_alloc
!= NULL
)
1951 inp_localgroup_free(grp_alloc
);
1952 grp_alloc
= inp_localgroup_alloc(grp
->il_af
,
1953 grp
->il_lport
, &grp
->il_dependladdr
,
1954 grp
->il_inpsiz
* 2);
1956 * Local group allocation could block and the
1957 * local group w/ the same property might have
1958 * been expanded by others when we were blocked;
1965 * Save the old local group, link the new one, and then
1966 * destroy the old local group
1968 inp_localgroup_copy(grp_alloc
, grp
);
1969 LIST_INSERT_HEAD(hdr
, grp_alloc
, il_list
);
1970 inp_localgroup_destroy(grp
);
1976 * Found the local group
1978 if (grp_alloc
!= NULL
) {
1980 * This would happen if the local group w/ the
1981 * same property was added or expanded when our
1982 * local group allocation blocked.
1984 inp_localgroup_free(grp_alloc
);
1989 KASSERT(grp
->il_inpcnt
< grp
->il_inpsiz
,
1990 ("invalid local group size %d and count %d",
1991 grp
->il_inpsiz
, grp
->il_inpcnt
));
1994 * Keep the local group sorted by the inpcb local group index
1995 * in ascending order.
1997 * This eases the multi-process userland application which uses
1998 * SO_REUSEPORT sockets and binds process to the owner cpu of
1999 * the SO_REUSEPORT socket:
2000 * If we didn't sort the local group by the inpcb local group
2001 * index and one of the process owning an inpcb in this local
2002 * group restarted, e.g. crashed and restarted by watchdog,
2003 * other processes owning a inpcb in this local group would have
2004 * to detect that event, refetch its socket's owner cpu, and
2007 idx
= grp
->il_inpcnt
;
2008 for (i
= 0; i
< idx
; ++i
) {
2009 struct inpcb
*oinp
= grp
->il_inp
[i
];
2011 if (oinp
->inp_lgrpindex
> i
) {
2012 if (inp
->inp_lgrpindex
< 0) {
2013 inp
->inp_lgrpindex
= i
;
2014 } else if (inp
->inp_lgrpindex
!= i
) {
2016 kprintf("inp %p: grpidx %d, "
2017 "assigned to %d, cpu%d\n",
2018 inp
, inp
->inp_lgrpindex
, i
,
2022 grp
->il_inp
[i
] = inp
;
2024 /* Pull down inpcbs */
2025 for (; i
< grp
->il_inpcnt
; ++i
) {
2026 struct inpcb
*oinp1
= grp
->il_inp
[i
+ 1];
2028 grp
->il_inp
[i
+ 1] = oinp
;
2036 if (inp
->inp_lgrpindex
< 0) {
2037 inp
->inp_lgrpindex
= idx
;
2038 } else if (inp
->inp_lgrpindex
!= idx
) {
2040 kprintf("inp %p: grpidx %d, assigned to %d, cpu%d\n",
2041 inp
, inp
->inp_lgrpindex
, idx
, mycpuid
);
2044 grp
->il_inp
[idx
] = inp
;
2049 in_pcbinswildcardhash_oncpu(struct inpcb
*inp
, struct inpcbinfo
*pcbinfo
)
2051 struct inpcontainer
*ic
;
2052 struct inpcontainerhead
*bucket
;
2054 GET_PCBINFO_TOKEN(pcbinfo
);
2056 in_pcbinslocalgrphash_oncpu(inp
, pcbinfo
);
2058 bucket
= &pcbinfo
->wildcardhashbase
[
2059 INP_PCBWILDCARDHASH(inp
->inp_lport
, pcbinfo
->wildcardhashmask
)];
2061 ic
= kmalloc(sizeof(struct inpcontainer
), M_TEMP
, M_INTWAIT
);
2063 LIST_INSERT_HEAD(bucket
, ic
, ic_list
);
2065 REL_PCBINFO_TOKEN(pcbinfo
);
2069 * Insert PCB into wildcard hash table.
2072 in_pcbinswildcardhash(struct inpcb
*inp
)
2074 struct inpcbinfo
*pcbinfo
= inp
->inp_pcbinfo
;
2076 KASSERT(&curthread
->td_msgport
== netisr_cpuport(pcbinfo
->cpu
),
2077 ("not in correct netisr"));
2078 ASSERT_INP_NOTINHASH(inp
);
2079 inp
->inp_flags
|= INP_WILDCARD
;
2081 in_pcbinswildcardhash_oncpu(inp
, pcbinfo
);
2085 in_pcbremlocalgrphash_oncpu(struct inpcb
*inp
, struct inpcbinfo
*pcbinfo
)
2087 struct inp_localgrphead
*hdr
;
2088 struct inp_localgroup
*grp
;
2090 ASSERT_PCBINFO_TOKEN_HELD(pcbinfo
);
2092 if (pcbinfo
->localgrphashbase
== NULL
)
2095 hdr
= &pcbinfo
->localgrphashbase
[
2096 INP_PCBLOCALGRPHASH(inp
->inp_lport
, pcbinfo
->localgrphashmask
)];
2098 LIST_FOREACH(grp
, hdr
, il_list
) {
2101 for (i
= 0; i
< grp
->il_inpcnt
; ++i
) {
2102 if (grp
->il_inp
[i
] != inp
)
2105 if (grp
->il_inpcnt
== 1) {
2106 /* Destroy this local group */
2107 inp_localgroup_destroy(grp
);
2109 /* Pull up inpcbs */
2110 for (; i
+ 1 < grp
->il_inpcnt
; ++i
)
2111 grp
->il_inp
[i
] = grp
->il_inp
[i
+ 1];
2120 in_pcbremwildcardhash_oncpu(struct inpcb
*inp
, struct inpcbinfo
*pcbinfo
)
2122 struct inpcontainer
*ic
;
2123 struct inpcontainerhead
*head
;
2125 GET_PCBINFO_TOKEN(pcbinfo
);
2127 in_pcbremlocalgrphash_oncpu(inp
, pcbinfo
);
2130 head
= &pcbinfo
->wildcardhashbase
[
2131 INP_PCBWILDCARDHASH(inp
->inp_lport
, pcbinfo
->wildcardhashmask
)];
2133 LIST_FOREACH(ic
, head
, ic_list
) {
2134 if (ic
->ic_inp
== inp
)
2137 REL_PCBINFO_TOKEN(pcbinfo
);
2138 return; /* not found! */
2141 LIST_REMOVE(ic
, ic_list
); /* remove container from bucket chain */
2142 REL_PCBINFO_TOKEN(pcbinfo
);
2143 kfree(ic
, M_TEMP
); /* deallocate container */
2147 * Remove PCB from wildcard hash table.
2150 in_pcbremwildcardhash(struct inpcb
*inp
)
2152 struct inpcbinfo
*pcbinfo
= inp
->inp_pcbinfo
;
2154 KASSERT(&curthread
->td_msgport
== netisr_cpuport(pcbinfo
->cpu
),
2155 ("not in correct netisr"));
2156 KASSERT(inp
->inp_flags
& INP_WILDCARD
, ("inp not wildcard"));
2158 in_pcbremwildcardhash_oncpu(inp
, pcbinfo
);
2159 inp
->inp_lgrpindex
= -1;
2160 inp
->inp_flags
&= ~INP_WILDCARD
;
2164 * Remove PCB from various lists.
2167 in_pcbremlists(struct inpcb
*inp
)
2169 in_pcbremporthash(inp
);
2170 if (inp
->inp_flags
& INP_WILDCARD
) {
2171 in_pcbremwildcardhash(inp
);
2172 } else if (inp
->inp_flags
& INP_CONNECTED
) {
2173 in_pcbremconnhash(inp
);
2176 if (inp
->inp_flags
& INP_ONLIST
)
2181 prison_xinpcb(struct thread
*td
, struct inpcb
*inp
)
2185 if (td
->td_proc
== NULL
)
2187 cr
= td
->td_proc
->p_ucred
;
2188 if (cr
->cr_prison
== NULL
)
2190 if (inp
->inp_socket
&& inp
->inp_socket
->so_cred
&&
2191 inp
->inp_socket
->so_cred
->cr_prison
&&
2192 cr
->cr_prison
== inp
->inp_socket
->so_cred
->cr_prison
)
2198 in_pcblist_global(SYSCTL_HANDLER_ARGS
)
2200 struct inpcbinfo
*pcbinfo_arr
= arg1
;
2201 int pcbinfo_arrlen
= arg2
;
2202 struct inpcb
*marker
;
2206 KASSERT(pcbinfo_arrlen
<= ncpus
&& pcbinfo_arrlen
>= 1,
2207 ("invalid pcbinfo count %d", pcbinfo_arrlen
));
2210 * The process of preparing the TCB list is too time-consuming and
2211 * resource-intensive to repeat twice on every request.
2214 if (req
->oldptr
== NULL
) {
2215 for (cpu
= 0; cpu
< pcbinfo_arrlen
; ++cpu
)
2216 n
+= pcbinfo_arr
[cpu
].ipi_count
;
2217 req
->oldidx
= (n
+ n
/8 + 10) * sizeof(struct xinpcb
);
2221 if (req
->newptr
!= NULL
)
2224 marker
= kmalloc(sizeof(struct inpcb
), M_TEMP
, M_WAITOK
|M_ZERO
);
2225 marker
->inp_flags
|= INP_PLACEMARKER
;
2228 * OK, now we're committed to doing something. Re-fetch ipi_count
2229 * after obtaining the generation count.
2233 for (cpu
= 0; cpu
< pcbinfo_arrlen
&& error
== 0; ++cpu
) {
2234 struct inpcbinfo
*pcbinfo
= &pcbinfo_arr
[cpu
];
2239 lwkt_migratecpu(cpu
);
2241 GET_PCBINFO_TOKEN(pcbinfo
);
2243 n
= pcbinfo
->ipi_count
;
2245 LIST_INSERT_HEAD(&pcbinfo
->pcblisthead
, marker
, inp_list
);
2247 while ((inp
= LIST_NEXT(marker
, inp_list
)) != NULL
&& i
< n
) {
2248 LIST_REMOVE(marker
, inp_list
);
2249 LIST_INSERT_AFTER(inp
, marker
, inp_list
);
2251 if (inp
->inp_flags
& INP_PLACEMARKER
)
2253 if (prison_xinpcb(req
->td
, inp
))
2256 bzero(&xi
, sizeof xi
);
2257 xi
.xi_len
= sizeof xi
;
2258 bcopy(inp
, &xi
.xi_inp
, sizeof *inp
);
2259 if (inp
->inp_socket
)
2260 sotoxsocket(inp
->inp_socket
, &xi
.xi_socket
);
2261 if ((error
= SYSCTL_OUT(req
, &xi
, sizeof xi
)) != 0)
2265 LIST_REMOVE(marker
, inp_list
);
2267 REL_PCBINFO_TOKEN(pcbinfo
);
2269 if (error
== 0 && i
< n
) {
2270 bzero(&xi
, sizeof xi
);
2271 xi
.xi_len
= sizeof xi
;
2273 error
= SYSCTL_OUT(req
, &xi
, sizeof xi
);
2281 lwkt_migratecpu(origcpu
);
2282 kfree(marker
, M_TEMP
);
2287 in_pcblist_global_ncpus2(SYSCTL_HANDLER_ARGS
)
2289 return in_pcblist_global(oidp
, arg1
, ncpus2
, req
);
2293 in_savefaddr(struct socket
*so
, const struct sockaddr
*faddr
)
2295 struct sockaddr_in
*sin
;
2297 KASSERT(faddr
->sa_family
== AF_INET
,
2298 ("not AF_INET faddr %d", faddr
->sa_family
));
2300 sin
= kmalloc(sizeof(*sin
), M_SONAME
, M_WAITOK
| M_ZERO
);
2301 sin
->sin_family
= AF_INET
;
2302 sin
->sin_len
= sizeof(*sin
);
2303 sin
->sin_port
= ((const struct sockaddr_in
*)faddr
)->sin_port
;
2304 sin
->sin_addr
= ((const struct sockaddr_in
*)faddr
)->sin_addr
;
2306 so
->so_faddr
= (struct sockaddr
*)sin
;
2310 in_pcbportinfo_init(struct inpcbportinfo
*portinfo
, int hashsize
,
2313 memset(portinfo
, 0, sizeof(*portinfo
));
2315 portinfo
->offset
= offset
;
2316 portinfo
->lastport
= offset
;
2317 portinfo
->lastlow
= offset
;
2318 portinfo
->lasthi
= offset
;
2320 portinfo
->porthashbase
= hashinit(hashsize
, M_PCB
,
2321 &portinfo
->porthashmask
);
2325 in_pcbportrange(u_short
*hi0
, u_short
*lo0
, u_short ofs
, u_short step
)
2335 hi
= rounddown2(hi
, step
);
2340 lo
= roundup2(lo
, step
);
2350 in_pcbglobalinit(void)
2354 in_pcbmarkers
= kmalloc(ncpus
* sizeof(struct inpcb
), M_PCB
,
2356 in_pcbcontainer_markers
= kmalloc(ncpus
* sizeof(struct inpcontainer
),
2357 M_PCB
, M_WAITOK
| M_ZERO
);
2359 for (cpu
= 0; cpu
< ncpus
; ++cpu
) {
2360 struct inpcontainer
*ic
= &in_pcbcontainer_markers
[cpu
];
2361 struct inpcb
*marker
= &in_pcbmarkers
[cpu
];
2363 marker
->inp_flags
|= INP_PLACEMARKER
;
2364 ic
->ic_inp
= marker
;
2369 in_pcbmarker(int cpuid
)
2371 KASSERT(cpuid
>= 0 && cpuid
< ncpus
, ("invalid cpuid %d", cpuid
));
2372 KASSERT(curthread
->td_type
== TD_TYPE_NETISR
, ("not in netisr"));
2374 return &in_pcbmarkers
[cpuid
];
2377 struct inpcontainer
*
2378 in_pcbcontainer_marker(int cpuid
)
2380 KASSERT(cpuid
>= 0 && cpuid
< ncpus
, ("invalid cpuid %d", cpuid
));
2381 KASSERT(curthread
->td_type
== TD_TYPE_NETISR
, ("not in netisr"));
2383 return &in_pcbcontainer_markers
[cpuid
];
2387 in_pcbresetroute(struct inpcb
*inp
)
2389 struct route
*ro
= &inp
->inp_route
;
2391 if (ro
->ro_rt
!= NULL
)
2393 bzero(ro
, sizeof(*ro
));
2397 in_pcblastport_down(volatile u_short
*lastport
, u_short first
, u_short last
,
2406 lport
= olport
- step
;
2407 if (__predict_false(lport
> first
|| lport
< last
))
2409 if (atomic_cmpset_short(lastport
, olport
, lport
))
2416 in_pcblastport_up(volatile u_short
*lastport
, u_short first
, u_short last
,
2425 lport
= olport
+ step
;
2426 if (__predict_false(lport
< first
|| lport
> last
))
2428 if (atomic_cmpset_short(lastport
, olport
, lport
))